# This script replicates changelists between 2 Perforce servers # Copyright (c) 2009 Robert Cowham, robert@vizim.com require "P4" # This should be official P4Perl 2007.3 or later should work require "getoptlong" require "net/smtp" iniFile = '' @@g_trace = 0 def trace(*args) if @@g_trace > 0 then $stdout.puts args $stdout.flush end end class IniFile attr_reader :file; attr_accessor :sections; def initialize(file = '') if !file.empty? and !test(?f, file) then raise "'#{file}' does not exist or is not a plain file.\n" end @file = file || ''; @config = Hash.new; unless @file.empty? then _parseIniFile; end end def to_s() result = ''; @config.keys.sort.each do |key| result.concat("#{key}="); if @config[key].is_a? Array then result.concat(@config[key].join("\n\t") + "\n\n") else result.concat(@config[key] + "\n\n"); end end return result; end def [](k) return @config[k]; end # --- Private methods --- private; # .ini parser def _parseIniFile currentKey = nil; open(@file, 'r') do |fp| line = ''; until fp.eof? if line.empty? then line = fp.gets; else tmp = fp.gets; line.concat(tmp[2, tmp.size - 2]); end # if line.empty? line.chomp!; line, currentKey = _parseLine(line, currentKey); if line != nil then next; end line = ''; end # until fp.eof? end # open end # _parseIniFile # Line parser def _parseLine(line, currentKey) case line when /^\s*\#|^\s*$/ ; # Ignore comment lines when /^([^=]+?)=/ value = $'; key = $1; @config[key] = value currentKey = '' when /^([^:]+?):/ # Views have terminating ":" value = $'; key = $1; currentKey = key @config[key] = Array.new when /^\s+(\S+.*)$/ # continuation of view view = $1 if !/view/.match(currentKey) then raise "view line defined out of order.\n"; end @config[currentKey] << view when '' currentKey = nil; when ';' ; end # case line return nil, currentKey; end end # class IniFile def sendMail(server, from, to, subject, text) raise "mail server not specified" if server.length == 0 raise "sender address not specified" if from.length == 0 raise "receiver address not specified" if to.length == 0 smtp = Net::SMTP.new(server) smtp.start() smtp.ready(from, to) do |wa| wa.write("From: #{from}\r\n") wa.write("Reply-To: #{from}\r\n") wa.write("To: #{to.join(',')}\r\n") wa.write("Subject: #{subject}\r\n") wa.write("\r\n") wa.write("#{text}\r\n") wa.write("\r\n") end # smtp.ready(...) end # def sendMail() class ConfigException < Exception attr_reader :param def initialize(param) @param = param end end class Counter attr_reader :value def initialize(p4, name) @p4 = p4 @name = name counter = p4.run_counter(name)[0] # different handling of P4D 2005.1 (if) and P4D 2005.2 (elseif) if counter.is_a? String @value = counter.to_i elsif counter.is_a? Hash @value = counter['value'].to_i else raise "Unknown counter result value returned: #{counter.class}" end end def incr @value = @value + 1 @p4.run_counter(@name, @value) end def set(value) @value = value @p4.run_counter(@name, @value) end end class Change def initialize(p4src, p4dest, change_number) @p4src = p4src @p4dest = p4dest @change_number = change_number @ignored_files = Array.new @src_change = @p4src.run("describe", "-s", change_number)[0] @depot_files = @src_change["depotFile"] @fstat = @p4src.run("fstat", "-l", "@#{change_number},#{change_number}") if !@depot_files.nil? then trace "Change depot_files:\n", @depot_files actions = @src_change["action"] i = 0 while i < @depot_files.size open_file(@depot_files[i], actions[i], @fstat[i]) i += 1 end end end def submit if !@depot_files.nil? then change = @p4dest.fetch_change extra_desc = "" if @ignored_files.size > 0 then extra_desc = "\nIgnored files from source changelist:\n" + @ignored_files.join("\n") end change["Description"] = "Replicated change #{@src_change['change']} " + "from server: '#{@p4src.port}' \n" + "User: " + @src_change["user"] + " Time: " + Time.at(@src_change["time"].to_i).strftime("%Y/%m/%d %H:%M:%S") + "\n" + "Original Description:\n\n" + @src_change["desc"] + extra_desc result = @p4dest.save_submit(change) trace "Submit result:\n" + result.join("\n") # Also edit Source change - requires admin privileges # change_no = '' # result.reverse.each do |line| # if /^Change (\d+) submitted.$/.match(line) then # change_no = $1 # break # elsif /^Change \d+ renamed change (\d+) and submitted.$/.match(line) then # change_no = $1 # break # end # end # if change_no == '' then # raise "Couldn't find change no in submitted change:\n" + result.join("\n") # end # src_update = @p4src.fetch_change(@src_change["change"]) # src_update["Description"] = "Replicated by change #{change_no} " + # "on server: '#{@p4dest.port}'\n" + src_update["Description"] # result = @p4src.save_change(src_update, "-f") trace "Src not changed update result:\n" + result.join("\n") end end private def open_file(depot_file_name, action, fstat) trace "fstat result:\n" + fstat.to_a.join(" ") # handle files in changelist but not in replication view if !fstat.has_key?("clientFile") || fstat["clientFile"] == "" then @ignored_files << depot_file_name return end local_file_name = fstat["clientFile"] file_type = fstat["headType"] trace "Action '#{action}' on '#{local_file_name}'\n" case action when 'add', 'branch', 'import' result = @p4dest.run("add", "-t" +file_type, local_file_name) trace "Action result: \n" + result.join("\n") if result.join(" ") =~ /can\'t add existing file/ then raise "Error adding existing file '#{local_file_name}'" end when 'edit', 'integrate' result = @p4dest.run("edit", "-t" +file_type, local_file_name) trace "Action result: \n" + result.join("\n") when 'delete' result = @p4dest.run("delete", local_file_name) trace "Action result: \n" + result.join("\n") else raise "Unknown action #{action}" end end end class View attr_reader :src attr_reader :depot_path attr_reader :dest attr_reader :view def initialize(view_line, client_name, is_src) # Parse for standard view lines - dealing with quotes if /^\s*(\S*)\s+(\S*)\s*$/.match(view_line) then parse_details($1, $2) elsif /^\s*\"([^\"]*)\"\s+(\S*)\s*$/.match(view_line) then parse_details($1, $2) elsif /^\s*\"([^\"]*)\"\s+\"([^\"]*)\"\s*$/.match(view_line) then parse_details($1, $2) elsif /^\s*(\S*)\s+\"([^\"]*)\"\s*$/.match(view_line) then parse_details($1, $2) else raise "Problem with view: #{view_line}" end # construct view if is_src then lhs = @src else # dest lhs = @dest end @view = "\"" + lhs + "\" \"//" + client_name + "/" + @depot_path + "\"" end private def parse_details(src, dest) @src = src @dest = dest # dg 2006-12-05 # allow exclude lines in view: -//depot/.... if /^\-?\/\/(.*)$/.match(@src) then @depot_path = $1 else raise "Invalid src part of view: #{@src}" end end end class Client attr_reader :name attr_reader :root def initialize(p4, root, view, is_src) @name = p4.client @root = root @client = p4.fetch_client(@name) trace "Client:\n" + @client.to_a.join("\n") @client["Root"] = @root # turn on clobber for src if is_src then @client["Options"].sub!("noclobber", "clobber") end # Now copy the views into the client - overwriting what's there @client["View"] = Array.new view.each do |view_line| v = View.new(view_line, @name, is_src) @client["View"] << v.view end p4.save_client(@client) end end def replicate(iniFile) begin p4src = P4.new p4dest = P4.new # Check on required parameters required_params = ["src_p4port", "src_p4client", "src_p4clientroot", "src_p4user", "dest_counter", "view", "dest_p4port", "dest_p4client", "dest_p4user"] required_params.each do |p| if iniFile[p].nil? then raise ConfigException.new(p) end end p4src.port = iniFile["src_p4port"] p4src.client = iniFile["src_p4client"] p4src.user = iniFile["src_p4user"] p4src.password = iniFile["src_p4password"] if !iniFile["src_p4password"].nil? p4src.exception_level = 1 p4src.connect p4dest.port = iniFile["dest_p4port"] p4dest.client = iniFile["dest_p4client"] p4dest.user = iniFile["dest_p4user"] p4dest.password = iniFile["dest_p4password"] if !iniFile["dest_p4password"].nil? p4dest.exception_level = 1 p4dest.connect counter = Counter.new(p4dest, iniFile["dest_counter"]) trace "P4Counter #{counter.value}\n" dest_client = Client.new(p4dest, iniFile["src_p4clientroot"], iniFile["view"], false) src_client = Client.new(p4src, iniFile["src_p4clientroot"], iniFile["view"], true) trace "Src_client: #{src_client.name}\n" trace "Src_client Root dir: #{src_client.root}\n" # Check for partial runs and get user to reset status opened = p4dest.run("opened") if opened.size > 0 then raise "Error - files are currently open in destination client:\n" + opened.join("\n") + "\nPlease revert files\n" end # dg: 22.06.06 # retrieve all relevant changelists in a single call and store # the numbers in an array. processing afterwards is by # iterating over this array without further calls to # 'p4 changes' # relevant_changes = Array.new() begin range = '' if iniFile["single_changelist"].nil? then # get all changelists from countervalue to current head result = p4src.run("changes", "-ssubmitted", "//" + src_client.name + "/...\@" + counter.value.to_s + ",#head") else # enable replication of only 1 single changelist by use of the # keyword single_changelist= in the inifile # replicate only a single changelist, so filter submitted on this cl trace "Limit replication to changelist #" + iniFile["single_changelist"].to_s trace "Do not use counter" result = p4src.run("changes", "-ssubmitted", "//" + src_client.name + "/...\@" + iniFile["single_changelist"].to_s + ",\@" + iniFile["single_changelist"].to_s) end # p4 server reports newest cl first, so use reverse order result.reverse_each do |cl| relevant_changes.push(cl["change"]) end rescue relevant_changes = Array.new() end trace "Relevant: " + relevant_changes.size.to_s relevant_changes.each do |rc| trace " CL: " + rc.to_s end changelist_count = 0 result = p4src.run("sync", "#none") result = p4dest.run("flush") max_client_change = 0 relevant_changes.each do |counter_value| # the list only contains relevant changes # so no further checks are necessary trace "Process changelist:" + counter_value.to_s trace "Syncing to @#{counter_value.to_s}\n" # Sync the content of the specific changelist only result = p4src.run("sync", "@" + counter_value.to_s + "," + counter_value.to_s) trace "Result of sync:\n" + result.join("\n") # dg: 2006-06-20# # always replicate because syncing to a single CL which only # deletes files does not give a visible result on the client # but it must be replicated# # ## Only replicate a change if the sync did something... # #if result.size > 0 then change = Change.new(p4src, p4dest, counter_value) change.submit changelist_count += 1 # #end trace "Synced: #{result.size} files\n" if iniFile["single_changelist"].nil? then new_counter = counter_value.to_i new_counter += 1 counter.set(new_counter) end max_client_change = counter_value end # Remove synced files from client (free space of large syncs) result = p4src.run("sync", "#none") p4src.disconnect p4dest.disconnect return max_client_change, changelist_count rescue ConfigException => ce $stderr.print "Config parameter #{ce.param} missing!" rescue P4Exception # If any errors occur, we'll jump in here. Just log them # and raise the exception up to the higher level p4src.errors.each { |e| $stderr.puts( e ) } p4dest.errors.each { |e| $stderr.puts( e ) } raise end end if __FILE__ == $0 # process the parsed options opts = GetoptLong.new( [ "--verbose", "-v", GetoptLong::NO_ARGUMENT ] ) opts.each do |opt, arg| if opt == '--verbose' then @@g_trace = 1 end end if ARGV[0] config_file = ARGV[0] else config_file = "p4replicate.ini" end begin iniFile = IniFile.new(config_file) last_change_number, changes_replicated = replicate(iniFile) if changes_replicated > 0 then msg = "Number of changes replicated: #{changes_replicated}\n" + "Last change processed: #{last_change_number}\n" print msg titl = "p4replicate: " + config_file + "; changes: #{changes_replicated}; last: #{last_change_number};" if !iniFile["mail_success_to"].nil? then sendMail(iniFile["mail_server"], iniFile["mail_from"], iniFile["mail_success_to"].split(","), titl, msg) end end rescue => err msg = "Error in script: #{err}\n\n" + "-----------------------\n" + "Stacktrace for support:\n" + $!.backtrace.join("\n") $stderr.puts msg if !iniFile["mail_fail_to"].nil? then sendMail(iniFile["mail_server"], iniFile["mail_from"], iniFile["mail_fail_to"].split(","), "p4replicate FAILURE with ini file: " + config_file, msg) end end end
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#5 | 7188 | Robert Cowham |
Optimize the fstats into a single call. Rename src_counter to dest-counter |
||
#4 | 6447 | Robert Cowham | Updated to use official P4Ruby | ||
#3 | 5965 | Robert Cowham |
Changes from Daniel Galbavy HARMAN/BECKER Automotive Systems GmbH - Now there is no longer a loop over every changelist number with a server call for each number. The server call is only done once at the beginning and the changelists to be replicated are collected in an array on which the loop runs afterwards. - Additional I added a possible keyword to the ini file to replicate only one single changelist if necessary. - Accepting exclude lines - We also reduced sync data amount so that only those files are synced which have to be replicated. After that we had to remove the check if at least one file has been synced because otherwise changelists with only 'delete' actions were not replicated. - And we remove the files from the client after replicating. This saves a lot of disk space between the runs. |
||
#2 | 5159 | Robert Cowham | Tidied up docs. | ||
#1 | 5148 | Robert Cowham | Added replicate script |