- #!/usr/bin/env ruby
- # NOTE: This was a prototype script created by Alan Teague for validating the
- # Helix Sync have table algorithm.
- require 'P4'
- require 'pp'
- require 'json'
- require 'digest'
- require 'set'
- $have_table_file = '.have_table'
- $device_client = ''
- $shelf_client = ENV['SHELF_CLIENT']
- $shelf_change = ''
- $shelf_gen = ''
- $shelf_working = {}
- $shelf_contents = {}
- $shelf_submitted = false
- $have_table = {}
- $last_change = ''
- $depot_sync = {}
- $client_sync = {}
- $depot_rec = {}
- def spew( msg, obj )
- if obj != nil and !obj.empty? then
- puts(msg)
- pp(obj)
- end
- end
- def local_path( p4, f )
- where = p4.run_where([f])
- if where != nil then
- return where[0]['path']
- else
- return nil
- end
- end
- def grab_shelf( p4 )
- # take over shelf ownership
- if $shelf_change != '' then
- change = p4.fetch_change($shelf_change)
- change["Client"] = $device_client
- p4.save_change( change )
- end
- end
- def release_shelf( p4 )
- # take over shelf ownership
- if $shelf_change != '' then
- change = p4.fetch_change($shelf_change)
- change["Client"] = $shelf_client
- p4.save_change( change )
- end
- end
- def create_shelf( p4 )
- change = p4.fetch_change
- change['Description'] = '{"shelfGen":"0", "working":{}}'
- change["Client"] = $shelf_client
- msg = p4.save_change( change )
- changes = p4.run_changes( ['-m1', "-c", "#{$shelf_client}"] )
- $shelf_change = changes[0]['change']
- end
- def load_have_table()
- if File.exist?($have_table_file) then
- File.open($have_table_file, "r") do |f|
- $have_table = JSON.load(f)
- $shelf_change = $have_table['shelfNum']
- end
- end
- end
- def get_last_change( p4 )
- changes = p4.run_changes(['-m1', "//#{$device_client}/..."])
- if !changes.empty? then
- $last_change = changes[0]['change']
- else
- $last_change = '0'
- end
- end
- def load_shelf( p4 )
- if $shelf_change == '' then
- changes = p4.run_changes( ["-m1", "-s", "shelved","-c", "#{$shelf_client}"] )
- if !changes.empty? then
- $shelf_change = changes[0]['change']
- else
- return
- end
- end
- shelf = p4.run_describe( ['-S', '-s', "#{$shelf_change}"] )
- if shelf[0] == nil then
- # p4 change -o -O 2 -s
- puts("Searching for shelf")
- change = p4.run_change( ["-o", "-s", "-O", "#{$shelf_change}"] )
- pp(change)
- if change != nil and !change.empty? then
- $shelf_change = change[0]['Change']
- puts("...found #{$shelf_change}")
- shelf = p4.run_describe( ['-S', '-s', "#{$shelf_change}"] )
- else
- puts("Bad news, cannot resolve previous shelf")
- end
- end
- shelf = shelf[0]
- # Extract details from shelf desc
- if shelf['status'] == "submitted" then
- $shelf_submitted = true
- end
- shelf_desc = shelf['desc']
- s = JSON.parse(shelf_desc)
- $shelf_gen = s['shelfGen']
- $shelf_working = s['working']
- ['depotFile', 'action', 'type', 'rev', 'fileSize', 'digest', 'fromFile', 'fromRev'].each {
- |k|
- if !shelf.has_key?(k) then
- shelf[k] = Array.new
- end
- }
- shelf_raw = shelf['depotFile'].zip(
- shelf['action'],
- shelf['type'],
- shelf['rev'],
- shelf['fileSize'],
- shelf['digest'],
- shelf['fromFile'],
- shelf['fromRev'])
- $shelf_contents = Hash.new
- shelf_raw.each {
- |e|
- $shelf_contents[e[0]] = e[1..-1]
- }
- end
- def load_sync_files( p4 )
- files = p4.run_sync(['-n', "//#{$device_client}/...@#{$last_change}"])
- $depot_sync = Hash.new
- $client_sync = Hash.new
- files.each {
- |f|
- $depot_sync[f['depotFile']] = f
- $client_sync[f['clientFile']] = f
- }
- end
- def load_rec_files( p4 )
- files = p4.run_reconcile(['-n', "//#{$device_client}/..."])
- files.delete_if { |entry| entry.is_a? String }
- $depot_rec = Hash.new
- $depot_to_client_map = Hash.new
- files.each {
- |f|
- $depot_rec[f['depotFile']] = f
- if f['action'] == 'move/add' then
- f['action'] = 'add'
- elsif f['action'] == 'move/delete' then
- f['action'] = 'delete'
- end
- $depot_to_client_map[f['depotFile']] = f['clientFile']
- }
- end
- def save_have_table()
- have_table = Hash.new
- have_table['shelfGen'] = $shelf_gen
- have_table['shelfNum'] = $shelf_change
- have_table['lastChange'] = $last_change
- have_table['have'] = $shelf_working
- File.open($have_table_file, "w") do |f|
- f.write(JSON.pretty_generate(have_table))
- end
- end
- def update_disk( p4, updates )
- if updates.empty? then
- return
- end
- grab_shelf( p4 )
- args = ["-s", "#{$shelf_change}"]
- args.concat(updates.to_a)
- stuff = p4.run_unshelve(args)
- args = ["-k"]
- args.concat(updates.to_a)
- stuff = p4.run_revert(args)
- release_shelf( p4 )
- save_have_table
- end
- def update_shelf( p4, update_to_shelf, reverting )
- if update_to_shelf.empty? and reverting.empty?
- return
- end
- if $shelf_change == '' then
- create_shelf( p4 )
- end
- grab_shelf( p4 )
- updated_working = false
- updated_gen_number = false
- update_to_shelf.each {
- |f|
- stuff = p4.run([$depot_rec[f]['action'], "-c", $shelf_change, f])
- }
- if !update_to_shelf.empty? then
- stuff = p4.run_shelve(["-c", "#{$shelf_change}", "-f"])
- stuff = p4.run_revert(["-k", "//#{$device_client}/..."])
- load_shelf(p4)
- if !updated_gen_number then
- $shelf_gen = Integer($shelf_gen) + 1
- updated_gen_number = true
- end
- update_to_shelf.each {
- |f|
- fileSize = $shelf_contents[f][3]
- fileDigest = $shelf_contents[f][4]
- $shelf_working[f] = ["#{$shelf_gen}", "#{fileSize}", "#{fileDigest}", nil, Time.now.to_i]
- }
- updated_working = true
- end
- if !reverting.empty? then
- reverting.each {
- |f|
- if !$shelf_working[f][3] then
- if !updated_gen_number then
- $shelf_gen = Integer($shelf_gen) + 1
- updated_gen_number = true
- end
- $shelf_working[f][0] = $shelf_gen
- $shelf_working[f][3] = true
- p4.run_shelve(["-d", "-c", "#{$shelf_change}", f])
- end
- }
- updated_working = true
- end
- if updated_working then
- new_desc = Hash.new
- new_desc["shelfGen"] = $shelf_gen
- new_desc["working"] = $shelf_working
- change = p4.fetch_change($shelf_change)
- change["Description"] = JSON.generate(new_desc)
- p4.save_change( change )
- save_have_table
- end
- release_shelf( p4 )
- end
- p4 = P4.new
- $device_client = ENV['P4CLIENT']
- puts("pwd: #{Dir.pwd}")
- puts("\nDeviceSync:\nDevice: #{$device_client}\nShelf: #{$shelf_client}")
- begin
- p4.connect
- p4.exception_level = P4::RAISE_ERRORS
- info = p4.run_info
- puts("info: #{info}")
- # Get HAVE, WORKING(shelf), CONTENTS(shelf), SYNC-N, REC-N, and LAST_CHANGE
- # Run rec -n, then last_change, then sync -n
- # rec is not bound by change number so run it first
- # last_change is needed by sync -n
- load_rec_files( p4 )
- get_last_change( p4 )
- load_sync_files( p4 )
- # These two can run in either order and before or after the rec/sync loading
- load_have_table
- load_shelf( p4 )
- if $shelf_submitted and ($last_change < $shelf_change) then
- puts("Start over again, submit happened midstream")
- exit
- end
- # Review disk for changes from server or from shelf
- changed_on_disk = Set.new
- not_changed_on_disk = Set.new
- have = $have_table['have']
- if have == nil then
- have = Hash.new
- end
- $depot_rec.each {
- |f,e|
- if have.key?(f) then
- if have[f][1] == '' then
- if File.exist?(e['clientFile']) then
- puts("#{f}: exists now")
- changed_on_disk.add(f)
- else
- not_changed_on_disk.add(f)
- end
- elsif !File.exist?(e['clientFile']) then
- puts("#{f}: not exists")
- changed_on_disk.add(f)
- elsif Integer(have[f][1]) != File.size(e['clientFile']) or
- have[f][2] != String(Digest::MD5.file(e['clientFile'])).upcase! then
- if Integer(have[f][1]) != File.size(e['clientFile']) then
- puts("#{f}: filesize")
- pp(e['clientFile'])
- pp(File.size(e['clientFile']))
- else
- puts("#{f}: digest")
- pp(e['clientFile'])
- pp(String(Digest::MD5.file(e['clientFile'])))
- end
- changed_on_disk.add( f )
- else
- not_changed_on_disk.add( f )
- end
- else
- changed_on_disk.add( f )
- end
- }
- spew("\nChanged on disk:", changed_on_disk)
- spew("\nNot changed on disk:", not_changed_on_disk)
- changed_on_shelf = Set.new
- not_changed_on_shelf = Set.new
- $shelf_working.each {
- |f,e|
- if have.key?(f) then
- if e == have[f] then
- not_changed_on_shelf.add( f )
- else
- changed_on_shelf.add( f )
- end
- else
- changed_on_shelf.add( f )
- end
- }
- spew("\nChanged on shelf:", changed_on_shelf)
- spew("\nNot changed on shelf:", not_changed_on_shelf)
- update_to_shelf = Set.new
- shelf_conflict = Set.new
- changed_on_disk.each {
- |f|
- if not_changed_on_shelf.include?(f) then
- update_to_shelf.add(f)
- elsif changed_on_shelf.include?(f) then
- shelf_conflict.add(f)
- else
- update_to_shelf.add(f)
- end
- }
- spew("\nUpdate shelf with:", update_to_shelf)
- spew("\nShelf conflicts:", shelf_conflict)
- update_from_shelf = Set.new
- reverted_shelf = Set.new
- changed_on_shelf.each {
- |f|
- if not_changed_on_disk.include?(f) then
- if $shelf_working[f][3] then
- reverted_shelf.add(f)
- else
- update_from_shelf.add(f)
- end
- elsif !changed_on_disk.include?(f) then
- update_from_shelf.add(f)
- # else already in shelf_conflict
- end
- }
- spew("\nUpdate from shelf:", update_from_shelf)
- reverted_disk = Set.new
- reverted_conflicts = Set.new
- have.each {
- |f,e|
- if !$depot_rec.key?(f) then
- if not_changed_on_shelf.include?(f) then
- if !e[3] then
- reverted_disk.add(f)
- end
- elsif changed_on_shelf.include?(f) then
- reverted_conflicts.add(f)
- else
- # should never happen: HAVE record but no WORKING record
- reverted_disk.add(f)
- end
- end
- }
- spew("\nReverted shelf files:", reverted_shelf)
- spew("\nReverted disk files:", reverted_disk)
- spew("\nReverted conflicts:", reverted_conflicts)
- shelf_conflict.each {
- |f|
- puts("Conflict for #{f}")
- local_file = local_path(p4, f)
- if File.exist?(local_file) then
- conflict_name = "#{f}.conflict"
- File.rename(local_file, local_path(p4, conflict_name))
- update_to_shelf.add(conflict_name)
- $depot_rec[conflict_name] = Hash.new
- $depot_rec[conflict_name]['action'] = 'add'
- end
- update_from_shelf.add( f )
- }
- reverted_conflicts.each {
- |f|
- puts("Revert conflict for #{f}")
- update_from_shelf.add( f )
- }
- # if shelf was submitted
- if $shelf_submitted then
- puts("DRAFT - Handle resolving submitted shelf")
- # update_to_shelf CHANGED ON DISK @ MY KNOWN SHELF
- # reverted_disk NEED NEW VERSION, CLEAR HAVE
- # update_from_shelf NEED NEW VERSION, CLEAR HAVE
- # reverted_shelf NEED NEW VERSION, CLEAR HAVE
- # $depot_sync NEED THESE MINUS update_to_shelf
- spew("\nNeed to shelve:", update_to_shelf)
- File.delete($have_table_file)
- $depot_sync.delete_if{|k,v| update_to_shelf.include?(k)}
- # Need to rerun because there may already be a shelf
- # that needs to be resolved against
- puts("\nRerun deviceSync after this run completes")
- else
- update_shelf( p4, update_to_shelf, reverted_disk )
- update_disk( p4, update_from_shelf )
- $depot_sync.delete_if{|k,v| $shelf_working.include?(k) and !$shelf_working[k][3]}
- if reverted_shelf.empty? then
- puts("\nNo files to force sync")
- else
- puts("\nForce sync files:(if not in sync)")
- reverted_shelf.each {
- |f|
- if !$depot_sync.include?(f) then
- puts(f)
- lf = local_path(p4, f)
- if lf != nil and File.exist?(lf) then
- File.delete(lf)
- end
- stuff = p4.run_sync(["-f", "#{f}@#{$last_change}"])
- end
- }
- save_have_table
- end
- end
- if $depot_sync.empty? then
- puts("\nNo files to sync")
- else
- spew("\nSync files:", $depot_sync)
- $depot_sync.each {
- |f,v|
- p4.run_sync(["#{v['depotFile']}@#{$last_change}"])
- }
- end
- rescue P4Exception
- p4.errors.each { |e| pp( e ) }
- ensure
- p4.disconnect
- end
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#1 | 15845 | Doug Scheirer | Integ from main | 9 years ago | |
//guest/perforce_software/helix-web-services/main/source/contrib/deviceSync | |||||
#3 | 15821 | tjuricek | Clean up diagnostic information. | 9 years ago | |
#2 | 15820 | tjuricek | Set P4CONFIG=.p4config before executing deviceSync script. | 9 years ago | |
#1 | 15773 | tjuricek | Add basic 'add' test for the submit shelf mechanism. Added a *slightly* modified device...Sync script, which ideally functions somewhat like the helix sync script. We'll start with this to generate our JSON examples for testing. « |
9 years ago |