specsaver.rb #6

#!/usr/bin/ruby
#--
# ******************************************************************************
#++
# = Introduction
#
#  A Ruby script to detect changes to different types of spec ( client/label/
#  branch/etc. ) and to record the changes by checking the specs in as text
#  files into a dedicated area of the depot. Designed to be periodically
#  executed via either cron or the Windows scheduler depending on platform.
#
#  Note: This script REQUIRES a working build of P4Ruby which you can
#        get from the Perforce public depot in
#
#     //guest/tony_smith/perforce/API/Ruby/...
#
#        You'll also need the Perforce API for your platform to build P4Ruby.
#
#  specsaver.rb tries to be as efficient as possible in determining whether
#  or not an object has been modified. It doesn't depend on a blanket
#  "p4 revert -a" approach, but uses a counter and the built-in timestamps for
#  objects that have them.
#
# = Getting Started
#
#  The script creates its own clientspec so minimal setup is required. All
#  you need to do is to edit the variables in the configuration section and
#  ensure that the script has write access to the SPEC_PATH in the depot and
#  then you can run it.
#
#  If you want to archive privileged metadata like your protections table,
#  then specsaver needs to be run under a super-user account. If you're just
#  after clientspecs etc. then no special privileges are required.
#
# = Usage
#
# specsaver.rb [ options ]
#
# where options are:
#
#    --all      - archive all types of object
#    --branches - archive branch specs
#    --clients  - archive client specs
#    --config   - archive typemap, protections and jobspec
#    --depots   - archive depot definitions
#    --groups   - archive group specs
#    --jobs     - archive jobs (but not fixes)
#    --labels   - archive label views and files
#    --users    - archive user specs (not passwords)
#
# = Known Uglies
#
# Currently we only use one counter to store a timestamp for all types of
# object. This is occasionally a source of frustration and at some point
# I'll adapt the script to use a counter for each type of object. Until then,
# if you suspect that specsaver.rb isn't catching all the changed objects -
# usually the result of some human interference - then delete the counter or
# reset it to zero and let it figure it out for itself. It'll take a while,
# but it will get it right.
#
# To avoid the problem altogether always use the same arguments when invoking
# specsaver.rb. i.e. if you invoke it once with --all, then always use --all.
# If you do want to change your mind, then just delete or reset the counter.
#
# = Version
#
# $Id: //guest/robert_cowham/perforce/utils/specsaver.rb#6 $
#--
# ******************************************************************************
#++

# ------------------------------------------------------------------------------
#  CONFIGURATION SECTION
# ------------------------------------------------------------------------------

# Depot path under which specs should be versioned. This path should not
# already exist.
SPEC_PATH = "//software/imports/cvs_specs"

# Name of the counter to use to keep track of update runs. One counter is
# used for *all* types of spec.
SPEC_COUNTER  = "specsaver"

# Name of the field in your jobspec which is used as the update date
JOB_DATE_FIELD  = "Date"

# Perforce environment setup
P4USER    = "robert"
P4PORT    = "1666"
P4PASSWD  = "Password"
P4CLIENT  = "specsaver"

# Set the client root to a path that the script can use as its workspace
CLIENTROOT  = "c:\\temp\\specsaver_root"

# ------------------------------------------------------------------------------
#  END OF CONFIGURATION SECTION
# ------------------------------------------------------------------------------

require "P4"
require "getoptlong"

#
# Virtual base class for handling all types of spec. For each spec you want
# to manage, derive a class which must implement at least the following
# methods:
#
# list_specs  - list specs ("p4 clients"/"p4 labels" etc.)
# spec_file - Locate spec file in workspace
#
# Optionally, you may also want to override the changed? method used to
# determine whether or not a spec has changed since the last time it was
# archived. The default method uses the "Update" timestamp in the spec,
# but not all specs have this. If in doubt, changed? should just evaluate
# to true anyway, and let the "p4 revert -a" sort it out.
#
class SpecMgr

    public

    #
    # Constructor: Supply the client root, and two P4 instances, one with
    # tagged mode enabled, and one without.
    #
    def initialize( root, p4tagged, p4untagged )
        @root = root
        @p4t = p4tagged
        @p4u = p4untagged
        @modlist = Hash.new
    end

    #
    # Updates the archives for all specs of the specified type. This is the
    # main public interface method for this class
    #
    def update( type, since )
        list_specs.each do
            |spec|
            save_spec( type, spec ) if ( changed?( spec, since ) )
        end
        submit( description() )
    end

    protected

    #
    # Method to determine whether or not a spec has changed since the
    # specified timestamp. The default implementation uses the "Update"
    # field of the spec, but this is not available in all types of spec so
    # this method should be overridden for those specs. Where there is no
    # easy way to decide whether or not a spec has changed, override
    # implementations of this method should just return true and let the
    # "p4 revert -a" sort it all out.
    #
    def changed?( spec, stamp )
        spec[ "Update" ].to_i > stamp
    end

    #
    # Opens the specified file for add or edit as appropriate
    #
    def add_edit_file( path, name )
        fs = @p4t.run_fstat( path ).shift
        if ( fs )
            @p4t.run_edit( path )
        else
            @p4t.run_add( "-t", "text", path )
            fs = @p4t.run_fstat( path ).shift
        end
        @modlist[ fs[ "depotFile" ] ] = name
    end

    #
    # Compute the name of a spec from its type. Necessary because the
    # field for jobs is "Job" whilst for clients it's "client" ...
    #
    def type2name( type, spec )
        return spec[ type ] if spec.has_key?( type )
        return spec[ type.capitalize ] if spec.has_key?( type.capitalize )
        raise( RuntimeError, "Can't determine object name from type" )
    end

    #
    # Save the named spec into its text file version. As we don't want
    # the spec in parsed form for this we use the non-tagged P4 instance
    # here to run the "p4 xxxx -o" and the output of that command is written
    # to the workspace file.
    #
    def save_spec( type, spec )
        path = spec_file( spec )
        name = type2name( type, spec )
        add_edit_file( path, name )

        form = eval( %Q{@p4u.fetch_#{type}( "#{name}" )} )
        write_ws_file( path, form )
    end

    #
    # Method to write form data into the workspace file
    #
    def write_ws_file( path, form )
        File.open( path, "w+" ) do
            |file|
            file.write( form )
        end
    end

    #
    # Revert all unchanged files and remove them from the modlist.
    #
    def revert_unchanged()
        @p4t.run_revert( "-a" ).each do
            |r|
            r = r.sub( /\#\d+.*/, "" )
            @modlist.delete( r ) if @modlist.has_key?( r )
        end
    end

    #
    # Submit the changelist and write out a message including the
    # list of objects being updated.
    #
    def submit( desc )
        revert_unchanged()
        change = @p4t.fetch_change
        if ( change.has_key?( "Files" ) )
            change[ "Description" ] = desc
            @p4t.submit_spec( change )
            puts( desc + "\n\t" + @modlist.values.sort.join( "\n\t" ) )
        end
        @modlist = Hash.new
    end

end


#
# Generic spec manager class. Provides the base implementation for the
# list_specs() and spec_file() methods and a shorthand for the update() method
#
class GenSpecMgr < SpecMgr

    public

    #
    # Constructor. Provide the type of spec ("client", "label" etc.), the
    # client root and tagged and noon-tagged P4 client instances.
    #
    def initialize( type, root, p4t, p4u )
        @type = type
        super( root, p4t, p4u )
    end

    #
    # Shorthand wrapper around SpecMgr#update()
    #
    def update( since )
        super( @type, since )
    end

    protected

    #
    # Generate a list of all specs of the current type. The default
    # implementation just appends an "s" to the type name and uses that
    # as a P4 command. e.g. "p4 clients" etc. etc.
    #
    def list_specs
        eval( %Q{@p4t.run_#{@type}s} )
    end

    #
    # Map a spec type and name to a depot path where this spec will be
    # archived.
    #
    def spec_file( spec )
        @root + "/#{@type}s/" + type2name( @type, spec )
    end

    #
    # Provide a description for changelists and for stdout
    #
    def description
        "Archive updated #{@type}s"
    end
end


#
# GenSpecMgr subclass for Clients. Nothing special needed for clientspecs,
# they are very much the generic model.
#
class ClientMgr < GenSpecMgr
    #
    # Constructor. Provide the client root and tagged and non-tagged P4
    # client instances.
    #
    def initialize( root, p4t, p4u )
        super( "client", root, p4t, p4u )
    end
end

#
# GenSpecMgr subclass for Labels. Labels differ slightly from the generic
# model in that we also archive the file list for the label.
#
class LabelMgr < GenSpecMgr
    #
    # Constructor. Provide the client root and tagged and non-tagged P4
    # client instances.
    #
    def initialize( root, p4t, p4u )
        super( "label", root, p4t, p4u )
    end

    def clean_name( name )
        return name.gsub(/[<>:]/, "_")
    end
    #
    # Map a group name into a depot path. Can't use the default implementation
    # as we don't have a hash available, just a group name
    #
    def spec_file( spec )
        label = type2name( @type, spec )
        cleaned = clean_name(label)
        if cleaned != label
            puts "Label #{label} stored as #{cleaned}"
        end
        @root + "/#{@type}s/" + cleaned
    end

    #
    # Override SpecMgr#update() so that we can archive the file list as well
    # as the spec
    #
    def update( since )
        list_specs.each do
            |l|
            if ( changed?( l, since ) )
            print "Saving label #{l['label']}\n"
            save_spec( "label", l )
            save_files( l[ "label" ] )
            end
        end
        submit( description() )
    end

    protected

    #
    # Method to save the file list for a label. Appends ".files" to the label
    # name and saves the output of "p4 files //...@label" there.
    #
    def save_files( label )
        filename = @root + "/labels/" + clean_name(label) + ".files"
        files = @p4t.run_files( "//...@#{label}" ).collect do
            |l|
            l[ "depotFile" ] + "#" + l[ "rev" ]
        end
        
        add_edit_file( filename, label )
        write_ws_file( filename, files.join( "\n" ) )
    end
end


#
# GenSpecMgr subclass for Users. Nothing special required.
#
class UserMgr < GenSpecMgr
    #
    # Constructor. Provide the client root, and tagged and non-tagged P4
    # client instances
    #
    def initialize( root, p4t, p4u )
        super( "user", root, p4t, p4u )
    end
end


# GenSpecMgr subclass for Groups. Groups don't have an update date so we have
# to depend on revert -a doing the honours. "p4 groups" also doesn't
# support tagged output at the moment (2002.1) so what are hashes in
# the other classes are just group names in this one.

class GroupMgr < GenSpecMgr

    public

    #
    # Constructor. Provide the client root and tagged and non-tagged P4 client
    # instances
    #
    def initialize( root, p4t, p4u )
        super( "group", root, p4t, p4u )
    end

    protected

    #
    # Groups are always considered to have changed - until proven otherwise
    # so we override the GenSpecMgr implementation of changed? and always
    # return true
    #
    def changed?( group, since )
        true
    end

    #
    # Map a group name into a depot path. Can't use the default implementation
    # as we don't have a hash available, just a group name
    #
    def spec_file( group )
        @root + "/#{@type}s/" + group
    end

    #
    # Convert a type and a group name into a group name. Pretty simple really.
    #
    def type2name( type, group )
        group
    end
end

#
# GenSpecMgr subclass for Depots. "p4 depots" doesn't support tagged mode so
# we have to do some extra work.
#

class DepotMgr < GenSpecMgr

    public

    #
    # Constructor. Provide the client root and tagged and non-tagged P4 client
    # instances
    #
    def initialize( root, p4t, p4u )
        super( "depot", root, p4t, p4u )
    end

    protected

    #
    # Depots are always considered to have changed - until proven otherwise
    # so we just return true here.
    #
    def changed?( depot, since )
        true
    end

    #
    # Map a depot name to a path in the depot for archiving the spec
    #
    def spec_file( depot )
        @root + "/#{@type}s/" + depot
    end

    #
    # Convert a type and a depot name into a depot name. Pretty simple really
    #
    def type2name( type, depot )
        depot
    end

    #
    # Generate a list of depots. Needs custom implementation because
    # "p4 depots" doesn't support tagged mode.
    #
    def list_specs
        @p4t.run_depots.collect do
            |rec|
            rec.split[ 1 ]
        end
    end
end


#
# SpecMgr subclass for Branches. As the plural of branch is branchES, this
# needs slightly different handling so it's derived from SpecMgr, rather
# than GenSpecMgr
#
class BranchMgr < SpecMgr

    public

    #
    # Main interface method. Nothing special required, just shorthand for
    # the SpecMgr implementation.
    #
    def update( since )
        super( "branch", since )
    end

    protected

    #
    # Generate a list of branchspecs using "p4 branches"
    #
    def list_specs
        @p4t.run_branches
    end

    #
    # Map a branchspec to a depot path.
    #
    def spec_file( spec )
        @root + "/branches/" + spec[ "branch" ]
    end

    #
    # Provide the description for changelist and stdout
    #
    def description
        "Archive updated branches"
    end
end

#
# SpecMgr subclass for Jobs. Because you can have a huge number of
# jobs, we use "p4 jobs -e" to grab only the jobs that have changed since
# our saved timestamp. We use SpecMgr rather than GenSpecMgr for reasons
# I don't really remember.
#
class JobMgr < SpecMgr

    public

    #
    # Main interface method. Saves each of the modified jobs.
    #
    def update( since )
        list_specs( since ).each do
            |spec|
            save_spec( "job", spec )
        end
        submit( description() )
    end

    protected

    #
    # Get a list of jobs modified since the saved timestamp
    #
    def list_specs( stamp )
        @p4t.run_jobs( "-e", "#{JOB_DATE_FIELD} > #{stamp}" )
    end

    #
    # Locate the workspace file for a given job
    #
    def spec_file( spec )
        @root + "/jobs/" + spec[ "Job" ]
    end

    #
    # Provide the description for changelist and stdout.
    #
    def description
        "Archive updated jobs"
    end
end

# Subclass for one off's like jobspec and protections. These are a little
# different as "there can be only one!" We also can't use a timestamp to
# see if they've changed so we rely on "p4 revert -a".

class ConfigMgr < SpecMgr

    public

    #
    # Main interface. Update the archives for the named object using
    # the specified label. Note that "label" in this context doesn't
    # mean a Perforce label, but rather a user friendly description of
    # what we're archiving. i.e. "protections table" rather than "protect"
    #
    def update( name, label )
        path = @root + "/" + name
        add_edit_file( path, name )

        form = eval %Q{@p4u.fetch_#{name}()}
        write_ws_file( path, form )
        submit( "Archive updated #{label}" )
    end
end

#
# Main top level class for co-ordinating the run. Here we can trap
# Perforce exceptions and look after the overall environment.
#
class SpecSaver

    SPECS = [ "branches", "clients", "depots", "groups", "jobs", "labels", "users" ]

    public

    # Constructor
    def initialize

        @flags  = Hash.new

        # Perforce client for handling tagged output
        @p4t = P4.new
        @p4t.parse_forms
        @p4t.exception_level = 1
        @p4t.port   = P4PORT
        @p4t.user   = P4USER
        @p4t.password   = P4PASSWD
        @p4t.client     = P4CLIENT
        @p4t.connect

        # Perforce client for non-tagged output
        @p4u = P4.new
        @p4u.exception_level = 1
        @p4u.port   = P4PORT
        @p4u.user   = P4USER
        @p4u.password   = P4PASSWD
        @p4u.client     = P4CLIENT
        @p4u.connect

        # Set the client root
        @root       = CLIENTROOT

        init_client()
    end

    #
    # Set the debug level. Useful levels are:
    #
    #     0 (default)    - No debug output
    #     1              - Some debug output
    #     2              - More debug output
    #
    # What more could you want?
    #
    def debug( level )
        $stderr.puts( "Setting debug level to #{level}" )
        @p4u.disconnect
        @p4t.disconnect
        @p4u.debug = level
        @p4t.debug = level
        @p4u.connect
        @p4t.connect
    end

    #
    # Main interface method. Update all requested specs.
    #
    def update

        # Timestamp for this run
        now = Time.now.to_i

        # time of last update
        since = counter?

        begin
            branches = BranchMgr.new( @root, @p4t, @p4u )
            clients  = ClientMgr.new( @root, @p4t, @p4u )
            depots   = DepotMgr.new(  @root, @p4t, @p4u )
            groups   = GroupMgr.new(  @root, @p4t, @p4u )
            jobs     = JobMgr.new(    @root, @p4t, @p4u )
            labels   = LabelMgr.new(  @root, @p4t, @p4u )
            users    = UserMgr.new(   @root, @p4t, @p4u )
            config   = ConfigMgr.new( @root, @p4t, @p4u )

            branches.update( since )    if ( self.branches? )
            clients.update( since ) if ( self.clients?  )
            depots.update( since )  if ( self.depots?   )
            groups.update( since )  if ( self.groups?   )
            jobs.update( since )    if ( self.jobs?     )
            labels.update( since )  if ( self.labels?   )
            users.update( since )   if ( self.users?    )

            if ( self.config? )
                config.update( "protect", "protections" )
                config.update( "jobspec", "jobspec" )
                config.update( "typemap", "typemap" )
            end

            # Save the counter.
            self.counter = now
        rescue
            @p4t.errors.each{ |e| puts( e ) }
            @p4u.errors.each{ |e| puts( e ) }
            raise
        end
    end

    #
    # Test whether or not any flags at all have been set.
    #
    def have_flags?
        return ! @flags.empty?
    end

    #
    # Resolve missing methods by looking in the flags hash. This is
    # used to implement get and set methods for specifying which types
    # of spec should be archived. If the method name ends in a "?" it's a
    # getter, if not, it's a setter.
    #
    def method_missing( m, *args )
        method = m.to_s
        if ( method !~ /\?$/ )
            # Assignment
            return @flags[ method ] = true
        else
            method.gsub!( /\?$/, "" )
        end

        if ( @flags.has_key?( method ) )
            @flags[ method ]
        elsif ( @flags.has_key?( "all" ) )
            @flags[ "all" ]
        else
            false
        end
    end

    protected

    #
    # Initialise the client environment. Check whether or not the client
    # workspace exists, and if not, create it. Then create any missing
    # directories and sync to the head revision.
    #
    def init_client()
        begin
            @p4t.run_info.each do
            |line|
            create_client() if ( line['clientName'] =~ /unknown/ )
            end
            init_workspace()
            @p4t.run_sync
        rescue
            @p4t.errors.each { |e| puts( e ) }
            raise
        end
    end

    #
    # Initialises the workspace by pre-creating a directory for each spec
    # type ready for the workspace files to be written into.
    #
    def init_workspace()
        Dir.mkdir( @root ) unless ( File.exists?( @root ) )
        SPECS.each do
            |type|
            dir = @root + "/" + type
            Dir.mkdir( dir ) unless ( File.exists?( dir ) )
        end
    end

    #
    # Automatically create the client workspace if necessary. This
    # ensures that specsaver.rb is less prone to dodgy client views
    # than scripts that require the client to be pre-created.
    #
    def create_client()
        begin
            spec = @p4t.fetch_client
            spec[ "Description"     ] = "Client for spec versioning script"
            spec[ "Root"        ] = @root

            # Don't use String#gsub! it seems to cause problems
            opt = spec[ "Options" ].gsub( /normdir/, "rmdir" )
            spec[ "Options" ] = opt

            spec[ "View" ] = Array.new
            spec[ "View" ].push( SPEC_PATH + "/... //" + @p4t.client? + "/..." )

            @p4t.save_client( spec )
        rescue
            @p4t.errors.each { |e| puts( e ) }
            raise
        end
    end

    #
    # Get the value of the SPEC_COUNTER. This is used as a timestamp rather
    # than a traditional counter.
    #
    def counter?
        c = @p4t.run_counter( SPEC_COUNTER ).shift['value'].to_i
        return c
    end

    #
    # Update the SPEC_COUNTER with a new value.
    #
    def counter=( val )
        @p4t.run_counter( SPEC_COUNTER, val )
        true
    end

end


#
# Dump the usage information to stdout and exit.
#

def croak_syntax
    puts( %Q{

Usage: specsaver.rb [ options ]

where options are:

    --all - archive all types of object
    --branches  - archive branch specs
    --clients - archive client specs
    --config  - archive typemap, protections and jobspec
    --depots  - archive depot definitions
    --groups  - archive group specs
    --jobs  - archive jobs (but not fixes)
    --labels  - archive label views and files
    --users - archive user specs (not passwords)

} )
    exit( 0 )
end

# ------------------------------------------------------------------------------
#         START OF MAIN SCRIPT
# ------------------------------------------------------------------------------

# Parse command line options

begin
    opts = GetoptLong.new([ "--all",   GetoptLong::NO_ARGUMENT ],
        [ "--branches", GetoptLong::NO_ARGUMENT ],
        [ "--clients",  GetoptLong::NO_ARGUMENT ],
        [ "--config",   GetoptLong::NO_ARGUMENT ],
        [ "--debug",  GetoptLong::OPTIONAL_ARGUMENT ],
        [ "--depots",   GetoptLong::NO_ARGUMENT],
        [ "--groups",   GetoptLong::NO_ARGUMENT ],
        [ "--jobs",   GetoptLong::NO_ARGUMENT ],
        [ "--labels",   GetoptLong::NO_ARGUMENT ],
        [ "--users",    GetoptLong::NO_ARGUMENT ])

    s = SpecSaver.new
    opts.each do
        |opt,arg|
        s.all   if ( opt == "--all" )
        s.branches  if ( opt == "--branches" )
        s.clients if ( opt == "--clients" )
        s.config  if ( opt == "--config" )
        s.depots  if ( opt == "--depots" )
        s.groups  if ( opt == "--groups" )
        s.jobs  if ( opt == "--jobs" )
        s.labels  if ( opt == "--labels" )
        s.users if ( opt == "--users" )

        s.debug( arg.empty? ? 1 : arg.to_i ) if ( opt == "--debug" )
    end

    # Ignore flags if being run from scheduled tasks
    croak_syntax unless ( s.have_flags? )
    # s.all if (!s.have_flags?)
    s.update
rescue
    $stderr.puts( "\nError during script execution:\n\n" )
    $stderr.puts( $!.to_s )
    $stderr.print( "\nStack Trace:\n\n\t" )
    $stderr.print( $!.backtrace.join( "\n\t" ) )
    $stderr.puts( "" )
    exit(1)
end