oldClientsTasksLabels.py #8

#!/usr/bin/env python3

################################################################################
#
# Copyright (c) 2024, Perforce Software, Inc.  All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1.  Redistributions of source code must retain the above copyright
#     notice, this list of conditions and the following disclaimer.
#
# 2.  Redistributions in binary form must reproduce the above copyright
#     notice, this list of conditions and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL PERFORCE SOFTWARE, INC. BE LIABLE FOR ANY
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# = Date
#   
#   $Date: 2024/08/21 $
#
# = Description
#
#   Collect information about workspaces, task streams, labels and output them
#   in csv format:
#   Client,name,owner,last_update_date,last_access_date,nb_synced_files,build?
#   Task,name,owner,last_update_date,last_access_date,nb_of_tmp_files
#   Label,name,owner,last_update_date,last_access_date,nb_of_tagged_files
#
#   The script will create a journal patch (journal.patch.gz) to remove db.have
#   records for non existent workspace spec and db.label records for non
#   existent label spec.
#
# = Usage
# 
#   oldClientsTasksLabels.py a_checkpoint|a_checkpoint.gz|a_checkpoint_dir
#
################################################################################

from __future__ import print_function
import re
import time
import sys
import gzip
import mimetypes
import os
import math

clients = {}
taskStreams = {}
labels = {} 
jnlPatch = None 
clientSpecRE = re.compile(b'@pv@ [0-9]* @db.domain@ @(.*?)@ 99 @.*?@ @.*?@ @.*?@ @.*?@ @(.*?)@ ([0-9]*) ([0-9]*) [0-9]* @(.*)')
haveRE = re.compile(b'@pv@ [0-9]* @db.have@ @//(.*?)/.*')
haverpRE = re.compile(b'@pv@ [0-9]* @db.have.rp@ @//(.*?)/.*')
streamRE = re.compile(b'@pv@ [0-9]* @db.stream@ @(//.*?/.*?)@ @.*@ @.*@ ([0-9]) ')
streamSpecRE = re.compile(b'@pv@ [0-9]* @db.domain@ @(//.*?/.*?)@ 115 @@ @@ @@ @@ @(.*?)@ ([0-9]*) ([0-9]*) ')
taskRE = re.compile(b'@pv@ [0-9]* @db.revtx@ @(//.*?/.*?)/.*?@')
labelSpecRE = re.compile(b'@pv@ [0-9]* @db.domain@ @(.*?)@ 108 @@ @@ @@ @@ @(.*?)@ ([0-9]*) ([0-9]*) ')
labelRE = re.compile(b'@pv@ [0-9]* @db.label@ @(.*?)@.*')
stopRE = re.compile(b'.*@db.revcx@.*')
ckpRE = re.compile('db.domain.ckp$|db.domain.ckp.gz$|db.have.ckp$|db.have.ckp.gz$|db.have.rp.ckp$|db.have.rp.ckp.gz$|db.stream.ckp$|db.stream.ckp.gz$|db.revtx.ckp$|db.revtx.ckp.gz$|db.label.ckp$|db.label.ckp.gz$')

def cmdUsage():
    sys.exit("Usage: oldClientsTasksLabels.py a_checkpoint|a_checkpoint.gz|a_checkpoint_dir")

def addClient(client):
    clients[client] = {}
    clients[client]["have"] = 0
    clients[client]["db.have.rp"] = "False"
    clients[client]["owner"] = b"unknown"
    clients[client]["update"] = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(0.0))
    clients[client]["access"] = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(0.0))

def updateClient(client, owner, update, access):
    if client not in clients:
       addClient(client)
    clients[client]["owner"] = owner
    clients[client]["update"] = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(float(update)))
    clients[client]["access"] = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(float(access)))

def increaseClientHave(client, record):
    if client not in clients:
       addClient(client)
    if clients[client]["owner"] == b"unknown":
       global jnlPatch
       if jnlPatch is None:
          print("client")
          jnlPatch = gzip.open("jnl.patch.gz", "wb")
       jnlPatch.write(record.replace(b"@pv@", b"@dv@"))
    clients[client]["have"] += 1

def increaseClientHaverp(client, record):
    if client not in clients:
       addClient(client)
    clients[client]["db.have.rp"] = "True"
    if clients[client]["owner"] == b"unknown":
       global jnlPatch
       if jnlPatch is None:
          jnlPatch = gzip.open("jnl.patch.gz", "wb")
       jnlPatch.write(record.replace(b"@pv@", b"@dv@"))
    clients[client]["have"] += 1

def addTaskStream(stream):
    taskStreams[stream] = {}
    taskStreams[stream]["rev"] = 0
    taskStreams[stream]["owner"] = b"unknown"
    taskStreams[stream]["update"] = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(0.0))
    taskStreams[stream]["access"] = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(0.0))

def updateTaskStreamType(stream, type):
    if stream not in taskStreams:
       addTaskStream(stream)
    taskStreams[stream]["type"] = type

def updateTaskStream(stream, owner, update, access):
    if stream not in taskStreams:
       addTaskStream(stream)
    taskStreams[stream]["owner"] = owner
    taskStreams[stream]["update"] = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(float(update)))
    taskStreams[stream]["access"] = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(float(access)))

def increaseTaskStreamRev(stream):
    if stream not in taskStreams:
       addTaskStream(stream)
    taskStreams[stream]["rev"] += 1

def addLabel(label):
    labels[label] = {}
    labels[label]["tag"] = 0
    labels[label]["owner"] = b"unknown"
    labels[label]["update"] = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(0.0))
    labels[label]["access"] = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(0.0))

def updateLabel(label, owner, update, access):
    if label not in labels:
       addLabel(label)
    labels[label]["owner"] = owner
    labels[label]["update"] = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(float(update)))   
    labels[label]["access"] = time.strftime("%Y/%m/%d %H:%M:%S", time.localtime(float(access)))   

def increaseLabelTag(label, record):
    if label not in labels:
       addLabel(label)
    if labels[label]["owner"] == b"unknown":
       global jnlPatch
       if jnlPatch is None:
          print("label")
          jnlPatch = gzip.open("jnl.patch.gz", "wb")
       jnlPatch.write(record.replace(b"@pv@", b"@dv@"))
    labels[label]["tag"] += 1

def parse(filename):
    if mimetypes.guess_type(filename)[1] == 'gzip':
       ckp = gzip.open(filename, "rb")
    else:
       ckp = open(filename, "rb")
    for line in ckp:
        match = clientSpecRE.search(line)
        if match:
           updateClient(match.group(1), match.group(2), match.group(3), match.group(4))
        else:
          match = haveRE.search(line)
          if match:
             increaseClientHave(match.group(1), line)
          match = haverpRE.search(line)
          if match:
             increaseClientHaverp(match.group(1), line)
          else:
             match = streamRE.search(line)
             if match:
                updateTaskStreamType(match.group(1), match.group(2))
             else:
                match = streamSpecRE.search(line)
                if match:
                   updateTaskStream(match.group(1), match.group(2), match.group(3), match.group(4))
                else:
                   match = taskRE.search(line)
                   if match: 
                      increaseTaskStreamRev(match.group(1))
                   else: 
                      match = labelRE.search(line)
                      if match:
                         increaseLabelTag(match.group(1), line)
                      else:
                         match = labelSpecRE.search(line)
                         if match:
                            updateLabel(match.group(1), match.group(2), match.group(3), match.group(4))
                         else:
                            if stopRE.search(line):
                               # stop, no need to look further
                               break
    ckp.close()

def report():
    # In order to output bytes to stdout, sys.stdout.buffer must be used in Python 3 
    # but this is not compatible with Python 2, so to make it compatible:
    output = getattr(sys.stdout, 'buffer', sys.stdout)
    for (key, value) in clients.items():
        output.write(b"Client," + key + b"," + value["owner"])
        sys.stdout.write("," + value["update"] + "," + value["access"] + "," + str(value["have"]) + "," + value["db.have.rp"] + "\n")
        sys.stdout.flush()
    for (key, value) in taskStreams.items():
        if "type" in value and value["type"] == b"4":
           output.write(b"Task," + key + b"," + value["owner"])
           sys.stdout.write("," + value["update"] + "," + value["access"] + "," + str(value["rev"]) + "\n")
           sys.stdout.flush()
    for (key, value) in labels.items():
        output.write(b"Label," + key + b"," + value["owner"])
        sys.stdout.write("," + value["update"] + "," + value["access"] + "," + str(value["tag"]) + "\n")
        sys.stdout.flush()

def main():
    if len(sys.argv) < 2:
       cmdUsage()
    filename = sys.argv[1]
    print("Processing, it may take several hours...\n", end='', file=sys.stderr)
    sys.stderr.flush()
    if os.path.isdir(filename):
       for file in os.listdir(filename):
           if ckpRE.match(file):
              filepath = os.path.join(filename, file)
              print("Processing", filepath)
              parse(filepath) 
    else:
        parse(filename)
    if jnlPatch:
       jnlPatch.close()
    report()

if __name__ == '__main__':
   main()