#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# ==============================================================================
# Copyright and license info is available in the LICENSE file included with
# the Server Deployment Package (SDP), and also available online:
# https://swarm.workshop.perforce.com/projects/perforce-software-sdp/view/main/LICENSE
# ------------------------------------------------------------------------------
"""
NAME:
depot_verify_chunks.py
DESCRIPTION:
This script prints out a list of directories to verify that don't exceed a specific
size limit, e.g. 100M or 1G.
It is intended as an optional input to p4verify.py script or to be run using GNU parallel.
It walks down the tree of specified paths using the p4 dirs/p4 sizes -az commands to identify
appropriate sub-trees.
The output is a list of depot paths:
//depot/...
//big_depot/subdir1/...
//big_depot/subdir2/...
//big_depot/subdir3/...
//big_depot/*
You can run:
depot_verify_chunks.py -m 1G //some/path/... //another_depot/...
With parallel (see https://www.gnu.org/software/parallel/):
depot_verify_chunks.py -m 1G //some/path/... | parallel p4 verify -qzt {} >> verify.out
nohup depot_verify_chunks.py -m 1G //some/path/... | parallel p4 verify -qzt {} >> verify.out 2>&1 &
"""
# Python 2.7/3.3 compatibility.
from __future__ import print_function
import sys
import os
import textwrap
import argparse
import logging
import P4
import re
from collections import OrderedDict
script_name = os.path.basename(os.path.splitext(__file__)[0])
LOGDIR = os.getenv('LOGS', '/p4/1/logs')
DEFAULT_LOG_FILE = "log-%s.log" % script_name
if os.path.exists(LOGDIR):
DEFAULT_LOG_FILE = os.path.join(LOGDIR, "%s.log" % script_name)
DEFAULT_VERBOSITY = 'DEBUG'
LOGGER_NAME = 'verify_dir_list'
def parse_human_fmt(opt):
"Parses 1, 1K, 2M, 3.1G etc and returns int"
units = "kmgt"
opt = opt.lower()
m = re.match("^([0-9]*\.{0,1}[0-9]*)([kmgt]*)$", opt)
if not m:
raise TypeError("Argument must be a human readable size number, e.g. 1, 1K, 2M, 3.1G")
num = float(m.group(1))
unit = m.group(2)
if unit and unit in units:
multiplier = 1.0
for u in ['k','m','g','t']:
multiplier *= 1024.0
if u == unit:
break
num *= multiplier
return int(num)
class depot_verify_chunks(object):
"""See module doc string for details"""
def __init__(self, *args, **kwargs):
self.parse_args(__doc__, args)
def parse_args(self, doc, args):
"""Common parsing and setting up of args"""
desc = textwrap.dedent(doc)
parser = argparse.ArgumentParser(
formatter_class=argparse.RawDescriptionHelpFormatter,
description=desc,
epilog="Copyright (c) 2008-2018 Perforce Software, Inc."
)
self.add_parse_args(parser)
self.options = parser.parse_args(args=args)
self.init_logger()
self.logger.debug("Command Line Options: %s\n" % self.options)
self.options.max_size = parse_human_fmt(self.options.max_size) # May raise error
def add_parse_args(self, parser, default_log_file=None, default_verbosity=None):
"""Default trigger arguments - common to all triggers
:param default_verbosity:
:param default_log_file:
:param parser:
"""
if not default_log_file:
default_log_file = DEFAULT_LOG_FILE
if not default_verbosity:
default_verbosity = DEFAULT_VERBOSITY
parser.add_argument('-p', '--port', default=None,
help="Perforce server port. Default: $P4PORT")
parser.add_argument('-u', '--user', default=None, help="Perforce user. Default: $P4USER")
parser.add_argument('-L', '--log', default=default_log_file, help="Default: " + default_log_file)
parser.add_argument('-m', '--max-size', help="Max size of each depot chunk. Can be specified in K/M/G/T, e.g. 1G")
parser.add_argument('path', nargs='*', help="Perforce depot path(s)")
parser.add_argument('-v', '--verbosity',
nargs='?',
const="INFO",
default=default_verbosity,
choices=('DEBUG', 'WARNING', 'INFO', 'ERROR', 'FATAL'),
help="Output verbosity level. Default is: " + default_verbosity)
def init_logger(self, logger_name=None):
if not logger_name:
logger_name = LOGGER_NAME
self.logger = logging.getLogger(logger_name)
self.logger.setLevel(self.options.verbosity)
logformat = '%(levelname)s %(asctime)s %(filename)s %(lineno)d: %(message)s'
logging.basicConfig(format=logformat, filename=self.options.log, level=self.options.verbosity)
formatter = logging.Formatter('%(message)s')
ch = logging.StreamHandler(sys.stderr)
ch.setLevel(logging.INFO)
ch.setFormatter(formatter)
self.logger.addHandler(ch)
def get_chunks(self, results, startdir, max_size):
if startdir.endswith("/..."):
startdir = startdir[:-4]
sizes = self.p4.run_sizes("-az", "%s/..." % startdir)
if int(sizes[0]['fileSize']) <= max_size:
results["%s/..." % startdir] = 1
return
for d in self.p4.run_dirs('-D', "%s/*" % startdir):
self.get_chunks(results, d['dir'], max_size)
results["%s/*" % startdir] = 1
results["%s/*" % startdir] = 1
def run(self):
"""Runs script"""
self.p4 = P4.P4()
self.p4.logger = self.logger
if self.options.port:
self.p4.port = self.options.port
if self.options.user:
self.p4.user = self.options.user
self.p4.connect()
results = OrderedDict()
for path in self.options.path:
self.get_chunks(results, path, int(self.options.max_size))
print("\n".join([k for k in results.keys()]))
return [k for k in results.keys()]
if __name__ == '__main__':
""" Main Program"""
obj = depot_verify_chunks(*sys.argv[1:])
obj.run()