#!/usr/bin/env python3 # -*- encoding: UTF8 -*- # Test harness for JobsCmdFilter.py from __future__ import print_function import sys import unittest import os import io import logging import re import sqlite3 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from pxlog2sql import PxLog2sql python3 = sys.version_info[0] >= 3 if python3: from io import StringIO else: from StringIO import StringIO fileHeader = """CREATE DATABASE IF NOT EXISTS testdb; USE testdb; DROP TABLE IF EXISTS syncs; CREATE TABLE syncs (lineNumber INT NOT NULL, pid INT NOT NULL, endTime DATETIME NULL, completedLapse FLOAT NULL, uCpu INT NULL, sCpu INT NULL, diskIn INT NULL, diskOut INT NULL, ipcIn INT NULL, ipcOut INT NULL, maxRss INT NULL, pageFaults INT NULL, rpcMsgsIn INT NULL, rpcMsgsOut INT NULL, rpcSizeIn INT NULL, rpcSizeOut INT NULL, rpcHimarkFwd INT NULL, rpcHimarkRev INT NULL, rpcSnd FLOAT NULL, rpcRcv FLOAT NULL, filesServer int NULL, sizeServerKB float NULL, filesCache int NULL, sizeCacheKB float NULL, PRIMARY KEY (lineNumber)); BEGIN TRANSACTION; """ def compressGzip(tmpFile, input): import gzip with gzip.open(tmpFile, 'wb') as f: if python3: f.write(bytes(input.encode())) else: f.write(input) def compressZip(tmpFile, input): from zipfile import ZipFile tmpContentsFile = "~tmpContents.log" with open(tmpContentsFile, "w") as f: f.write(input) with ZipFile(tmpFile, 'w') as myzip: myzip.write(tmpContentsFile) class MyOptions(): def __init__(self, dbname, logname, sql=False, csv=True, verbosity=logging.DEBUG): self.dbname = dbname self.logname = logname self.logfile = [logname] self.sql = sql self.no_sql = not sql self.verbosity = verbosity self.interval = 10 self.outlog = None self.reset = False self.fileInterval = None self.csv = csv class TestPxLogParser(unittest.TestCase): def __init__(self, methodName='runTest'): super(TestPxLogParser, self).__init__(methodName=methodName) self.maxDiff = None def assertRegex(self, *args, **kwargs): if python3: return super(TestPxLogParser, self).assertRegex(*args, **kwargs) else: return super(TestPxLogParser, self).assertRegexpMatches(*args, **kwargs) def setUp(self): pass def tearDown(self): pass def runTestText(self, expected, expectedCsv, input, longlines=False, compressed=False): if not compressed: optionsText = MyOptions("testdb", "") else: optionsText = MyOptions("testdb", input) self.saved_stdoutput = StringIO() self.saved_stderr = StringIO() self.saved_csv = StringIO() if not compressed: parser = PxLog2sql(optionsText, inStream=StringIO(input), outStream=self.saved_stdoutput, errStream=self.saved_stderr, csvStream=self.saved_csv) else: optionsText.logname = input parser = PxLog2sql(optionsText, outstream=self.saved_stdoutput, errstream=self.saved_stderr) parser.processLogs() actual = self.saved_stdoutput.getvalue() actualCsv = self.saved_csv.getvalue() self.assertMultiLineEqual(fileHeader, actual[:len(fileHeader)]) if not longlines: exp = "\n".join(sorted(expected.split("\n"))) act = "\n".join(sorted(actual[len(fileHeader):].split("\n"))) self.assertMultiLineEqual(exp, act) expCsv = "\n".join(sorted(expectedCsv.split("\n"))) actCsv = "\n".join(sorted(actualCsv.split("\n")[1:-1])) self.assertMultiLineEqual(expCsv, actCsv) else: rest = actual[len(fileHeader):] for expline in expected.split("\n"): reLine = re.compile(expline.replace("(", "\("), re.MULTILINE) self.assertRegex(rest, reLine, re.MULTILINE) reUnrecognised = re.compile("Unrecognised track:", re.MULTILINE) if re.search(reUnrecognised, self.saved_stderr.getvalue()): self.assertTrue(False, "Didn't process all track records: %s" % self.saved_stderr.getvalue()) def runTestSQL(self, input, compressed=False): if not compressed: optionsSQL = MyOptions("testdb", "", sql=True, csv=None) else: optionsSQL = MyOptions("testdb", input, sql=True, csv=None) dbname = "%s.db" % optionsSQL.dbname if os.path.exists(dbname): os.remove(dbname) saved_stdoutput = StringIO() saved_stderr = StringIO() if not compressed: parser = PxLog2sql(optionsSQL, inStream=StringIO(input), outStream=saved_stdoutput, errStream=saved_stderr) else: optionsSQL.logname = input parser = PxLog2sql(optionsSQL, outStream=saved_stdoutput, errStream=saved_stderr) parser.processLogs() return sqlite3.connect(dbname) def testProxyLog(self): "Process standard proxy logs" input = r""" Perforce proxy info: 2018/04/06 13:46:19 pid 51449 completed .246s --- lapse .246s --- usage 122+92us 0+0io 6+10143net 1101824k 0pf --- rpc msgs/size in+out 5+10010/0mb+39mb himarks 2000/2000 snd/rcv .005s/.009s --- proxy faults 0 MB 0 other 0 flushes 2 cached 2 --- proxytotals files/size svr+cache 0+2/0B+40M Perforce proxy info: 2018/04/13 09:08:13 pid 99941 completed 1.61s --- lapse 1.61s --- usage 255+279us 0+8io 2458+10147net 1622016k 64pf --- rpc msgs/size in+out 6+10011/0mb+39mb himarks 2000/2000 snd/rcv .000s/.010s --- proxy faults 0 MB 0 other 0 flushes 2 cached 0 --- proxytotals files/size svr+cache 2+0/1.2G+1.1TB Perforce proxy info: 2018/04/13 09:08:14 pid 99948 completed .268s --- lapse .268s --- usage 100+126us 0+0io 6+10143net 1019904k 1pf --- rpc msgs/size in+out 5+10010/0mb+39mb himarks 2000/2000 snd/rcv .012s/.008s --- proxy faults 0 MB 0 other 0 flushes 2 cached 2 --- proxytotals files/size svr+cache 0+2/3K+1.3P """ expected = """INSERT IGNORE INTO syncs VALUES (2,51449,"2018-04-06 13:46:19",0.246,5,10010,0,39,2000,2000,.005,.009,122,92,0,0,6,10143,1101824,0,0,0.0,2,40960.0); INSERT IGNORE INTO syncs VALUES (9,99941,"2018-04-13 09:08:13",1,6,10011,0,39,2000,2000,.000,.010,255,279,0,8,2458,10147,1622016,64,2,1258291.2,0,1181116006.4); INSERT IGNORE INTO syncs VALUES (16,99948,"2018-04-13 09:08:14",0.268,5,10010,0,39,2000,2000,.012,.008,100,126,0,0,6,10143,1019904,1,0,3.0,2,1429365116108.8); COMMIT; BEGIN TRANSACTION; """ expectedCsv = """2,51449,"2018-04-06 13:46:19",0.246,0,0.0,2,40960.0 9,99941,"2018-04-13 09:08:13",1,2,1258291.2,0,1181116006.4 16,99948,"2018-04-13 09:08:14",0.268,0,3.0,2,1429365116108.8""" self.runTestText(expected, expectedCsv, input) conn = self.runTestSQL(input) c = conn.cursor() c.execute('SELECT * from Syncs;') rows = c.fetchall() self.assertEqual(3, len(rows)) row = rows[0] self.assertEqual(row[0], 2) self.assertEqual(row[1], 51449) self.assertEqual(row[2], u'2018-04-06 13:46:19') conn.close() def testEmptyProxyLog(self): "Process standard proxy logs" input = r""" """ expected = """COMMIT; BEGIN TRANSACTION; """ expectedCsv = "" self.runTestText(expected, expectedCsv, input) conn = self.runTestSQL(input) c = conn.cursor() c.execute('SELECT * from Syncs;') rows = c.fetchall() self.assertEqual(0, len(rows)) conn.close() def testOldProxyLog(self): "Process old style proxy logs - no proxytotals line" input = r""" Perforce proxy info: 2018/04/06 13:46:19 pid 51449 completed .246s --- lapse .246s --- usage 122+92us 0+0io 6+10143net 1101824k 0pf --- rpc msgs/size in+out 5+10010/0mb+39mb himarks 2000/2000 snd/rcv .005s/.009s --- proxy faults 0 MB 0 other 0 flushes 2 cached 2 Perforce proxy info: 2018/04/13 09:08:13 pid 99941 completed 1.61s --- lapse 1.61s --- usage 255+279us 0+8io 2458+10147net 1622016k 64pf --- rpc msgs/size in+out 6+10011/0mb+39mb himarks 2000/2000 snd/rcv .000s/.010s --- proxy faults 0 MB 0 other 0 flushes 2 cached 0 """ expected = """INSERT IGNORE INTO syncs VALUES (2,51449,"2018-04-06 13:46:19",0.246,5,10010,0,39,2000,2000,.005,.009,122,92,0,0,6,10143,1101824,0,NULL,NULL,NULL,NULL); INSERT IGNORE INTO syncs VALUES (8,99941,"2018-04-13 09:08:13",1,6,10011,0,39,2000,2000,.000,.010,255,279,0,8,2458,10147,1622016,64,NULL,NULL,NULL,NULL); COMMIT; BEGIN TRANSACTION; """ expectedCsv = """2,51449,"2018-04-06 13:46:19",0.246,0,0.0,0,0.0 8,99941,"2018-04-13 09:08:13",1,0,0.0,0,0.0""" self.runTestText(expected, expectedCsv, input) conn = self.runTestSQL(input) c = conn.cursor() c.execute('SELECT * from Syncs;') rows = c.fetchall() self.assertEqual(2, len(rows)) row = rows[0] self.assertEqual(row[0], 2) self.assertEqual(row[1], 51449) self.assertEqual(row[2], u'2018-04-06 13:46:19') conn.close() if __name__ == '__main__': unittest.main()