# -*- encoding: UTF8 -*- # Test harness for JobsCmdFilter.py from __future__ import print_function import sys import unittest import os sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from log2sql import Log2sql python3 = sys.version_info[0] >= 3 if python3: from io import StringIO else: from StringIO import StringIO fileHeader = """CREATE DATABASE IF NOT EXISTS testdb; USE testdb; DROP TABLE IF EXISTS process; CREATE TABLE process (processkey CHAR(32) NOT NULL, lineNumber INT NOT NULL, pid INT NOT NULL, startTime DATETIME NOT NULL,endTime DATETIME NULL, computedLapse FLOAT NULL,completedLapse FLOAT NULL, user TEXT NOT NULL, workspace TEXT NOT NULL, ip TEXT NOT NULL, app TEXT NOT NULL, cmd TEXT NOT NULL, args TEXT NULL, uCpu INT NULL, sCpu INT NULL, diskIn INT NULL, diskOut INT NULL, ipcIn INT NULL, ipcOut INT NULL, maxRss INT NULL, pageFaults INT NULL, rpcMsgsIn INT NULL, rpcMsgsOut INT NULL, rpcSizeIn INT NULL, rpcSizeOut INT NULL, rpcHimarkFwd INT NULL, rpcHimarkRev INT NULL, running INT NULL, error TEXT NULL, PRIMARY KEY (processkey, lineNumber)); DROP TABLE IF EXISTS tableUse; CREATE TABLE tableUse (processkey CHAR(32) NOT NULL, lineNumber INT NOT NULL, tableName VARCHAR(255) NOT NULL, pagesIn INT NULL, pagesOut INT NULL, pagesCached INT NULL, readLocks INT NULL, writeLocks INT NULL, getRows INT NULL, posRows INT NULL, scanRows INT NULL, putRows int NULL, delRows INT NULL, totalReadWait INT NULL, totalReadHeld INT NULL, totalWriteWait INT NULL, totalWriteHeld INT NULL, maxReadWait INT NULL, maxReadHeld INT NULL, maxWriteWait INT NULL, maxWriteHeld INT NULL, PRIMARY KEY (processkey, lineNumber, tableName)); SET autocommit=0; """ class TestLogParser(unittest.TestCase): def __init__(self, methodName='runTest'): super(TestLogParser, self).__init__(methodName=methodName) def assertRegex(self, *args, **kwargs): if python3: return super(TestLogParser, self).assertRegex(*args, **kwargs) else: return super(TestLogParser, self).assertRegexpMatches(*args, **kwargs) def setUp(self): pass def tearDown(self): pass def testBasicParsing(self): "simple parsing" input = """ Perforce server info: 2015/09/02 15:23:09 pid 1616 robert@robert-test 127.0.0.1 [Microsoft Visual Studio 2013/12.0.21005.1] 'user-sync //...' Perforce server info: 2015/09/02 15:23:09 pid 1616 compute end .031s Perforce server info: 2015/09/02 15:23:09 pid 1616 completed .031s """ md5 = "51ae6850c357585e594720f7efe9bfc0" if python3: md5 = "51ae6850c357585e594720f7efe9bfc0" expected = """INSERT IGNORE INTO process VALUES ("%s",3,1616,"2015-09-02 15:23:09",NULL,NULL,NULL,"robert","robert-test","127.0.0.1","Microsoft Visual Studio 2013/12.0.21005.1","user-sync","//...",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,NULL); COMMIT; """ % md5 saved_stdoutput = StringIO() log2sql = Log2sql("", "testdb", instream=StringIO(input), outstream=saved_stdoutput) log2sql.processLog() actual = saved_stdoutput.getvalue() self.assertEqual(fileHeader, actual[:len(fileHeader)]) self.assertEqual(expected, actual[len(fileHeader):]) input = """ Perforce server info: 2015/09/02 15:23:11 pid 1616 robert@robert-test 127.0.0.1 [Microsoft Visual Studio 2013/12.0.21005.1] 'user-changes -c robert-test -m 1 -s pending -u robert' Perforce server info: 2015/09/02 15:23:11 pid 1616 compute end .047s Perforce server info: 2015/09/02 15:23:11 pid 1616 completed .047s """ md5 = "df655d67bf61ab0e6e53d67bb0fe3674" if python3: md5 = "df655d67bf61ab0e6e53d67bb0fe3674" expected = """INSERT IGNORE INTO process VALUES ("%s",3,1616,"2015-09-02 15:23:11",NULL,NULL,NULL,"robert","robert-test","127.0.0.1","Microsoft Visual Studio 2013/12.0.21005.1","user-changes","-c robert-test -m 1 -s pending -u robert",NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,0,NULL); COMMIT; """ % md5 saved_stdoutput = StringIO() log2sql = Log2sql("", "testdb", instream=StringIO(input), outstream=saved_stdoutput) log2sql.processLog() actual = saved_stdoutput.getvalue() self.assertEqual(fileHeader, actual[:len(fileHeader)]) self.assertEqual(expected, actual[len(fileHeader):]) if __name__ == '__main__': unittest.main()
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#32 | 23632 | Robert Cowham | Placeholder to handle trigger output - not known about previously! | ||
#31 | 23591 | Robert Cowham |
Handle meta/db and similar records which may occur multiple times for a command with vtrack output. |
||
#30 | 23493 | Robert Cowham |
Process peek track records Process meta/client/change lock track records Check for unrecognised track records (debug output if found) |
||
#29 | 23446 | Robert Cowham | Duplicate pids for same command within same second now handled correctly! | ||
#28 | 23445 | Robert Cowham | Fixed testChanges - if a cmd has track info then it is completed. | ||
#27 | 23444 | Robert Cowham | Refactored tests | ||
#26 | 23443 | Robert Cowham |
Avoid processing cmds until time changes as this means out of order track entries are processed appropriately. (Normally a 'completed' is immediately followed by a track entry for the original cmd - but sometimes the track entry comes a bit later after some other unrelated cmds are logged. Previously this would lead to being treated as extra cmds and ones which would never be completed. This in turn lead to lots of performance loss in processCompletedCommands() function) |
||
#25 | 23438 | Robert Cowham | Add some logging to track down performance issues | ||
#24 | 23401 | Robert Cowham | Fix tests failing in python3 due to list re-ordering | ||
#23 | 23399 | Robert Cowham | Handle utf encoding errors | ||
#22 | 23375 | Robert Cowham | Show diffs better when tests fail | ||
#21 | 23369 | Robert Cowham | Remove quotes around datetime in sqlite | ||
#20 | 21829 | Robert Cowham | Fix problem with changes -o commands. | ||
#19 | 21796 | Robert Cowham | Handle rmt-FileFetch records which may be duplicates as they don't have 'completed' entries. | ||
#18 | 21712 | Robert Cowham | Check for correct line number when looking to update for track entries | ||
#17 | 21708 | Robert Cowham |
Tweak parameters to improve error messages. Make output to stdout optional |
||
#16 | 21705 | Robert Cowham |
Minor refactoring for clarity. Process all completed records as we go. |
||
#15 | 21704 | Robert Cowham | A couple more tests for log formats. | ||
#14 | 21703 | Robert Cowham |
Fix failing python3 test Add new test to ignore error blocks |
||
#13 | 21702 | Robert Cowham |
Rework algorithm to process blocks at a time, and to avoid the SQL update statements. Risk of increased memory vs improved speed. |
||
#12 | 21697 | Robert Cowham | Parse rcpSnd/rpcRcv and add to database. | ||
#11 | 21696 | Robert Cowham | Test for client/locks entry | ||
#10 | 21694 | Robert Cowham | Handle null values properly | ||
#9 | 21660 | Robert Cowham | Handle utf8 in python2 strings | ||
#8 | 21656 | Robert Cowham | Avoid SQL quoting issues with alternative insert statement. | ||
#7 | 21643 | Robert Cowham | Handle git-fusion entries (strip the json) - for --sql option | ||
#6 | 21402 | Robert Cowham |
Filter out swarm commands Test for db content in a basic fashion. |
||
#5 | 21396 | Robert Cowham | Test for update rows. | ||
#4 | 21393 | Robert Cowham |
Add a swarm test - large record. Improve performance by skipping easy matches early. |
||
#3 | 21374 | Robert Cowham | Write to sqllite db | ||
#2 | 21368 | Robert Cowham | Make tests work under python3 | ||
#1 | 21367 | Robert Cowham | Restructured for unit tests, and first unit tests. |