"""PSLA - Perforce Server Log Analyzer """ import os from flask import Flask, request, redirect, url_for, render_template, flash from flask_wtf.csrf import CSRFProtect from werkzeug import secure_filename from config import Config from forms import UploadForm import logging import pandas as pd import sqlite3 import traceback from app import app from log2sql import Log2sql UPLOAD_FOLDER = '/logs' ALLOWED_EXTENSIONS = set(['txt']) app.config.from_object(Config) app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER csrf = CSRFProtect(app) def allowed_file(filename): return True return '.' in filename and \ filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS class MyOptions(): def __init__(self, dbname, logfile, sql=False, verbosity=logging.INFO, outlog=None): self.dbname = dbname self.logfile = logfile self.sql = sql self.verbosity = verbosity self.interval = 10 self.outlog = outlog self.output = None @app.route('/') def index(): """Front page for application""" return render_template('index.html') @app.route('/uploadLog', methods=['GET', 'POST']) def uploadLog(): "Upload log file" form = UploadForm() app.logger.debug("uploadLog: %s" % form.errors) app.logger.debug('------ {0}'.format(request.form)) if form.is_submitted(): app.logger.debug(form.errors) if form.validate(): app.logger.debug(form.errors) app.logger.debug(form.errors) if form.validate_on_submit(): db_folder = app.config['UPLOAD_FOLDER'] filename = secure_filename(form.uploadFile.data.filename) file_path = os.path.join(db_folder, filename) form.uploadFile.data.save(file_path) os.chdir(db_folder) optionsSQL = MyOptions("testdb", [file_path], sql=True, outlog='log.out') dbname = "%s.db" % optionsSQL.dbname if os.path.exists(dbname): os.remove(dbname) log2sql = Log2sql(optionsSQL) log2sql.processLog() msgs = [] return redirect(url_for('analyzeLog')) return render_template('uploadLog.html', title='Upload Log', form=form) queries = [{'title': 'Count of Cmds', 'explanation': 'How many commands of each type', 'sql': """ select cmd, count(cmd) from process group by cmd; """}, {'title': 'Cmds per second', 'explanation': 'How many commands each second', 'sql': """ SELECT SUBSTR(startTime, 0, 19) as timesec, MAX(running) FROM process GROUP BY timesec; """}] @app.route('/analyzeLog') def analyzeLog(): "Access previously uploaded log" db_folder = app.config['UPLOAD_FOLDER'] dbname = os.path.join(db_folder, "testdb.db") if not os.path.exists(dbname): flash('Database does not exist', 'error') return render_template('error.html', title='Database error') try: conn = sqlite3.connect(dbname) data_results = [] for q in queries: app.logger.debug("running: %s - %s" % (q['title'], q['sql'])) if q['sql']: data = pd.read_sql_query(q['sql'], conn) data_results.append((data.to_html(), q['title'], q['explanation'], q['sql'])) app.logger.debug("data: %s" % (",".join([d[0] for d in data_results]))) app.logger.debug("titles: %s" % (",".join([d[1] for d in data_results]))) app.logger.debug("expl: %s" % (",".join([d[2] for d in data_results]))) return render_template('analyzeLog.html', tables=[d[0] for d in data_results], titles=[d[1] for d in data_results], explanations=[d[2] for d in data_results], sql=[d[3] for d in data_results], LogFile='t1.log') except Exception as e: flash('Error executing SQL Query: %s, %s' % (str(e), traceback.format_exc()), 'error') return render_template('error.html', title='Database SQL error')