"""PSLA - Perforce Server Log Analyzer """ import os from flask import Flask, request, redirect, url_for, render_template, flash from flask_wtf.csrf import CSRFProtect from werkzeug import secure_filename from config import Config from forms import UploadForm import logging import pandas as pd import sqlite3 import traceback from datetime import datetime, timedelta import json from altair import Chart, X, Y, Axis, Data, DataFormat from app import app from log2sql import Log2sql import queries # Chart dimensions WIDTH = 600 HEIGHT = 300 UPLOAD_FOLDER = '/logs' ALLOWED_EXTENSIONS = set(['txt']) app.config.from_object(Config) app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER csrf = CSRFProtect(app) def allowed_file(filename): return True return '.' in filename and \ filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS class MyOptions(): def __init__(self, dbname, logfile, sql=False, verbosity=logging.INFO, outlog=None): self.dbname = dbname self.logfile = logfile self.sql = sql self.verbosity = verbosity self.interval = 10 self.outlog = outlog self.output = None @app.route('/') def index(): """Front page for application""" return render_template('index.html') @app.route('/uploadLog', methods=['GET', 'POST']) def uploadLog(): "Upload log file" form = UploadForm() app.logger.debug("uploadLog: %s" % form.errors) app.logger.debug('------ {0}'.format(request.form)) if form.is_submitted(): app.logger.debug(form.errors) if form.validate(): app.logger.debug(form.errors) app.logger.debug(form.errors) if form.validate_on_submit(): db_folder = app.config['UPLOAD_FOLDER'] filename = secure_filename(form.uploadFile.data.filename) file_path = os.path.join(db_folder, filename) form.uploadFile.data.save(file_path) os.chdir(db_folder) optionsSQL = MyOptions("db", [file_path], sql=True, outlog='log.out') dbname = "%s.db" % optionsSQL.dbname if os.path.exists(dbname): os.remove(dbname) log2sql = Log2sql(optionsSQL) log2sql.processLog() msgs = [] return redirect(url_for('analyzeLog')) return render_template('uploadLog.html', title='Upload Log', form=form) @app.route('/analyzeLog') def analyzeLog(): "Access previously uploaded log" db_folder = app.config['UPLOAD_FOLDER'] dbname = os.path.join(db_folder, "db.db") if not os.path.exists(dbname): flash('Database does not exist', 'error') return render_template('error.html', title='Database error') try: conn = sqlite3.connect(dbname) except Exception as e: app.logger.error(traceback.format_exc()) flash('Error: %s' % (str(e)), 'error') return render_template('error.html', title='Error in database reporting') tables = [] for q in queries.queries: if not 'sql' in q or not q['sql']: continue app.logger.debug("running table query: %s - %s" % (q['title'], q['sql'])) start = datetime.now() try: df = pd.read_sql_query(q['sql'], conn) table = df.to_html() except Exception as e: table = "<h3>ERROR: %s</h3>" % str(e) end = datetime.now() delta = end - start tables.append({'data': table, 'title': q['title'], 'explanation': q['explanation'], 'sql': q['sql'], 'time_taken': str(delta)}) charts = [] i = 0 for q in queries.graphs: i += 1 if not 'sql' in q or not q['sql']: continue app.logger.debug("running chart query: %s - %s" % (q['title'], q['sql'])) start = datetime.now() try: df = pd.read_sql_query(q['sql'], conn) except Exception as e: app.logger.error(str(e)) continue end = datetime.now() delta = end - start chart = Chart(data=df, height=HEIGHT, width=WIDTH).mark_line().encode( X(q['x']['field'], axis=Axis(title=q['x']['title'])), Y(q['y']['field'], axis=Axis(title=q['y']['title'])) ) charts.append({'id': "chart-%d" % i, 'data': chart.to_json(), 'title': q['title'], 'explanation': q['explanation'], 'sql': q['sql'], 'time_taken': str(delta)}) return render_template('analyzeLog.html', tables=tables, charts=charts, LogFile='t1.log')
# | Change | User | Description | Committed | |
---|---|---|---|---|---|
#18 | 25220 | robert_cowham |
Moved project files to new location: //guest/perforce_software/log-analyzer/psla/... Required by Swarm project structure for workshop. |
||
#17 | 24989 | robert_cowham | Expand schema help for tableuse | ||
#16 | 24979 | robert_cowham | Tidy up charting to add db selection and titles. Clarify README | ||
#15 | 24321 | robert_cowham | Allow canned queries to be included | ||
#14 | 24291 | robert_cowham | Ensure imports will work for local files | ||
#13 | 24257 | robert_cowham | Output SQL as part of results | ||
#12 | 23934 | robert_cowham | Clean database and auto-create on log upload | ||
#11 | 23933 | robert_cowham | Create new seperate charting page | ||
#10 | 23924 | robert_cowham | Make bar charts and fix axis labels | ||
#9 | 23890 | robert_cowham | Basic progressbar | ||
#8 | 23845 | robert_cowham | Basic query form working | ||
#7 | 23833 | robert_cowham | Document how to run with flask and make /logs configurable | ||
#6 | 23785 | robert_cowham |
Tweak interface Remove unused module |
||
#5 | 23774 | robert_cowham | Ensure chart labels don't overlap | ||
#4 | 23773 | robert_cowham | Only display rest of page when analyzing | ||
#3 | 23772 | robert_cowham | Allow user to select from all db files | ||
#2 | 23771 | robert_cowham | Error handling for queries - refactor and move to seperate file | ||
#1 | 23765 | robert_cowham | Moved things down one level to psla dir to make it easier to see what belongs | ||
//guest/perforce_software/utils/log_analyzer/app/routes.py | |||||
#10 | 23762 | robert_cowham | New graph working | ||
#9 | 23761 | robert_cowham | Create a parameterised loop for graphs | ||
#8 | 23760 | robert_cowham | Simple basics with Altair chart | ||
#7 | 23723 | robert_cowham | First basic chart with plotly | ||
#6 | 23718 | robert_cowham | New statements. Time query execution | ||
#5 | 23714 | robert_cowham | New queries | ||
#4 | 23712 | robert_cowham | Proper formatting of table data | ||
#3 | 23711 | robert_cowham | More or less working with multiple SQL statements | ||
#2 | 23705 | robert_cowham | Basics working in simplified form | ||
#1 | 23704 | robert_cowham | Save before simplifying |