From 9421395c790187ff219bb57cda5ba3b1d32b0665 Mon Sep 17 00:00:00 2001 From: Tom Powell Date: Fri, 2 Nov 2018 00:05:39 -0700 Subject: [PATCH] Stats and API --- test.py | 20 +++++-- ui/app.py | 119 ++++++++++++++++++++++++++++++++++++---- ui/config.py | 1 + ui/models.py | 33 +++++++++++ ui/templates/stats.html | 50 +++++++++++++++++ 5 files changed, 209 insertions(+), 14 deletions(-) create mode 100644 ui/templates/stats.html diff --git a/test.py b/test.py index 40121fa..a73f5d3 100644 --- a/test.py +++ b/test.py @@ -102,10 +102,10 @@ class TestWebhooks(UsesModels): class TestWeb(UsesModels): - def test_home(self): - build1 = Build(build_id=1) - runner = Runner(runner_name="foobar", runner_id="foobar") - build2 = Build(build_id=2, build_status="success", build_device="mako", build_version="cm-14.1", build_type="userdebug", build_date=datetime.datetime.now(), build_runner=runner) + def test_get(self): + build1 = Build(build_id=1, build_date=datetime.datetime.strptime("2018-01-01", "%Y-%m-%d"), build_status="pending", build_version="cm-14.1") + runner = Runner(runner_name="foobar", runner_id="foobar", runner_sponsor="Me", runner_sponsor_url="You") + build2 = Build(build_id=2, build_status="success", build_device="mako", build_version="cm-14.1", build_type="userdebug", build_date=datetime.datetime.strptime("2018-01-01", "%Y-%m-%d"), build_runner=runner) db.session.add(build1) db.session.add(runner) db.session.add(build2) @@ -122,5 +122,17 @@ class TestWeb(UsesModels): response = self.client.get("/?status=success&device=mako") assert response.status_code == 200 + response = self.client.get("/api/v1/runners") + assert response.status_code == 200 + + response = self.client.get("/api/v1/builds") + assert response.status_code == 200 + + response = self.client.get("/api/v1/stats") + assert response.status_code == 200 + + response = self.client.get("/stats") + assert response.status_code == 200 + if __name__ == "__main__": unittest.main() diff --git a/ui/app.py b/ui/app.py index 60084fd..dc7895e 100644 --- a/ui/app.py +++ b/ui/app.py @@ -2,7 +2,7 @@ import datetime import os import requests -from flask import Flask, render_template, request, abort +from flask import Flask, render_template, request, abort, jsonify from flask_bootstrap import Bootstrap from flask_caching import Cache from flask_migrate import Migrate @@ -10,6 +10,8 @@ from flask_nav import Nav from flask_nav.elements import Navbar, Text, View from flask_sqlalchemy import SQLAlchemy +from sqlalchemy import orm, func + from ui import gitlab, config, models app = Flask(__name__) @@ -22,8 +24,9 @@ nav = Nav(app) nav.register_element('top', Navbar( "LineageOS Builds", - View('Builds', '.index'), - View('Runners', '.runners') + View('Builds', '.web_index'), + View('Runners', '.web_runners'), + View('Stats', '.web_stats') )) headers = {'Private-Token': os.environ.get('GITLAB_TOKEN', '')} @@ -44,31 +47,127 @@ def parse_args(): args['build_date'] = datetime.datetime.strptime(request.args.get('date'), '%Y-%m-%d').date() return args +def stats(): + + runner_build_times = models.Build.query.join(models.Build.build_runner).with_entities( + models.Runner.runner_name, + models.Build.build_version, + func.avg(models.Build.build_duration), + func.max(models.Build.build_duration), + func.min(models.Build.build_duration), + func.sum(models.Build.build_duration) + ).group_by(models.Build.build_version, models.Runner.runner_name).all() + + all_build_times = models.Build.query.with_entities( + models.Build.build_version, + func.avg(models.Build.build_duration), + func.max(models.Build.build_duration), + func.min(models.Build.build_duration), + func.sum(models.Build.build_duration) + ).group_by(models.Build.build_version).all() + + runner_build_status = models.Build.query.join(models.Build.build_runner).with_entities( + models.Runner.runner_name, + models.Build.build_status, + func.count(models.Build.build_status) + ).group_by(models.Runner.runner_name, models.Build.build_status).all() + + + stats = { + 'builds': { + 'all': {} + }, + 'times': { + 'all': {} + } + } + + for build_time in all_build_times: + stats['times']['all'][build_time[0]] = { + 'avg': build_time[1] if build_time[1] else 0, + 'max': build_time[2] if build_time[2] else 0, + 'min': build_time[3] if build_time[3] else 0, + 'sum': build_time[4] if build_time[4] else 0, + } + + for build_time in runner_build_times: + stats['times'].setdefault(build_time[0], {})[build_time[1]] = { + 'avg': build_time[2] if build_time[2] else 0, + 'max': build_time[3] if build_time[3] else 0, + 'min': build_time[4] if build_time[4] else 0, + 'sum': build_time[5] if build_time[5] else 0, + } + + for build_status in runner_build_status: + stats['builds']['all'].setdefault(build_status[1], 0) + stats['builds']['all'][build_status[1]] += build_status[2] + + stats['builds'].setdefault(build_status[0], {})[build_status[1]] = build_status[2] + return stats + @app.route('/') -def index(): +def web_index(): try: args = parse_args() except ValueError: return "Invalid Date", 400 - builds = models.Build.query.filter_by(**args).order_by(models.Build.build_date.desc(), models.Build.build_id).paginate(per_page=20) + builds = models.Build.paginate(args) return render_template('builds.html', builds=builds) @app.route('/runners/') -def runner(runner): +def web_runner(runner): try: args = parse_args() except ValueError: return "Invalid Date", 400 - runner = models.Runner.query.filter_by(runner_name=runner).first() + runner = models.Runner.get({'runner_name': runner}).first() args['build_runner'] = runner - builds = models.Build.query.filter_by(**args).order_by(models.Build.build_date.desc(), models.Build.build_id).paginate(per_page=20) + builds = models.Build.paginate(args) return render_template('runner.html', runner=runner, builds=builds) +@app.route('/stats') +def web_stats(): + return render_template('stats.html', stats=stats()) + @app.route("/runners/") -def runners(): - runners = models.Runner.query.order_by(models.Runner.runner_sponsor, models.Runner.runner_name).all() +def web_runners(): + runners = models.Runner.get().all() return render_template('runners.html', runners=runners) +@app.route('/api/v1/builds') +def api_builds(): + try: + args = parse_args() + except ValueError: + return jsonify({'error': 'Invalid Date'}), 400 + builds = models.Build.paginate(args).items + if not builds: + abort(404) + return jsonify([x.as_dict() for x in builds]) + +@app.route('/api/v1/runners') +def api_runners(): + runners = models.Runner.get().all() + if not runners: + abort(404) + return jsonify([x.as_dict() for x in runners]) + +@app.route('/api/v1/runners/') +def api_runner(runner): + try: + args = parse_args() + except ValueError: + return jsonify({"Invalid Date"}), 400 + runner = models.Runner.get({"runner_name": runner}).first() + if not runner: + abort(404) + return jsonify(runner.as_dict()) + +@app.route('/api/v1/stats') +def api_stats(): + return jsonify(stats()) + +@app.route('/stats') @app.route("/webhook", methods=('POST',)) def process_webhook(): gitlab.webhooks.process(request) diff --git a/ui/config.py b/ui/config.py index 523ea16..d01f6f6 100644 --- a/ui/config.py +++ b/ui/config.py @@ -2,6 +2,7 @@ import os SQLALCHEMY_DATABASE_URI = os.environ.get("SQLALCHEMY_DATABASE_URI", 'sqlite:////tmp/ui.db') SQLALCHEMY_TRACK_MODIFICATIONS = False +SQLALCHEMY_ECHO = 'FLASK_DEBUG' in os.environ CACHE_TYPE = 'simple' GITLAB_WEBHOOK_TOKEN = os.environ.get("GITLAB_WEBHOOK_TOKEN", "secret") diff --git a/ui/models.py b/ui/models.py index 4b27a13..31f3d82 100644 --- a/ui/models.py +++ b/ui/models.py @@ -1,3 +1,4 @@ +import datetime from flask_sqlalchemy import SQLAlchemy db = SQLAlchemy() @@ -16,11 +17,29 @@ class Build(db.Model): def __repr__(self): return f"{self.build_id} {self.build_device} {self.build_version} {self.build_type} {self.build_runner_id}" + def as_dict(self): + return { + 'id': self.build_id, + 'status': self.build_status, + 'device': self.build_device, + 'version': self.build_version, + 'type': self.build_type, + 'date': self.build_date.strftime('%Y-%m-%d'), + 'duration': self.build_duration, + 'runner': self.build_runner.runner_name if self.build_runner else None + } + @classmethod def get_or_create_by_id(cls, build_id): build = cls.query.filter_by(build_id=build_id).first() return build if build else cls(build_id=build_id) + @classmethod + def paginate(cls, args=None): + if not args: + args = {} + return cls.query.filter_by(**args).order_by(cls.build_date.desc(), cls.build_id).paginate(per_page=min([args.get("per_page", 20), 200])) + class Runner(db.Model): runner_id = db.Column(db.String, primary_key=True, autoincrement=False) runner_name = db.Column(db.String) @@ -28,6 +47,14 @@ class Runner(db.Model): runner_sponsor = db.Column(db.String) runner_sponsor_url = db.Column(db.String) + def as_dict(self): + return { + 'id': self.runner_id, + 'name': self.runner_name, + 'sponsor': self.runner_sponsor, + 'sponsor_url': self.runner_sponsor_url + } + @classmethod def get_or_create_by_id(cls, runner_id): runner = cls.query.filter_by(runner_id=runner_id).first() @@ -36,3 +63,9 @@ class Runner(db.Model): db.session.add(runner) db.session.commit() return runner + + @classmethod + def get(cls, args=None): + if not args: + args = {} + return cls.query.filter_by(**args).order_by(cls.runner_sponsor, cls.runner_name) diff --git a/ui/templates/stats.html b/ui/templates/stats.html new file mode 100644 index 0000000..96b9bfb --- /dev/null +++ b/ui/templates/stats.html @@ -0,0 +1,50 @@ +{%- extends "base.html" %} + + +{% import "bootstrap/utils.html" as utils %} +{% from "bootstrap/pagination.html" import render_pagination %} +{% block content %} +{% set statuses = stats['builds']['all'].keys() | sort %} +
+ Status + + + + {% for status in statuses %} + + {% endfor %} + + {% for runner in stats['builds'].keys() | sort %} + + + {% for status in statuses %} + + {% endfor %} + + {% endfor %} +
Runner{{status | title}}
{% if runner != 'all' %}{{runner}}{% else %}{{runner}}{% endif %}{{stats['builds'].get(runner).get(status, 0)}}
+ + {% for version in stats['times']['all'] | sort(reverse=True) %} + Times for {{version}} + + + + + + + + + {% for runner in stats['builds'].keys() | sort %} + + + + + + + + {% endfor %} +
RunnerAverageMinimumMaximumTotal
{% if runner != 'all' %}{{runner}}{% else %}{{runner}}{% endif %}{{(stats['times'][runner][version]['avg'] / 60) | round(2)}} minutes{{(stats['times'][runner][version]['min'] / 60) | round(2)}} minutes{{(stats['times'][runner][version]['max'] / 60) | round(2)}} minutes{{(stats['times'][runner][version]['sum'] / 60 / 60 / 24) | round(2)}} days
+ {% endfor %} + +
+{% endblock %} -- 2.20.1