From 31329c3f5b3c53b9bfcfc8f84d0aa4daa8fcffda Mon Sep 17 00:00:00 2001 From: Tom Powell Date: Sun, 28 Oct 2018 15:29:58 -0700 Subject: [PATCH 1/1] init --- .gitignore | 4 ++ .gitlab-ci.yml | 28 ++++++++ .python-version | 1 + Dockerfile | 8 +++ README.md | 0 migrations | 1 + requirements.txt | 21 ++++++ setup.py | 23 +++++++ test.py | 79 ++++++++++++++++++++++ tests/pipeline_webhook.json | 51 +++++++++++++++ ui/app.py | 30 +++++++++ ui/config.py | 8 +++ ui/gitlab/__init__.py | 1 + ui/gitlab/api.py | 11 ++++ ui/gitlab/webhooks.py | 32 +++++++++ ui/migrations/README | 1 + ui/migrations/alembic.ini | 45 +++++++++++++ ui/migrations/env.py | 87 +++++++++++++++++++++++++ ui/migrations/script.py.mako | 24 +++++++ ui/migrations/versions/c3a5cacf3bff_.py | 44 +++++++++++++ ui/models.py | 34 ++++++++++ ui/templates/index.html | 30 +++++++++ 22 files changed, 563 insertions(+) create mode 100644 .gitignore create mode 100644 .gitlab-ci.yml create mode 100644 .python-version create mode 100644 Dockerfile create mode 100644 README.md create mode 120000 migrations create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 test.py create mode 100644 tests/pipeline_webhook.json create mode 100644 ui/app.py create mode 100644 ui/config.py create mode 100644 ui/gitlab/__init__.py create mode 100644 ui/gitlab/api.py create mode 100644 ui/gitlab/webhooks.py create mode 100644 ui/migrations/README create mode 100644 ui/migrations/alembic.ini create mode 100644 ui/migrations/env.py create mode 100644 ui/migrations/script.py.mako create mode 100644 ui/migrations/versions/c3a5cacf3bff_.py create mode 100644 ui/models.py create mode 100644 ui/templates/index.html diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2dbaffb --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +*.pyc +*.db +__pycache__/ +*.egg-info/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..f481995 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,28 @@ +stages: + - test + - build + +test: + stage: test + image: python:3.6 + script: + - pip install -e . + - python test.py + +build: + stage: build + image: docker:stable + services: + - docker:dind + variables: + DOCKER_HOST: tcp://docker:2375 + DOCKER_DRIVER: overlay2 + script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + - docker build -t $CI_REGISTRY_IMAGE:latest . + - docker tag $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + - docker push $CI_REGISTRY_IMAGE:latest + - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + only: + refs: + - master diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..6c89acc --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +builder-ui diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..356ec78 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM python:3.6 + +COPY . /app +WORKDIR /app +RUN pip install gunicorn +RUN pip install . + +CMD gunicorn -b 0.0.0.0:8080 ui.app:app diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/migrations b/migrations new file mode 120000 index 0000000..cc9f9ce --- /dev/null +++ b/migrations @@ -0,0 +1 @@ +ui/migrations \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2666f32 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,21 @@ +alembic==1.0.1 +certifi==2018.10.15 +chardet==3.0.4 +Click==7.0 +Flask==1.0.2 +Flask-Caching==1.4.0 +Flask-Migrate==2.3.0 +Flask-SQLAlchemy==2.3.2 +Flask-Testing==0.7.1 +idna==2.7 +itsdangerous==1.1.0 +Jinja2==2.10 +Mako==1.0.7 +MarkupSafe==1.0 +python-dateutil==2.7.5 +python-editor==1.0.3 +requests==2.20.0 +six==1.11.0 +SQLAlchemy==1.2.12 +urllib3==1.24 +Werkzeug==0.14.1 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..52fd14d --- /dev/null +++ b/setup.py @@ -0,0 +1,23 @@ +import setuptools + +import setuptools + +with open("README.md", "r") as fh: + long_description = fh.read() +with open("requirements.txt", "r") as fh: + requirements = fh.read().split() +setuptools.setup( + name="ui", + version="0.0.1", + description="LineageOS Builds", + url="https://gitlab.com/lineageos/builder/ui.git", + author_email="infra@lineageos.org", + author="LineageOS Infrastructure Team", + long_description=long_description, + long_description_content_type="text/markdown", + package_dir={"ui": "ui"}, + packages=setuptools.find_packages(), + classifiers=("Programming Language :: Python 3"), + install_requires=requirements, +) + diff --git a/test.py b/test.py new file mode 100644 index 0000000..34470d4 --- /dev/null +++ b/test.py @@ -0,0 +1,79 @@ +import os +import unittest +import flask_migrate +import flask_testing + +from ui import config +from ui.app import app +from ui.models import Build, Runner, db + +class UsesApp(flask_testing.TestCase): + def create_app(self): + return app + +class UsesModels(UsesApp): + def setUp(self): + flask_migrate.upgrade(revision="head") + + def tearDown(self): + flask_migrate.downgrade(revision="base") + +class TestModels(UsesModels): + + def test_build(self): + build = Build(build_id=1) + db.session.add(build) + db.session.commit() + + assert build in db.session + + def test_get_or_create_build_by_id(self): + build = Build(build_id=1) + db.session.add(build) + db.session.commit() + + assert Build.get_or_create_by_id(1).build_id == 1 + assert Build.get_or_create_by_id(2).build_id == 2 + + def test_runner(self): + runner = Runner(runner_id="foobar") + db.session.add(runner) + db.session.commit() + + assert runner in db.session + + def test_get_or_create_runner_by_id(self): + runner = Runner(runner_id="foobar", runner_name="foobar") + db.session.add(runner) + db.session.commit() + + assert Runner.get_or_create_by_id('foobar').runner_name == "foobar" + assert Runner.get_or_create_by_id("foobaz").runner_name == None + + def test_runner_build_rel(self): + runner = Runner(runner_id="foobar", runner_name="foobar") + build = Build(build_id=1, build_runner=runner) + db.session.add(runner) + db.session.add(build) + db.session.commit() + + assert build.build_runner == runner + assert build in runner.builds + +class TestWebhooks(UsesModels): + + def test_invalid_token(self): + with open("tests/pipeline_webhook.json", "r") as f: + response = self.client.post("/webhook", headers={'X-Gitlab-Event': 'Pipeline Hook', 'X-Gitlab-Token': 'invalidsecret'}, data=f.read(), content_type='application/json') + assert response.status_code == 403 + + def test_pipeline_webhook(self): + with open("tests/pipeline_webhook.json", "r") as f: + response = self.client.post("/webhook", headers={'X-Gitlab-Event': 'Pipeline Hook', 'X-Gitlab-Token': 'secret'}, data=f.read(), content_type='application/json') + assert response.status_code == 200 + build = Build.query.filter_by(build_id=1).first() + assert build.build_id == 1 + assert build.build_runner_id == "foobar" + +if __name__ == "__main__": + unittest.main() diff --git a/tests/pipeline_webhook.json b/tests/pipeline_webhook.json new file mode 100644 index 0000000..510cc48 --- /dev/null +++ b/tests/pipeline_webhook.json @@ -0,0 +1,51 @@ +{ + "object_kind": "pipeline", + "object_attributes":{ + "id": 1, + "status": "success", + "stages":[ + "build", + "sign" + ], + "duration": 63, + "variables": [ + { + "key": "VERSION", + "value": "lineage-15.1" + }, + { + "key": "DEVICE", + "value": "mako" + }, + { + "key": "TYPE", + "value": "userdebug" + } + ] + }, + "builds":[ + { + "id": 380, + "stage": "build", + "name": "build", + "status": "success", + "created_at": "2016-08-12 15:23:28 UTC", + "when": "manual", + "manual": true, + "user":{ + "name": "Administrator", + "username": "root", + "avatar_url": "" + }, + "runner": "foobar" + }, + { + "id": 377, + "stage": "sign", + "name": "sign", + "status": "success", + "created_at": "2016-08-12 15:23:28 UTC", + "runner": "foobar" + } + ] +} diff --git a/ui/app.py b/ui/app.py new file mode 100644 index 0000000..e693962 --- /dev/null +++ b/ui/app.py @@ -0,0 +1,30 @@ +import os + +import requests +from flask import Flask, render_template, request, abort +from flask_caching import Cache +from flask_migrate import Migrate +from flask_sqlalchemy import SQLAlchemy + +from ui import gitlab, config, models + +app = Flask(__name__) +app.config.from_object(config) +cache = Cache(app) +models.db.init_app(app) +migrate = Migrate(app, models.db) + + +headers = {'Private-Token': os.environ.get('GITLAB_TOKEN', '')} + +@app.route('/') +def main(): + return render_template('index.html') + +@app.route("/webhook", methods=('POST',)) +def process_webhook(): + gitlab.webhooks.process(request) + return "OK", 200 + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=5000) diff --git a/ui/config.py b/ui/config.py new file mode 100644 index 0000000..4602370 --- /dev/null +++ b/ui/config.py @@ -0,0 +1,8 @@ +import os + +SQLALCHEMY_DATABASE_URI = os.environ.get("SQLALCHEMY_DATABASE_URI", 'sqlite:////tmp/ui.db') +CACHE_TYPE = 'simple' + +GITLAB_WEBHOOK_TOKEN = 'secret' + +PRESERVE_CONTEXT_ON_EXCEPTION = False diff --git a/ui/gitlab/__init__.py b/ui/gitlab/__init__.py new file mode 100644 index 0000000..0008ce0 --- /dev/null +++ b/ui/gitlab/__init__.py @@ -0,0 +1 @@ +from . import api, webhooks diff --git a/ui/gitlab/api.py b/ui/gitlab/api.py new file mode 100644 index 0000000..463e132 --- /dev/null +++ b/ui/gitlab/api.py @@ -0,0 +1,11 @@ +from datetime import datetime +import requests +import os + +GITLAB_BASE = "https://gitlab.com/api/v4/" + +def create_pipeline(project, trigger, data): + resp = requests.post(f"{GITLAB_BASE}projects/{project}/trigger/pipeline?token={token}", data=data) + if resp.status_code != 201: + raise Exception("ERROR", req.status_code, req.json()) + return resp.json() diff --git a/ui/gitlab/webhooks.py b/ui/gitlab/webhooks.py new file mode 100644 index 0000000..16612d3 --- /dev/null +++ b/ui/gitlab/webhooks.py @@ -0,0 +1,32 @@ +from flask import abort + +from ui import config +from ui.gitlab import api +from ui.models import Build, Runner, db + +def process(request): + if request.headers.get('X-Gitlab-Token', None) != config.GITLAB_WEBHOOK_TOKEN: + abort(403) + webhook_type = request.headers.get('X-Gitlab-Event') + data = request.get_json() + if webhook_type == 'Pipeline Hook': + pipeline = data.get('object_attributes') + stages = data.get('builds') + build = Build.get_or_create_by_id(pipeline.get('id')) + build.status = pipeline.get('status') + build.duration = pipeline.get('duration') + for variable in pipeline.get("variables"): + if variable.get('key') == "VERSION": + build.build_version = variable.get("value") + elif variable.get("key") == "DEVICE": + build.build_device = variable.get("value") + elif variable.get("key") == "TYPE": + build.build_type = variable.get("value") + build_stage = {} + for stage in stages: + if stage.get('name') == 'build': + build_stage = stage + if build_stage.get("runner"): + build.build_runner = Runner.get_or_create_by_id(build_stage['runner']) + db.session.add(build) + db.session.commit() diff --git a/ui/migrations/README b/ui/migrations/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/ui/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/ui/migrations/alembic.ini b/ui/migrations/alembic.ini new file mode 100644 index 0000000..f8ed480 --- /dev/null +++ b/ui/migrations/alembic.ini @@ -0,0 +1,45 @@ +# A generic, single database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/ui/migrations/env.py b/ui/migrations/env.py new file mode 100644 index 0000000..23663ff --- /dev/null +++ b/ui/migrations/env.py @@ -0,0 +1,87 @@ +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig +import logging + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger('alembic.env') + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from flask import current_app +config.set_main_option('sqlalchemy.url', + current_app.config.get('SQLALCHEMY_DATABASE_URI')) +target_metadata = current_app.extensions['migrate'].db.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure(url=url) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + engine = engine_from_config(config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + connection = engine.connect() + context.configure(connection=connection, + target_metadata=target_metadata, + process_revision_directives=process_revision_directives, + **current_app.extensions['migrate'].configure_args) + + try: + with context.begin_transaction(): + context.run_migrations() + finally: + connection.close() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/ui/migrations/script.py.mako b/ui/migrations/script.py.mako new file mode 100644 index 0000000..2c01563 --- /dev/null +++ b/ui/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/ui/migrations/versions/c3a5cacf3bff_.py b/ui/migrations/versions/c3a5cacf3bff_.py new file mode 100644 index 0000000..d939826 --- /dev/null +++ b/ui/migrations/versions/c3a5cacf3bff_.py @@ -0,0 +1,44 @@ +"""empty message + +Revision ID: c3a5cacf3bff +Revises: +Create Date: 2018-10-28 14:17:59.970010 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c3a5cacf3bff' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('runner', + sa.Column('runner_id', sa.String(), autoincrement=False, nullable=False), + sa.Column('runner_name', sa.String(), nullable=True), + sa.PrimaryKeyConstraint('runner_id') + ) + op.create_table('build', + sa.Column('build_id', sa.Integer(), autoincrement=False, nullable=False), + sa.Column('build_status', sa.String(), nullable=True), + sa.Column('build_device', sa.String(), nullable=True), + sa.Column('build_version', sa.String(), nullable=True), + sa.Column('build_type', sa.String(), nullable=True), + sa.Column('build_date', sa.Date(), nullable=True), + sa.Column('build_runner_id', sa.String(), nullable=True), + sa.ForeignKeyConstraint(['build_runner_id'], ['runner.runner_id'], ), + sa.PrimaryKeyConstraint('build_id') + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('build') + op.drop_table('runner') + # ### end Alembic commands ### diff --git a/ui/models.py b/ui/models.py new file mode 100644 index 0000000..a65fb16 --- /dev/null +++ b/ui/models.py @@ -0,0 +1,34 @@ +from flask_sqlalchemy import SQLAlchemy + +db = SQLAlchemy() + +class Build(db.Model): + build_id = db.Column(db.Integer, primary_key=True, autoincrement=False) + build_status = db.Column(db.String) + build_device = db.Column(db.String) + build_version = db.Column(db.String) + build_type = db.Column(db.String) + build_date = db.Column(db.Date) + build_runner_id = db.Column(db.String, db.ForeignKey('runner.runner_id')) + build_runner = db.relationship('Runner', backref=db.backref('builds', lazy=True)) + + def __repr__(self): + return f"{self.build_id} {self.build_device} {self.build_version} {self.build_type} {self.build_runner_id}" + + @classmethod + def get_or_create_by_id(cls, build_id): + build = cls.query.filter_by(build_id=build_id).first() + return build if build else cls(build_id=build_id) + +class Runner(db.Model): + runner_id = db.Column(db.String, primary_key=True, autoincrement=False) + runner_name = db.Column(db.String) + + @classmethod + def get_or_create_by_id(cls, runner_id): + runner = cls.query.filter_by(runner_id=runner_id).first() + if not runner: + runner = cls(runner_id=runner_id) + db.session.add(runner) + db.session.commit() + return runner diff --git a/ui/templates/index.html b/ui/templates/index.html new file mode 100644 index 0000000..6de2b91 --- /dev/null +++ b/ui/templates/index.html @@ -0,0 +1,30 @@ + + + + + LineageOS Builds + + + + + + + + + + + + {% for build in builds %} + + + + + + + + + {% endfor %} +
IDStatusDeviceVersionTypeDate
{{build.id}}{{build.status}}{{build.device}}{{build.version}}{{build.type}}{{build.date}}
+ + + -- 2.20.1