#!/usr/bin/python3 # SPDX-License-Identifier: GPL-2.0+ import datetime import lzma import apt_pkg apt_pkg.init() version_compare = apt_pkg.version_compare import flask import flask_sqlalchemy import jinja2 import sqlalchemy import werkzeug import werkzeug.security app = flask.Flask("crossqa") app.config["SQLALCHEMY_DATABASE_URI"] = 'sqlite:///db' app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False db = flask_sqlalchemy.SQLAlchemy(app) index_template = """ Debian cross build quality assurance

Debian cross build quality assurance

Recently failed builds

{%- for build in builds|sort(attribute='starttime', reverse=true) -%} {%- endfor -%}
source version build architecture host architecture started result log bugs
{{- build.source|e -}} {{ build.version|e }} {{ build.buildarch|e }} {{ build.hostarch|e }} {{- build.starttime|sqltimestamp|formatts -}} log xz {%- if build.buglvl == 2 -%} patch reported {%- elif build.buglvl == 1 -%} bug reported {%- endif -%}
""" src_template = """ {%- macro render_bug(bugobj) -%} #{{ bugobj.bugnum }} {%- if bugobj.patched %} [+] {%- endif -%}: {% if bugobj.package != "src:" + bugobj.affects and not bugobj.title.startswith(bugobj.affects + ":") -%} {{- bugobj.package|e }}: {% endif -%} {{- bugobj.title|e -}} {%- endmacro -%} {{ sourcepackage|e }} - Debian cross build

{{- sourcepackage|e -}}

{%- if bugs.ftbfs -%}

Reported FTBFS bugs

{%- endif -%}

Cross build dependency satisfiability

{%- if bugs.bdsat -%}
Reported satisfiability problems
{%- endif -%} {%- set okarchs = depresult.pop(None, None) -%} {%- for reason, archs in depresult.items()|sort -%} {%- endfor -%} {%- if okarchs -%} {%- endif -%}
state architectures
{{ reason|e }} {{ archs|archpairs_format }}
ok {{ okarchs|archpairs_format }}
{%- if show_debcheck -%}
See also
{%- endif -%}

Cross builds

{%- if bugs.ftcbfs -%}
Reported cross build failures
{%- endif -%} {%- if builds -%} {%- for build in builds|sort(attribute='starttime', reverse=true) -%} {%- endfor -%}
started version build architecture host architecture result log
{{- build.starttime|sqltimestamp|formatts -}} {{ build.version|e }} {{ build.buildarch|e }} {{ build.hostarch|e }} {{- "ok" if build.success else "failed" -}} xz
{%- else -%}

No build performed yet.

{%- endif -%}
{{ sourcepackage|e }} for
""" schedule_template = """

Scheduled a build of {{ request.form["source"]|e }} {%- if buildarch or hostarch %} for {{ (buildarch|default("any"), hostarch|default("any"))|archpair_format -}} {%- endif %}.

""" @app.template_filter("sqltimestamp") def sqltimestamp_filter(s): strptime = datetime.datetime.strptime try: return strptime(s, "%Y-%m-%d %H:%M:%S.%f").replace(microsecond=0) except ValueError: return strptime(s, "%Y-%m-%d %H:%M:%S") def formatts(ts): assert isinstance(ts, datetime.datetime) dt = datetime.datetime.utcnow() - ts if dt < datetime.timedelta(seconds=1): return "now" if dt < datetime.timedelta(seconds=100): return "%d s" % dt.seconds if dt < datetime.timedelta(minutes=100): return "%d m" % (dt.seconds // 60) if dt < datetime.timedelta(days=1): return "%d h" % (dt.seconds // (60 * 60)) return "%d d" % dt.days @app.template_filter("formatts") def formatts_filter(ts): return jinja2.utils.markupsafe.Markup( '' % (ts, ts, formatts(ts)) ) @app.template_filter("archpair_format") def archpair_format_filter(archpair): return jinja2.utils.markupsafe.Markup( "%s → %s" % tuple(map(jinja2.utils.markupsafe.escape, archpair)) ) def group_pairs(pairs): result = {} for v, w in pairs: result.setdefault(v, set()).add(w) return result def render_archset(subset, all_archs): if len(subset) == 1: return next(iter(subset)) if subset == all_archs: return "any" return "{%s}" % ", ".join( map(jinja2.utils.markupsafe.escape, sorted(subset)) ) @app.template_filter('archpairs_format') @jinja2.pass_context def archpairs_format_filter(context, some_archs): architectures = group_pairs(context["architectures"]) fwdmap = {} # build architecture -> host architecture set representation for buildarch, hostarchs in group_pairs(some_archs).items(): fwdmap[buildarch] = render_archset(hostarchs, architectures[buildarch]) allbuildarchs = set(architectures.keys()) # host architecture set representation -> build architecture set flippedit = group_pairs((v, k) for (k, v) in fwdmap.items()).items() maps = ("%s → %s" % (render_archset(buildarchs, allbuildarchs), hostarchrep) for hostarchrep, buildarchs in flippedit) return jinja2.utils.markupsafe.Markup("; ".join(sorted(maps))) def collect_depstate(conn, source): version = None depstate = None query = sqlalchemy.text(""" SELECT version, buildarch, hostarch, satisfiable, reason FROM depstate WHERE source = :source;""") for row in conn.execute(query, source=source): if version is None or version_compare(version, row.version) > 0: version = row.version depstate = {} depstate[row.buildarch, row.hostarch] = \ None if row.satisfiable else row.reason if version is None: raise werkzeug.exceptions.NotFound() depresult = {} for archpair, reason in depstate.items(): depresult.setdefault(reason, set()).add(archpair) return version, depresult @app.route("/") def show_index(): with db.engine.connect() as conn: builds = list(conn.execute(""" SELECT source, version, buildarch, hostarch, starttime, filename, ifnull((SELECT max(patched + 1) FROM bugs WHERE affects = source), 0) AS buglvl FROM builds WHERE success = 0 ORDER BY starttime DESC LIMIT 10;""")) return flask.render_template_string(index_template, builds=builds) @app.route("/src/") def show_source(source): context = dict(sourcepackage=source) with db.engine.connect() as conn: query = sqlalchemy.text("SELECT buildarch, hostarch FROM depcheck;") context["architectures"] = set(map(tuple, conn.execute(query))) context["version"], context["depresult"] = collect_depstate(conn, source) query = sqlalchemy.text(""" SELECT version, buildarch, hostarch, success, starttime, filename FROM builds WHERE source = :source;""") context["builds"] = list(conn.execute(query, source=source)) query = sqlalchemy.text(""" SELECT bugnum, kind, title, package, patched, affects FROM bugs WHERE affects = :affects;""") context["bugs"] = {} for bug in conn.execute(query, affects=source): context["bugs"].setdefault(bug.kind, []).append(bug) context["show_bootstrapdn"] = \ any(reason and not reason.startswith("skew ") for reason in context["depresult"].keys()) context["show_debcheck"] = \ any(context["depresult"].keys()) return flask.render_template_string(src_template, **context) @app.route("/build/") def show_log(filename): if filename.endswith(".xz"): return flask.send_from_directory("logs", filename, mimetype="application/octet-stream") filename = werkzeug.security.safe_join("logs", filename + ".xz") try: return flask.send_file(lzma.open(filename, "rb"), mimetype="text/plain") except FileNotFoundError: raise werkzeug.exceptions.NotFound() @app.route("/schedule", methods=["POST"]) def request_schedule(): source = flask.request.form["source"] try: buildarch, hostarch = flask.request.form["archpair"].split("_") except ValueError: raise werkzeug.exceptions.BadRequest() if buildarch == "any": buildarch = None if hostarch == "any": hostarch = None with db.engine.connect() as conn: query = sqlalchemy.text(""" SELECT 1 FROM depstate WHERE source = :source;""") if not conn.execute(query, source=source).first(): raise werkzeug.exceptions.BadRequest() query = sqlalchemy.text(""" SELECT 1 FROM depcheck WHERE buildarch = ifnull(:buildarch, buildarch) AND hostarch = ifnull(:hostarch, hostarch);""") if not conn.execute(query, buildarch=buildarch, hostarch=hostarch).first(): raise werkzeug.exceptions.BadRequest() query = sqlalchemy.text(""" INSERT INTO buildrequests (source, buildarch, hostarch, requesttime) VALUES (:source, :buildarch, :hostarch, datetime('now'));""") conn.execute(query, source=source, buildarch=buildarch, hostarch=hostarch) return flask.render_template_string(schedule_template, buildarch=buildarch, hostarch=hostarch)