From a6ccbd113de22ddf606440c05a4134fb28a0d380 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Mon, 13 Jan 2020 20:07:00 +0100 Subject: fetch and display bugs Retrieve bugs from udd-mirror by tags/usertags and display relevant bug numbers in the webapp. --- fetchbugs.py | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ schema.sql | 9 ++++++ webapp.py | 65 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 169 insertions(+), 2 deletions(-) create mode 100755 fetchbugs.py diff --git a/fetchbugs.py b/fetchbugs.py new file mode 100755 index 0000000..5b86416 --- /dev/null +++ b/fetchbugs.py @@ -0,0 +1,97 @@ +#!/usr/bin/python3 +# SPDX-License-Identifier: GPL-2.0+ + +import logging + +import sqlalchemy + +logger = logging.getLogger() + +def strip_title(title, package): + for prefix in (package + ": ", package + " "): + if title.startswith(prefix): + return title[len(prefix):] + return title + +def get_affects(result, kind): + for bugnum, package, affected, title, patched in result: + row = dict(bugnum=bugnum, package=package, patched=patched, kind=kind) + somesource = False + for pkg in package.split(","): + if not pkg.startswith("src:"): + continue + arow = row.copy() + arow.update(affects=pkg[4:], title=strip_title(title, pkg[4:])) + yield arow + if somesource: + logger.warning("%s #%d assigned to multiple source packages", + kind, bugnum) + somesource = True + if not somesource and affected: + for aff in affected.split(","): + if not aff.startswith("src:"): + continue + arow = row.copy() + arow.update(affects=aff[4:], title=strip_title(title, aff[4:])) + yield arow + somesource = True + if not somesource: + logger.warning("%s #%d affects no packages", kind, bugnum) + +def make_table(name, columns): + return sqlalchemy.table(name, *map(sqlalchemy.column, columns.split())) + +udd_all_bugs = make_table( + "all_bugs", "id package affected_packages title severity affects_unstable") +udd_bugs_tags = make_table("bugs_tags", "id tag") +udd_bugs_usertags = make_table("bugs_usertags", "id email tag") + +def tagged_clause(id_, tag): + return sqlalchemy.exists().where( + sqlalchemy.and_(udd_bugs_tags.c.id == id_, udd_bugs_tags.c.tag == tag)) + +def get_bugs_where(conn, whereclause, kind): + query = sqlalchemy.select( + [udd_all_bugs.c.id, udd_all_bugs.c.package, + udd_all_bugs.c.affected_packages, udd_all_bugs.c.title, + tagged_clause(udd_all_bugs.c.id, "patch")], + sqlalchemy.and_(whereclause, udd_all_bugs.c.affects_unstable == 't')) + return get_affects(conn.execute(query), kind) + +def get_ftbfs(conn): + clause = sqlalchemy.and_( + tagged_clause(udd_all_bugs.c.id, "ftbfs"), + udd_all_bugs.c.severity.in_(["serious", "critical", "grave"])) + return get_bugs_where(conn, clause, 'ftbfs') + +def get_usertagged(conn, email, tag, kind): + clause = sqlalchemy.exists().where( + sqlalchemy.and_(udd_bugs_usertags.c.id == udd_all_bugs.c.id, + udd_bugs_usertags.c.email == email, + udd_bugs_usertags.c.tag == tag)) + return get_bugs_where(conn, clause, kind) + +def get_bugs(conn): + yield from get_ftbfs(conn) + email = 'debian-cross@lists.debian.org' + yield from get_usertagged(conn, email, 'cross-satisfiability', 'bdsat') + yield from get_usertagged(conn, email, 'ftcbfs', 'ftcbfs') + +def main(): + udde = sqlalchemy.create_engine( + 'postgresql://udd-mirror:udd-mirror@udd-mirror.debian.net/' + 'udd?client_encoding=utf8') + with udde.connect() as conn: + bugs = list(get_bugs(conn)) + + query = sqlalchemy.text(""" + INSERT INTO bugs (kind, bugnum, package, affects, title, patched) + VALUES (:kind, :bugnum, :package, :affects, :title, :patched);""") + crosse = sqlalchemy.create_engine('sqlite:///db') + with crosse.connect() as conn: + with conn.begin(): + conn.execute("DELETE FROM bugs;") + conn.execute(query, bugs) + +if __name__ == "__main__": + main() diff --git a/schema.sql b/schema.sql index c785080..ccd78ec 100644 --- a/schema.sql +++ b/schema.sql @@ -38,3 +38,12 @@ CREATE TABLE buildrequests ( architecture TEXT, requesttime TIMESTAMP NOT NULL, priority INTEGER NOT NULL DEFAULT 0); + +CREATE TABLE bugs ( + kind TEXT NOT NULL CHECK (kind in ('bdsat', 'ftbfs', 'ftcbfs')), + bugnum INTEGER NOT NULL, + package TEXT, + affects TEXT NOT NULL, + title TEXT, + patched BOOLEAN NOT NULL CHECK (patched in (0, 1))); +CREATE INDEX bugs_affects_index ON bugs(affects); diff --git a/webapp.py b/webapp.py index 6d1ad76..8acf92f 100644 --- a/webapp.py +++ b/webapp.py @@ -38,6 +38,7 @@ index_template = """ architecture started result log + bugs @@ -57,6 +58,13 @@ index_template = """ log xz + + {%- if build.buglvl == 2 -%} + patch reported + {%- elif build.buglvl == 1 -%} + bug reported + {%- endif -%} + {%- endfor -%} @@ -73,7 +81,19 @@ index_template = """ """ -src_template = """ +src_template = """ +{%- macro render_bug(bugobj) -%} + #{{ bugobj.bugnum }} + {%- if bugobj.patched %} + [+] + {%- endif -%}: + {% if bugobj.package != "src:" + bugobj.affects and + not bugobj.title.startswith(bugobj.affects + ":") -%} + {{- bugobj.package|e }}: + {% endif -%} + {{- bugobj.title|e -}} +{%- endmacro -%} + {{ sourcepackage|e }} - Debian cross build @@ -121,8 +141,30 @@ footer { + {%- if bugs.ftbfs -%} +
+

Reported FTBFS bugs

+ +
+ {%- endif -%}

Cross build dependency satisfiability

+ {%- if bugs.bdsat -%} +
Reported satisfiability problems
+ + {%- endif -%} @@ -162,6 +204,16 @@ footer {

Cross builds

+ {%- if bugs.ftcbfs -%} +
Reported cross build failures
+
    + {%- for bug in bugs.ftcbfs|sort(attribute="bugnum") -%} +
  • + {{- render_bug(bug) -}} +
    • + {%- endfor -%} +
+ {%- endif -%} {%- if builds -%}
@@ -285,7 +337,10 @@ def collect_depstate(conn, source): def show_index(): with db.engine.connect() as conn: builds = list(conn.execute(""" - SELECT source, version, architecture, starttime, filename + SELECT source, version, architecture, starttime, filename, + ifnull((SELECT max(patched + 1) FROM bugs + WHERE affects = source), + 0) AS buglvl FROM builds WHERE success = 0 ORDER BY starttime @@ -304,6 +359,12 @@ def show_source(source): SELECT version, architecture, success, starttime, filename FROM builds WHERE source = :source;""") context["builds"] = list(conn.execute(query, source=source)) + query = sqlalchemy.text(""" + SELECT bugnum, kind, title, package, patched, affects + FROM bugs WHERE affects = :affects;""") + context["bugs"] = {} + for bug in conn.execute(query, affects=source): + context["bugs"].setdefault(bug.kind, []).append(bug) context["show_bootstrapdn"] = \ any(reason and not reason.startswith("skew ") for reason in context["depresult"].keys()) -- cgit v1.2.3