fetch and display bugs
authorHelmut Grohne <helmut@subdivi.de>
Mon, 13 Jan 2020 19:07:00 +0000 (20:07 +0100)
committerHelmut Grohne <helmut@subdivi.de>
Mon, 13 Jan 2020 19:07:00 +0000 (20:07 +0100)
Retrieve bugs from udd-mirror by tags/usertags and display relevant bug
numbers in the webapp.

fetchbugs.py [new file with mode: 0755]
schema.sql
webapp.py

diff --git a/fetchbugs.py b/fetchbugs.py
new file mode 100755 (executable)
index 0000000..5b86416
--- /dev/null
@@ -0,0 +1,97 @@
+#!/usr/bin/python3
+# SPDX-License-Identifier: GPL-2.0+
+
+import logging
+
+import sqlalchemy
+
+logger = logging.getLogger()
+
+def strip_title(title, package):
+    for prefix in (package + ": ", package + " "):
+        if title.startswith(prefix):
+            return title[len(prefix):]
+    return title
+
+def get_affects(result, kind):
+    for bugnum, package, affected, title, patched in result:
+        row = dict(bugnum=bugnum, package=package, patched=patched, kind=kind)
+        somesource = False
+        for pkg in package.split(","):
+            if not pkg.startswith("src:"):
+                continue
+            arow = row.copy()
+            arow.update(affects=pkg[4:], title=strip_title(title, pkg[4:]))
+            yield arow
+            if somesource:
+                logger.warning("%s #%d assigned to multiple source packages",
+                               kind, bugnum)
+            somesource = True
+        if not somesource and affected:
+            for aff in affected.split(","):
+                if not aff.startswith("src:"):
+                    continue
+                arow = row.copy()
+                arow.update(affects=aff[4:], title=strip_title(title, aff[4:]))
+                yield arow
+                somesource = True
+        if not somesource:
+            logger.warning("%s #%d affects no packages", kind, bugnum)
+
+def make_table(name, columns):
+    return sqlalchemy.table(name, *map(sqlalchemy.column, columns.split()))
+
+udd_all_bugs = make_table(
+    "all_bugs", "id package affected_packages title severity affects_unstable")
+udd_bugs_tags = make_table("bugs_tags", "id tag")
+udd_bugs_usertags = make_table("bugs_usertags", "id email tag")
+
+def tagged_clause(id_, tag):
+    return sqlalchemy.exists().where(
+        sqlalchemy.and_(udd_bugs_tags.c.id == id_, udd_bugs_tags.c.tag == tag))
+
+def get_bugs_where(conn, whereclause, kind):
+    query = sqlalchemy.select(
+        [udd_all_bugs.c.id, udd_all_bugs.c.package,
+         udd_all_bugs.c.affected_packages, udd_all_bugs.c.title,
+         tagged_clause(udd_all_bugs.c.id, "patch")],
+        sqlalchemy.and_(whereclause, udd_all_bugs.c.affects_unstable == 't'))
+    return get_affects(conn.execute(query), kind)
+
+def get_ftbfs(conn):
+    clause = sqlalchemy.and_(
+        tagged_clause(udd_all_bugs.c.id, "ftbfs"),
+        udd_all_bugs.c.severity.in_(["serious", "critical", "grave"]))
+    return get_bugs_where(conn, clause, 'ftbfs')
+
+def get_usertagged(conn, email, tag, kind):
+    clause = sqlalchemy.exists().where(
+        sqlalchemy.and_(udd_bugs_usertags.c.id == udd_all_bugs.c.id,
+                        udd_bugs_usertags.c.email == email,
+                        udd_bugs_usertags.c.tag == tag))
+    return get_bugs_where(conn, clause, kind)
+
+def get_bugs(conn):
+    yield from get_ftbfs(conn)
+    email = 'debian-cross@lists.debian.org'
+    yield from get_usertagged(conn, email, 'cross-satisfiability', 'bdsat')
+    yield from get_usertagged(conn, email, 'ftcbfs', 'ftcbfs')
+
+def main():
+    udde = sqlalchemy.create_engine(
+        'postgresql://udd-mirror:udd-mirror@udd-mirror.debian.net/'
+        'udd?client_encoding=utf8')
+    with udde.connect() as conn:
+        bugs = list(get_bugs(conn))
+
+    query = sqlalchemy.text("""
+        INSERT INTO bugs (kind, bugnum, package, affects, title, patched)
+            VALUES (:kind, :bugnum, :package, :affects, :title, :patched);""")
+    crosse = sqlalchemy.create_engine('sqlite:///db')
+    with crosse.connect() as conn:
+        with conn.begin():
+            conn.execute("DELETE FROM bugs;")
+            conn.execute(query, bugs)
+
+if __name__ == "__main__":
+    main()
index c785080..ccd78ec 100644 (file)
@@ -38,3 +38,12 @@ CREATE TABLE buildrequests (
        architecture TEXT,
        requesttime TIMESTAMP NOT NULL,
         priority INTEGER NOT NULL DEFAULT 0);
+
+CREATE TABLE bugs (
+       kind TEXT NOT NULL CHECK (kind in ('bdsat', 'ftbfs', 'ftcbfs')),
+       bugnum INTEGER NOT NULL,
+       package TEXT,
+       affects TEXT NOT NULL,
+       title TEXT,
+       patched BOOLEAN NOT NULL CHECK (patched in (0, 1)));
+CREATE INDEX bugs_affects_index ON bugs(affects);
index 6d1ad76..8acf92f 100644 (file)
--- a/webapp.py
+++ b/webapp.py
@@ -38,6 +38,7 @@ index_template = """<!DOCTYPE html>
       <th>architecture</th>
       <th>started</th>
       <th>result log</th>
+      <th>bugs</th>
      </tr>
     </thead>
     <tbody>
@@ -57,6 +58,13 @@ index_template = """<!DOCTYPE html>
         <a href="{{ url_for("show_log", filename=build.filename[:-3]) }}">log</a>
         <a href="{{ url_for("show_log", filename=build.filename) }}">xz</a>
        </td>
+       <td>
+        {%- if build.buglvl == 2 -%}
+         patch reported
+        {%- elif build.buglvl == 1 -%}
+         bug reported
+        {%- endif -%}
+       </td>
       </tr>
      {%- endfor -%}
     </tbody>
@@ -73,7 +81,19 @@ index_template = """<!DOCTYPE html>
 </html>
 """
 
-src_template = """<!DOCTYPE html>
+src_template = """
+{%- macro render_bug(bugobj) -%}
+ <a href="https://bugs.debian.org/{{ bugobj.bugnum }}">#{{ bugobj.bugnum }}</a>
+ {%- if bugobj.patched %}
+  [<abbr title="patch available">+</abbr>]
+ {%- endif -%}:
+ {% if bugobj.package != "src:" + bugobj.affects and
+       not bugobj.title.startswith(bugobj.affects + ":") -%}
+  {{- bugobj.package|e }}:
+ {% endif -%}
+ {{- bugobj.title|e -}}
+{%- endmacro -%}
+<!DOCTYPE html>
 <html>
  <head>
   <title>{{ sourcepackage|e }} - Debian cross build</title>
@@ -121,8 +141,30 @@ footer {
     </a>
    </h1>
   </header>
+  {%- if bugs.ftbfs -%}
+   <section>
+    <h3>Reported <abbr title="fails to build from source">FTBFS</abbr> bugs</h3>
+    <ul>
+     {%- for bug in bugs.ftbfs|sort(attribute="bugnum") -%}
+      <li>
+       {{- render_bug(bug) -}}
+      </li>
+     {%- endfor -%}
+    </ul>
+   </section>
+  {%- endif -%}
   <section>
    <h3>Cross build dependency satisfiability</h3>
+   {%- if bugs.bdsat -%}
+    <h5>Reported satisfiability problems</h5>
+    <ul>
+     {%- for bug in bugs.bdsat|sort(attribute="bugnum") -%}
+      <li>
+       {{- render_bug(bug) -}}
+      </li>
+     {%- endfor -%}
+    </ul>
+   {%- endif -%}
    <table>
     <thead>
      <tr>
@@ -162,6 +204,16 @@ footer {
   </section>
   <section>
    <h3>Cross builds</h3>
+   {%- if bugs.ftcbfs -%}
+    <h5>Reported cross build failures</h5>
+    <ul>
+     {%- for bug in bugs.ftcbfs|sort(attribute="bugnum") -%}
+      <li>
+       {{- render_bug(bug) -}}
+      </li>
+     {%- endfor -%}
+    </ul>
+   {%- endif -%}
    {%- if builds -%}
     <table>
      <thead>
@@ -285,7 +337,10 @@ def collect_depstate(conn, source):
 def show_index():
     with db.engine.connect() as conn:
         builds = list(conn.execute("""
-            SELECT source, version, architecture, starttime, filename
+            SELECT source, version, architecture, starttime, filename,
+                ifnull((SELECT max(patched + 1) FROM bugs
+                                                WHERE affects = source),
+                       0) AS buglvl
                 FROM builds
                 WHERE success = 0
                 ORDER BY starttime
@@ -304,6 +359,12 @@ def show_source(source):
             SELECT version, architecture, success, starttime, filename
                 FROM builds WHERE source = :source;""")
         context["builds"] = list(conn.execute(query, source=source))
+        query = sqlalchemy.text("""
+            SELECT bugnum, kind, title, package, patched, affects
+                FROM bugs WHERE affects = :affects;""")
+        context["bugs"] = {}
+        for bug in conn.execute(query, affects=source):
+            context["bugs"].setdefault(bug.kind, []).append(bug)
         context["show_bootstrapdn"] = \
                 any(reason and not reason.startswith("skew ")
                     for reason in context["depresult"].keys())