depcheck: add python3.9-minimal to bad_foreign_packages
[~helmut/crossqa.git] / webapp.py
1 #!/usr/bin/python3
2 # SPDX-License-Identifier: GPL-2.0+
3
4 import datetime
5 import lzma
6
7 import apt_pkg
8 apt_pkg.init()
9 version_compare = apt_pkg.version_compare
10
11 import flask
12 import flask_sqlalchemy
13 import jinja2
14 import sqlalchemy
15 import werkzeug
16
17 app = flask.Flask("crossqa")
18 app.config["SQLALCHEMY_DATABASE_URI"] = 'sqlite:///db'
19 app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
20 db = flask_sqlalchemy.SQLAlchemy(app)
21
22 index_template = """<!DOCTYPE html>
23 <html>
24  <head>
25   <title>Debian cross build quality assurance</title>
26  </head>
27  <body>
28   <header>
29    <h1>Debian cross build quality assurance</h1>
30   </header>
31   <section>
32    <h3>Recently failed builds</h3>
33    <table>
34     <thead>
35      <tr>
36       <th>source</th>
37       <th>version</th>
38       <th>architecture</th>
39       <th>started</th>
40       <th>result log</th>
41       <th>bugs</th>
42      </tr>
43     </thead>
44     <tbody>
45      {%- for build in builds|sort(attribute='starttime', reverse=true) -%}
46       <tr>
47        <td>
48         <a href="{{ url_for("show_source", source=build.source) }}">
49          {{- build.source|e -}}
50         </a>
51        </td>
52        <td>{{ build.version|e }}</td>
53        <td>{{ build.architecture|e }}</td>
54        <td>
55         {{- build.starttime|sqltimestamp|formatts -}}
56        </td>
57        <td>
58         <a href="{{ url_for("show_log", filename=build.filename[:-3]) }}">log</a>
59         <a href="{{ url_for("show_log", filename=build.filename) }}">xz</a>
60        </td>
61        <td>
62         {%- if build.buglvl == 2 -%}
63          patch reported
64         {%- elif build.buglvl == 1 -%}
65          bug reported
66         {%- endif -%}
67        </td>
68       </tr>
69      {%- endfor -%}
70     </tbody>
71    </table>
72   </section>
73   <footer>
74    <h3>Details about this service</h3>
75    <ul>
76     <li>Maintainer: Helmut Grohne &lt;helmut@subdivi.de&gt;</li>
77     <li>Source: git://git.subdivi.de/~helmut/crossqa.git</li>
78    </ul>
79   </footer>
80  </body>
81 </html>
82 """
83
84 src_template = """
85 {%- macro render_bug(bugobj) -%}
86  <a href="https://bugs.debian.org/{{ bugobj.bugnum }}">#{{ bugobj.bugnum }}</a>
87  {%- if bugobj.patched %}
88   [<abbr title="patch available">+</abbr>]
89  {%- endif -%}:
90  {% if bugobj.package != "src:" + bugobj.affects and
91        not bugobj.title.startswith(bugobj.affects + ":") -%}
92   {{- bugobj.package|e }}:
93  {% endif -%}
94  {{- bugobj.title|e -}}
95 {%- endmacro -%}
96 <!DOCTYPE html>
97 <html>
98  <head>
99   <title>{{ sourcepackage|e }} - Debian cross build</title>
100   <style>
101 tr.dep.bad td:nth-child(1) {
102     background-color: #faa;
103 }
104 tr.dep.tempbad td:nth-child(1) {
105     background-color: #ffa;
106 }
107 tr.dep.good td:nth-child(1) {
108     background-color: #afa;
109 }
110 tr.build.bad td:nth-child(4) {
111     background-color: #faa;
112 }
113 tr.build.tempbad td:nth-child(4) {
114     background-color: #ffa;
115 }
116 tr.build.good td:nth-child(4) {
117     background-color: #afa;
118 }
119 th {
120     padding-left: 1em;
121     padding-right: 1em;
122 }
123 td {
124     padding-left: 1px;
125     padding-right: 1em;
126 }
127 td:last-child {
128     padding-right: 1px;
129 }
130 footer {
131     margin-top: 3em;
132     border-top: 1px solid;
133 }
134   </style>
135  </head>
136  <body>
137   <header>
138    <h1>
139     <a href="https://tracker.debian.org/pkg/{{ sourcepackage|e }}">
140      {{- sourcepackage|e -}}
141     </a>
142    </h1>
143   </header>
144   {%- if bugs.ftbfs -%}
145    <section>
146     <h3>Reported <abbr title="fails to build from source">FTBFS</abbr> bugs</h3>
147     <ul>
148      {%- for bug in bugs.ftbfs|sort(attribute="bugnum") -%}
149       <li>
150        {{- render_bug(bug) -}}
151       </li>
152      {%- endfor -%}
153     </ul>
154    </section>
155   {%- endif -%}
156   <section>
157    <h3>Cross build dependency satisfiability</h3>
158    {%- if bugs.bdsat -%}
159     <h5>Reported satisfiability problems</h5>
160     <ul>
161      {%- for bug in bugs.bdsat|sort(attribute="bugnum") -%}
162       <li>
163        {{- render_bug(bug) -}}
164       </li>
165      {%- endfor -%}
166     </ul>
167    {%- endif -%}
168    <table>
169     <thead>
170      <tr>
171       <th>state</th>
172       <th>architectures</th>
173      </tr>
174     </thead>
175     <tbody>
176      {%- set okarchs = depresult.pop(None, None) -%}
177      {%- for reason, archs in depresult.items()|sort -%}
178       <tr class="dep {{ "tempbad" if reason.startswith("skew") else "bad" }}">
179        <td>{{ reason|e }}</td>
180        <td>{{ archs|arch_format }}</td>
181       </tr>
182      {%- endfor -%}
183      {%- if okarchs -%}
184       <tr class="dep good">
185        <td>ok</td>
186        <td>{{ okarchs|arch_format }}</td>
187       </tr>
188      {%- endif -%}
189     </tbody>
190    </table>
191    {%- if show_debcheck -%}
192     <h5>See also</h5>
193     <ul>
194      {%- if show_bootstrapdn -%}
195       <li>
196        <a href="https://bootstrap.debian.net/cross_all/{{ sourcepackage|e }}.html">bootstrap.debian.net</a>
197       </li>
198      {%- endif -%}
199      <li>
200       <a href="https://qa.debian.org/dose/debcheck/cross_unstable_main_amd64/latest/packages/{{ sourcepackage|e }}.html">debcheck</a>
201      </li>
202     </ul>
203    {%- endif -%}
204   </section>
205   <section>
206    <h3>Cross builds</h3>
207    {%- if bugs.ftcbfs -%}
208     <h5>Reported cross build failures</h5>
209     <ul>
210      {%- for bug in bugs.ftcbfs|sort(attribute="bugnum") -%}
211       <li>
212        {{- render_bug(bug) -}}
213       </li>
214      {%- endfor -%}
215     </ul>
216    {%- endif -%}
217    {%- if builds -%}
218     <table>
219      <thead>
220       <tr>
221        <th>started</th>
222        <th>version</th>
223        <th>architecture</th>
224        <th>result log</th>
225       </tr>
226      </thead>
227      <tbody>
228       {%- for build in builds|sort(attribute='starttime', reverse=true) -%}
229        <tr class="build {{ "good" if build.success else "bad" }}">
230         <td>
231          {{- build.starttime|sqltimestamp|formatts -}}
232         </td>
233         <td>{{ build.version|e }}</td>
234         <td>{{ build.architecture|e }}</td>
235         <td>
236          <a href="{{ url_for("show_log", filename=build.filename[:-3]) }}">
237           {{- "ok" if build.success else "failed" -}}
238          </a>
239          <a href="{{ url_for("show_log", filename=build.filename) }}">xz</a>
240         </td>
241        </tr>
242       {%- endfor -%}
243      </tbody>
244     </table>
245    {%- else -%}
246     <p>No build performed yet.</p>
247    {%- endif -%}
248    <form method="POST" action="{{ url_for("request_schedule")|e }}">
249     <input type="submit" name="schedule" value="cross build" />
250     {{ sourcepackage|e }} for
251     <input type="hidden" name="source" value="{{ sourcepackage|e }}" />
252     <select name="architecture">
253      <option value="any">any</option>
254      {%- for architecture in architectures|sort -%}
255       <option value="{{ architecture|e }}">{{ architecture|e }}</option>
256      {%- endfor -%}
257     </select>
258    </form>
259   </section>
260   <footer>
261    <h3>Details about this service</h3>
262    <ul>
263     <li>Maintainer: Helmut Grohne &lt;helmut@subdivi.de&gt;</li>
264     <li>Source: git://git.subdivi.de/~helmut/crossqa.git</li>
265    </ul>
266   </footer>
267  </body>
268 </html>
269 """
270
271 schedule_template = """<!DOCTYPE html>
272 <html>
273  <body>
274   <p>Scheduled a build of {{ request.form["source"]|e }}
275    {%- if request.form["architecture"] != "any" %}
276     for {{ request.form["architecture"]|e -}}
277    {%- endif %}.
278   <p>
279  </body>
280 </html>
281 """
282
283 @app.template_filter("sqltimestamp")
284 def sqltimestamp_filter(s):
285     strptime = datetime.datetime.strptime
286     try:
287         return strptime(s, "%Y-%m-%d %H:%M:%S.%f").replace(microsecond=0)
288     except ValueError:
289         return strptime(s, "%Y-%m-%d %H:%M:%S")
290
291
292 def formatts(ts):
293     assert isinstance(ts, datetime.datetime)
294     dt = datetime.datetime.utcnow() - ts
295     if dt < datetime.timedelta(seconds=1):
296         return "now"
297     if dt < datetime.timedelta(seconds=100):
298         return "%d s" % dt.seconds
299     if dt < datetime.timedelta(minutes=100):
300         return "%d m" % (dt.seconds // 60)
301     if dt < datetime.timedelta(days=1):
302         return "%d h" % (dt.seconds // (60 * 60))
303     return "%d d" % dt.days
304
305
306 @app.template_filter("formatts")
307 def formatts_filter(ts):
308     return jinja2.Markup('<time title="%s" datetime="%s">%s</time>' %
309                          (ts, ts, formatts(ts)))
310
311 @app.template_filter('arch_format')
312 @jinja2.contextfilter
313 def arch_format_filter(context, some_archs):
314     if context["architectures"] == some_archs:
315         return "any"
316     return ", ".join(sorted(some_archs))
317
318 def collect_depstate(conn, source):
319     version = None
320     depstate = None
321     query = sqlalchemy.text("""
322         SELECT version, architecture, satisfiable, reason
323             FROM depstate WHERE source = :source;""")
324     for row in conn.execute(query, source=source):
325         if version is None or version_compare(version, row.version) > 0:
326             version = row.version
327             depstate = {}
328         depstate[row.architecture] = None if row.satisfiable else row.reason
329     if version is None:
330         raise werkzeug.exceptions.NotFound()
331     depresult = {}
332     for arch, reason in depstate.items():
333         depresult.setdefault(reason, set()).add(arch)
334     return version, depresult
335
336 @app.route("/")
337 def show_index():
338     with db.engine.connect() as conn:
339         builds = list(conn.execute("""
340             SELECT source, version, architecture, starttime, filename,
341                 ifnull((SELECT max(patched + 1) FROM bugs
342                                                 WHERE affects = source),
343                        0) AS buglvl
344                 FROM builds
345                 WHERE success = 0
346                 ORDER BY starttime
347                 DESC LIMIT 10;"""))
348     return flask.render_template_string(index_template, builds=builds)
349
350 @app.route("/src/<source>")
351 def show_source(source):
352     context = dict(sourcepackage=source)
353     with db.engine.connect() as conn:
354         query = sqlalchemy.text("SELECT architecture FROM depcheck;")
355         context["architectures"] = set(row[0] for row in conn.execute(query))
356         context["version"], context["depresult"] = collect_depstate(conn,
357                                                                     source)
358         query = sqlalchemy.text("""
359             SELECT version, architecture, success, starttime, filename
360                 FROM builds WHERE source = :source;""")
361         context["builds"] = list(conn.execute(query, source=source))
362         query = sqlalchemy.text("""
363             SELECT bugnum, kind, title, package, patched, affects
364                 FROM bugs WHERE affects = :affects;""")
365         context["bugs"] = {}
366         for bug in conn.execute(query, affects=source):
367             context["bugs"].setdefault(bug.kind, []).append(bug)
368         context["show_bootstrapdn"] = \
369                 any(reason and not reason.startswith("skew ")
370                     for reason in context["depresult"].keys())
371         context["show_debcheck"] = \
372                 any(context["depresult"].keys())
373     return flask.render_template_string(src_template, **context)
374
375 @app.route("/build/<path:filename>")
376 def show_log(filename):
377     if filename.endswith(".xz"):
378         return flask.send_from_directory("logs", filename,
379                                          mimetype="application/octet-stream")
380     filename = flask.safe_join("logs", filename + ".xz")
381     try:
382         return flask.send_file(lzma.open(filename, "rb"),
383                                mimetype="text/plain")
384     except FileNotFoundError:
385         raise werkzeug.exceptions.NotFound()
386
387
388 @app.route("/schedule", methods=["POST"])
389 def request_schedule():
390     source = flask.request.form["source"]
391     architecture = flask.request.form["architecture"]
392     with db.engine.connect() as conn:
393         query = sqlalchemy.text("""
394             SELECT 1 FROM depstate WHERE source = :source;""")
395         if not conn.execute(query, source=source).first():
396             raise werkzeug.exceptions.BadRequest()
397         if architecture == "any":
398             architecture = None
399         else:
400             query = sqlalchemy.text("""
401                 SELECT 1 FROM depcheck WHERE architecture = :architecture;""")
402             if not conn.execute(query, architecture=architecture).first():
403                 raise werkzeug.exceptions.BadRequest()
404         query = sqlalchemy.text("""
405             INSERT INTO buildrequests (source, architecture, requesttime)
406                 VALUES (:source, :architecture, datetime('now'));""")
407         conn.execute(query, source=source, architecture=architecture)
408     return flask.render_template_string(schedule_template)