simplify common functions
[~helmut/crossqa.git] / webapp.py
1 #!/usr/bin/python3
2 # SPDX-License-Identifier: GPL-2.0+
3
4 import datetime
5 import lzma
6
7 import apt_pkg
8 apt_pkg.init()
9 version_compare = apt_pkg.version_compare
10
11 import flask
12 import flask_sqlalchemy
13 import jinja2
14 import sqlalchemy
15 import werkzeug
16
17 app = flask.Flask("crossqa")
18 app.config["SQLALCHEMY_DATABASE_URI"] = 'sqlite:///db'
19 app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
20 db = flask_sqlalchemy.SQLAlchemy(app)
21
22 index_template = """<!DOCTYPE html>
23 <html>
24  <head>
25   <title>Debian cross build quality assurance</title>
26  </head>
27  <body>
28   <header>
29    <h1>Debian cross build quality assurance</h1>
30   </header>
31   <section>
32    <h3>Recently failed builds</h3>
33    <table>
34     <thead>
35      <tr>
36       <th>source</th>
37       <th>version</th>
38       <th>build architecture</th>
39       <th>host architecture</th>
40       <th>started</th>
41       <th>result log</th>
42       <th>bugs</th>
43      </tr>
44     </thead>
45     <tbody>
46      {%- for build in builds|sort(attribute='starttime', reverse=true) -%}
47       <tr>
48        <td>
49         <a href="{{ url_for("show_source", source=build.source) }}">
50          {{- build.source|e -}}
51         </a>
52        </td>
53        <td>{{ build.version|e }}</td>
54        <td>{{ build.buildarch|e }}</td>
55        <td>{{ build.hostarch|e }}</td>
56        <td>
57         {{- build.starttime|sqltimestamp|formatts -}}
58        </td>
59        <td>
60         <a href="{{ url_for("show_log", filename=build.filename[:-3]) }}">log</a>
61         <a href="{{ url_for("show_log", filename=build.filename) }}">xz</a>
62        </td>
63        <td>
64         {%- if build.buglvl == 2 -%}
65          patch reported
66         {%- elif build.buglvl == 1 -%}
67          bug reported
68         {%- endif -%}
69        </td>
70       </tr>
71      {%- endfor -%}
72     </tbody>
73    </table>
74   </section>
75   <footer>
76    <h3>Details about this service</h3>
77    <ul>
78     <li>Maintainer: Helmut Grohne &lt;helmut@subdivi.de&gt;</li>
79     <li>Source: git://git.subdivi.de/~helmut/crossqa.git</li>
80    </ul>
81   </footer>
82  </body>
83 </html>
84 """
85
86 src_template = """
87 {%- macro render_bug(bugobj) -%}
88  <a href="https://bugs.debian.org/{{ bugobj.bugnum }}">#{{ bugobj.bugnum }}</a>
89  {%- if bugobj.patched %}
90   [<abbr title="patch available">+</abbr>]
91  {%- endif -%}:
92  {% if bugobj.package != "src:" + bugobj.affects and
93        not bugobj.title.startswith(bugobj.affects + ":") -%}
94   {{- bugobj.package|e }}:
95  {% endif -%}
96  {{- bugobj.title|e -}}
97 {%- endmacro -%}
98 <!DOCTYPE html>
99 <html>
100  <head>
101   <title>{{ sourcepackage|e }} - Debian cross build</title>
102   <style>
103 tr.dep.bad td:nth-child(1) {
104     background-color: #faa;
105 }
106 tr.dep.tempbad td:nth-child(1) {
107     background-color: #ffa;
108 }
109 tr.dep.good td:nth-child(1) {
110     background-color: #afa;
111 }
112 tr.build.bad td:nth-child(5) {
113     background-color: #faa;
114 }
115 tr.build.tempbad td:nth-child(5) {
116     background-color: #ffa;
117 }
118 tr.build.good td:nth-child(5) {
119     background-color: #afa;
120 }
121 th {
122     padding-left: 1em;
123     padding-right: 1em;
124 }
125 td {
126     padding-left: 1px;
127     padding-right: 1em;
128 }
129 td:last-child {
130     padding-right: 1px;
131 }
132 footer {
133     margin-top: 3em;
134     border-top: 1px solid;
135 }
136   </style>
137  </head>
138  <body>
139   <header>
140    <h1>
141     <a href="https://tracker.debian.org/pkg/{{ sourcepackage|e }}">
142      {{- sourcepackage|e -}}
143     </a>
144    </h1>
145   </header>
146   {%- if bugs.ftbfs -%}
147    <section>
148     <h3>Reported <abbr title="fails to build from source">FTBFS</abbr> bugs</h3>
149     <ul>
150      {%- for bug in bugs.ftbfs|sort(attribute="bugnum") -%}
151       <li>
152        {{- render_bug(bug) -}}
153       </li>
154      {%- endfor -%}
155     </ul>
156    </section>
157   {%- endif -%}
158   <section>
159    <h3>Cross build dependency satisfiability</h3>
160    {%- if bugs.bdsat -%}
161     <h5>Reported satisfiability problems</h5>
162     <ul>
163      {%- for bug in bugs.bdsat|sort(attribute="bugnum") -%}
164       <li>
165        {{- render_bug(bug) -}}
166       </li>
167      {%- endfor -%}
168     </ul>
169    {%- endif -%}
170    <table>
171     <thead>
172      <tr>
173       <th>state</th>
174       <th>architectures</th>
175      </tr>
176     </thead>
177     <tbody>
178      {%- set okarchs = depresult.pop(None, None) -%}
179      {%- for reason, archs in depresult.items()|sort -%}
180       <tr class="dep {{ "tempbad" if reason.startswith("skew") else "bad" }}">
181        <td>{{ reason|e }}</td>
182        <td>{{ archs|archpairs_format }}</td>
183       </tr>
184      {%- endfor -%}
185      {%- if okarchs -%}
186       <tr class="dep good">
187        <td>ok</td>
188        <td>{{ okarchs|archpairs_format }}</td>
189       </tr>
190      {%- endif -%}
191     </tbody>
192    </table>
193    {%- if show_debcheck -%}
194     <h5>See also</h5>
195     <ul>
196      {%- if show_bootstrapdn -%}
197       <li>
198        <a href="https://bootstrap.debian.net/cross_all/{{ sourcepackage|e }}.html">bootstrap.debian.net</a>
199       </li>
200      {%- endif -%}
201      <li>
202       <a href="https://qa.debian.org/dose/debcheck/cross_unstable_main_amd64/latest/packages/{{ sourcepackage|e }}.html">debcheck</a>
203      </li>
204     </ul>
205    {%- endif -%}
206   </section>
207   <section>
208    <h3>Cross builds</h3>
209    {%- if bugs.ftcbfs -%}
210     <h5>Reported cross build failures</h5>
211     <ul>
212      {%- for bug in bugs.ftcbfs|sort(attribute="bugnum") -%}
213       <li>
214        {{- render_bug(bug) -}}
215       </li>
216      {%- endfor -%}
217     </ul>
218    {%- endif -%}
219    {%- if builds -%}
220     <table>
221      <thead>
222       <tr>
223        <th>started</th>
224        <th>version</th>
225        <th>build architecture</th>
226        <th>host architecture</th>
227        <th>result log</th>
228       </tr>
229      </thead>
230      <tbody>
231       {%- for build in builds|sort(attribute='starttime', reverse=true) -%}
232        <tr class="build {{ "good" if build.success else "bad" }}">
233         <td>
234          {{- build.starttime|sqltimestamp|formatts -}}
235         </td>
236         <td>{{ build.version|e }}</td>
237         <td>{{ build.buildarch|e }}</td>
238         <td>{{ build.hostarch|e }}</td>
239         <td>
240          <a href="{{ url_for("show_log", filename=build.filename[:-3]) }}">
241           {{- "ok" if build.success else "failed" -}}
242          </a>
243          <a href="{{ url_for("show_log", filename=build.filename) }}">xz</a>
244         </td>
245        </tr>
246       {%- endfor -%}
247      </tbody>
248     </table>
249    {%- else -%}
250     <p>No build performed yet.</p>
251    {%- endif -%}
252    <form method="POST" action="{{ url_for("request_schedule")|e }}">
253     <input type="submit" name="schedule" value="cross build" />
254     {{ sourcepackage|e }} for
255     <input type="hidden" name="source" value="{{ sourcepackage|e }}" />
256     <select name="archpair">
257      <option value="any_any">{{ ("any", "any")|archpair_format }}</option>
258      {%- for buildarch, hostarch in architectures|sort -%}
259       <option value="{{ buildarch|e }}_{{ hostarch|e }}">
260        {{- (buildarch, hostarch)|archpair_format -}}
261       </option>
262      {%- endfor -%}
263     </select>
264    </form>
265   </section>
266   <footer>
267    <h3>Details about this service</h3>
268    <ul>
269     <li>Maintainer: Helmut Grohne &lt;helmut@subdivi.de&gt;</li>
270     <li>Source: git://git.subdivi.de/~helmut/crossqa.git</li>
271    </ul>
272   </footer>
273  </body>
274 </html>
275 """
276
277 schedule_template = """<!DOCTYPE html>
278 <html>
279  <body>
280   <p>Scheduled a build of {{ request.form["source"]|e }}
281    {%- if buildarch or hostarch %}
282     for {{ (buildarch|default("any"), hostarch|default("any"))|archpair_format -}}
283    {%- endif %}.
284   <p>
285  </body>
286 </html>
287 """
288
289 @app.template_filter("sqltimestamp")
290 def sqltimestamp_filter(s):
291     strptime = datetime.datetime.strptime
292     try:
293         return strptime(s, "%Y-%m-%d %H:%M:%S.%f").replace(microsecond=0)
294     except ValueError:
295         return strptime(s, "%Y-%m-%d %H:%M:%S")
296
297
298 def formatts(ts):
299     assert isinstance(ts, datetime.datetime)
300     dt = datetime.datetime.utcnow() - ts
301     if dt < datetime.timedelta(seconds=1):
302         return "now"
303     if dt < datetime.timedelta(seconds=100):
304         return "%d s" % dt.seconds
305     if dt < datetime.timedelta(minutes=100):
306         return "%d m" % (dt.seconds // 60)
307     if dt < datetime.timedelta(days=1):
308         return "%d h" % (dt.seconds // (60 * 60))
309     return "%d d" % dt.days
310
311
312 @app.template_filter("formatts")
313 def formatts_filter(ts):
314     return jinja2.Markup('<time title="%s" datetime="%s">%s</time>' %
315                          (ts, ts, formatts(ts)))
316
317 @app.template_filter("archpair_format")
318 def archpair_format_filter(archpair):
319     return jinja2.Markup("%s &rarr; %s" % tuple(map(jinja2.escape, archpair)))
320
321 def group_pairs(pairs):
322     result = {}
323     for v, w in pairs:
324         result.setdefault(v, set()).add(w)
325     return result
326
327 def render_archset(subset, all_archs):
328     if len(subset) == 1:
329         return next(iter(subset))
330     if subset == all_archs:
331         return "any"
332     return "{%s}" % ", ".join(map(jinja2.escape, sorted(subset)))
333
334 @app.template_filter('archpairs_format')
335 @jinja2.contextfilter
336 def archpairs_format_filter(context, some_archs):
337     architectures = group_pairs(context["architectures"])
338     fwdmap = {}  # build architecture -> host architecture set representation
339     for buildarch, hostarchs in group_pairs(some_archs).items():
340         fwdmap[buildarch] = render_archset(hostarchs, architectures[buildarch])
341     allbuildarchs = set(architectures.keys())
342     # host architecture set representation -> build architecture set
343     flippedit = group_pairs((v, k) for (k, v) in fwdmap.items()).items()
344     maps = ("%s &rarr; %s" % (render_archset(buildarchs, allbuildarchs),
345                               hostarchrep)
346             for hostarchrep, buildarchs in flippedit)
347     return jinja2.Markup("; ".join(sorted(maps)))
348
349 def collect_depstate(conn, source):
350     version = None
351     depstate = None
352     query = sqlalchemy.text("""
353         SELECT version, buildarch, hostarch, satisfiable, reason
354             FROM depstate WHERE source = :source;""")
355     for row in conn.execute(query, source=source):
356         if version is None or version_compare(version, row.version) > 0:
357             version = row.version
358             depstate = {}
359         depstate[row.buildarch, row.hostarch] = \
360                 None if row.satisfiable else row.reason
361     if version is None:
362         raise werkzeug.exceptions.NotFound()
363     depresult = {}
364     for archpair, reason in depstate.items():
365         depresult.setdefault(reason, set()).add(archpair)
366     return version, depresult
367
368 @app.route("/")
369 def show_index():
370     with db.engine.connect() as conn:
371         builds = list(conn.execute("""
372             SELECT source, version, buildarch, hostarch, starttime, filename,
373                 ifnull((SELECT max(patched + 1) FROM bugs
374                                                 WHERE affects = source),
375                        0) AS buglvl
376                 FROM builds
377                 WHERE success = 0
378                 ORDER BY starttime
379                 DESC LIMIT 10;"""))
380     return flask.render_template_string(index_template, builds=builds)
381
382 @app.route("/src/<source>")
383 def show_source(source):
384     context = dict(sourcepackage=source)
385     with db.engine.connect() as conn:
386         query = sqlalchemy.text("SELECT buildarch, hostarch FROM depcheck;")
387         context["architectures"] = set(map(tuple, conn.execute(query)))
388         context["version"], context["depresult"] = collect_depstate(conn,
389                                                                     source)
390         query = sqlalchemy.text("""
391             SELECT version, buildarch, hostarch, success, starttime, filename
392                 FROM builds WHERE source = :source;""")
393         context["builds"] = list(conn.execute(query, source=source))
394         query = sqlalchemy.text("""
395             SELECT bugnum, kind, title, package, patched, affects
396                 FROM bugs WHERE affects = :affects;""")
397         context["bugs"] = {}
398         for bug in conn.execute(query, affects=source):
399             context["bugs"].setdefault(bug.kind, []).append(bug)
400         context["show_bootstrapdn"] = \
401                 any(reason and not reason.startswith("skew ")
402                     for reason in context["depresult"].keys())
403         context["show_debcheck"] = \
404                 any(context["depresult"].keys())
405     return flask.render_template_string(src_template, **context)
406
407 @app.route("/build/<path:filename>")
408 def show_log(filename):
409     if filename.endswith(".xz"):
410         return flask.send_from_directory("logs", filename,
411                                          mimetype="application/octet-stream")
412     filename = flask.safe_join("logs", filename + ".xz")
413     try:
414         return flask.send_file(lzma.open(filename, "rb"),
415                                mimetype="text/plain")
416     except FileNotFoundError:
417         raise werkzeug.exceptions.NotFound()
418
419
420 @app.route("/schedule", methods=["POST"])
421 def request_schedule():
422     source = flask.request.form["source"]
423     try:
424         buildarch, hostarch = flask.request.form["archpair"].split("_")
425     except ValueError:
426         raise werkzeug.exceptions.BadRequest()
427     if buildarch == "any":
428         buildarch = None
429     if hostarch == "any":
430         hostarch = None
431     with db.engine.connect() as conn:
432         query = sqlalchemy.text("""
433             SELECT 1 FROM depstate WHERE source = :source;""")
434         if not conn.execute(query, source=source).first():
435             raise werkzeug.exceptions.BadRequest()
436         query = sqlalchemy.text("""
437             SELECT 1 FROM depcheck
438                 WHERE buildarch = ifnull(:buildarch, buildarch)
439                     AND hostarch = ifnull(:hostarch, hostarch);""")
440         if not conn.execute(query, buildarch=buildarch,
441                             hostarch=hostarch).first():
442             raise werkzeug.exceptions.BadRequest()
443         query = sqlalchemy.text("""
444             INSERT INTO buildrequests (source, buildarch, hostarch,
445                                        requesttime)
446                 VALUES (:source, :buildarch, :hostarch, datetime('now'));""")
447         conn.execute(query, source=source, buildarch=buildarch,
448                      hostarch=hostarch)
449     return flask.render_template_string(schedule_template, buildarch=buildarch,
450                                         hostarch=hostarch)