52defa3d66c0e516160ec577ca9bb0613ec63995
[~helmut/crossqa.git] / webapp.py
1 #!/usr/bin/python3
2 # SPDX-License-Identifier: GPL-2.0+
3
4 import datetime
5 import lzma
6
7 import apt_pkg
8 apt_pkg.init()
9 version_compare = apt_pkg.version_compare
10
11 import flask
12 import flask_sqlalchemy
13 import jinja2
14 import sqlalchemy
15 import werkzeug
16
17 app = flask.Flask("crossqa")
18 app.config["SQLALCHEMY_DATABASE_URI"] = 'sqlite:///db'
19 app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
20 db = flask_sqlalchemy.SQLAlchemy(app)
21
22 index_template = """<!DOCTYPE html>
23 <html>
24  <head>
25   <title>Debian cross build quality assurance</title>
26  </head>
27  <body>
28   <header>
29    <h1>Debian cross build quality assurance</h1>
30   </header>
31   <section>
32    <h3>Recently failed builds</h3>
33    <table>
34     <thead>
35      <tr>
36       <th>source</th>
37       <th>version</th>
38       <th>architecture</th>
39       <th>started</th>
40       <th>result log</th>
41      </tr>
42     </thead>
43     <tbody>
44      {%- for build in builds|sort(attribute='starttime', reverse=true) -%}
45       <tr>
46        <td>
47         <a href="{{ url_for("show_source", source=build.source) }}">
48          {{- build.source|e -}}
49         </a>
50        </td>
51        <td>{{ build.version|e }}</td>
52        <td>{{ build.architecture|e }}</td>
53        <td>
54         {{- build.starttime|sqltimestamp|formatts -}}
55        </td>
56        <td>
57         <a href="{{ url_for("show_log", filename=build.filename[:-3]) }}">log</a>
58         <a href="{{ url_for("show_log", filename=build.filename) }}">xz</a>
59        </td>
60       </tr>
61      {%- endfor -%}
62     </tbody>
63    </table>
64   </section>
65   <footer>
66    <h3>Details about this service</h3>
67    <ul>
68     <li>Maintainer: Helmut Grohne &lt;helmut@subdivi.de&gt;</li>
69     <li>Source: git://git.subdivi.de/~helmut/crossqa.git</li>
70    </ul>
71   </footer>
72  </body>
73 </html>
74 """
75
76 src_template = """<!DOCTYPE html>
77 <html>
78  <head>
79   <title>{{ sourcepackage|e }} - Debian cross build</title>
80   <style>
81 tr.dep.bad td:nth-child(1) {
82     background-color: #faa;
83 }
84 tr.dep.tempbad td:nth-child(1) {
85     background-color: #ffa;
86 }
87 tr.dep.good td:nth-child(1) {
88     background-color: #afa;
89 }
90 tr.build.bad td:nth-child(4) {
91     background-color: #faa;
92 }
93 tr.build.tempbad td:nth-child(4) {
94     background-color: #ffa;
95 }
96 tr.build.good td:nth-child(4) {
97     background-color: #afa;
98 }
99 th {
100     padding-left: 1em;
101     padding-right: 1em;
102 }
103 td {
104     padding-left: 1px;
105     padding-right: 1em;
106 }
107 td:last-child {
108     padding-right: 1px;
109 }
110 footer {
111     margin-top: 3em;
112     border-top: 1px solid;
113 }
114   </style>
115  </head>
116  <body>
117   <header>
118    <h1>
119     <a href="https://tracker.debian.org/pkg/{{ sourcepackage|e }}">
120      {{- sourcepackage|e -}}
121     </a>
122    </h1>
123   </header>
124   <section>
125    <h3>Cross build dependency satisfiability</h3>
126    <table>
127     <thead>
128      <tr>
129       <th>state</th>
130       <th>architectures</th>
131      </tr>
132     </thead>
133     <tbody>
134      {%- set okarchs = depresult.pop(None, None) -%}
135      {%- for reason, archs in depresult.items()|sort -%}
136       <tr class="dep {{ "tempbad" if reason.startswith("skew") else "bad" }}">
137        <td>{{ reason|e }}</td>
138        <td>{{ archs|arch_format }}</td>
139       </tr>
140      {%- endfor -%}
141      {%- if okarchs -%}
142       <tr class="dep good">
143        <td>ok</td>
144        <td>{{ okarchs|arch_format }}</td>
145       </tr>
146      {%- endif -%}
147     </tbody>
148    </table>
149    <h5>See also</h5>
150    <ul>
151     {%- if show_bootstrapdn -%}
152      <li>
153       <a href="https://bootstrap.debian.net/cross_all/{{ sourcepackage|e }}.html">bootstrap.debian.net</a>
154      </li>
155     {%- endif -%}
156     <li>
157      <a href="https://qa.debian.org/dose/debcheck/cross_unstable_main_amd64/">debcheck</a>
158     </li>
159    </ul>
160   </section>
161   <section>
162    <h3>Cross builds</h3>
163    {%- if builds -%}
164     <table>
165      <thead>
166       <tr>
167        <th>started</th>
168        <th>version</th>
169        <th>architecture</th>
170        <th>result log</th>
171       </tr>
172      </thead>
173      <tbody>
174       {%- for build in builds|sort(attribute='starttime', reverse=true) -%}
175        <tr class="build {{ "good" if build.success else "bad" }}">
176         <td>
177          {{- build.starttime|sqltimestamp|formatts -}}
178         </td>
179         <td>{{ build.version|e }}</td>
180         <td>{{ build.architecture|e }}</td>
181         <td>
182          <a href="{{ url_for("show_log", filename=build.filename[:-3]) }}">
183           {{- "ok" if build.success else "failed" -}}
184          </a>
185          <a href="{{ url_for("show_log", filename=build.filename) }}">xz</a>
186         </td>
187        </tr>
188       {%- endfor -%}
189      </tbody>
190     </table>
191    {%- else -%}
192     <p>No build performed yet.</p>
193    {%- endif -%}
194    <form method="POST" action="{{ url_for("request_schedule")|e }}">
195     <input type="submit" name="schedule" value="cross build" />
196     {{ sourcepackage|e }} for
197     <input type="hidden" name="source" value="{{ sourcepackage|e }}" />
198     <select name="architecture">
199      <option value="any">any</option>
200      {%- for architecture in architectures|sort -%}
201       <option value="{{ architecture|e }}">{{ architecture|e }}</option>
202      {%- endfor -%}
203     </select>
204    </form>
205   </section>
206   <footer>
207    <h3>Details about this service</h3>
208    <ul>
209     <li>Maintainer: Helmut Grohne &lt;helmut@subdivi.de&gt;</li>
210     <li>Source: git://git.subdivi.de/~helmut/crossqa.git</li>
211    </ul>
212   </footer>
213  </body>
214 </html>
215 """
216
217 schedule_template = """<!DOCTYPE html>
218 <html>
219  <body>
220   <p>Scheduled a build of {{ request.form["source"]|e }}
221    {%- if request.form["architecture"] != "any" %}
222     for {{ request.form["architecture"]|e -}}
223    {%- endif %}.
224   <p>
225  </body>
226 </html>
227 """
228
229 @app.template_filter("sqltimestamp")
230 def sqltimestamp_filter(s):
231     strptime = datetime.datetime.strptime
232     try:
233         return strptime(s, "%Y-%m-%d %H:%M:%S.%f").replace(microsecond=0)
234     except ValueError:
235         return strptime(s, "%Y-%m-%d %H:%M:%S")
236
237
238 def formatts(ts):
239     assert isinstance(ts, datetime.datetime)
240     dt = datetime.datetime.utcnow() - ts
241     if dt < datetime.timedelta(seconds=1):
242         return "now"
243     if dt < datetime.timedelta(seconds=100):
244         return "%d s" % dt.seconds
245     if dt < datetime.timedelta(minutes=100):
246         return "%d m" % (dt.seconds // 60)
247     if dt < datetime.timedelta(days=1):
248         return "%d h" % (dt.seconds // (60 * 60))
249     return "%d d" % dt.days
250
251
252 @app.template_filter("formatts")
253 def formatts_filter(ts):
254     return jinja2.Markup('<time title="%s" datetime="%s">%s</time>' %
255                          (ts, ts, formatts(ts)))
256
257 @app.template_filter('arch_format')
258 @jinja2.contextfilter
259 def arch_format_filter(context, some_archs):
260     if context["architectures"] == some_archs:
261         return "any"
262     return ", ".join(sorted(some_archs))
263
264 def collect_depstate(conn, source):
265     version = None
266     depstate = None
267     query = sqlalchemy.text("""
268         SELECT version, architecture, satisfiable, reason
269             FROM depstate WHERE source = :source;""")
270     for row in conn.execute(query, source=source):
271         if version is None or version_compare(version, row.version) > 0:
272             version = row.version
273             depstate = {}
274         depstate[row.architecture] = None if row.satisfiable else row.reason
275     if version is None:
276         raise werkzeug.exceptions.NotFound()
277     depresult = {}
278     for arch, reason in depstate.items():
279         depresult.setdefault(reason, set()).add(arch)
280     return version, depresult
281
282 @app.route("/")
283 def show_index():
284     with db.engine.connect() as conn:
285         builds = list(conn.execute("""
286             SELECT source, version, architecture, starttime, filename
287                 FROM builds
288                 WHERE success = 0
289                 ORDER BY starttime
290                 DESC LIMIT 10;"""))
291     return flask.render_template_string(index_template, builds=builds)
292
293 @app.route("/src/<source>")
294 def show_source(source):
295     context = dict(sourcepackage=source)
296     with db.engine.connect() as conn:
297         query = sqlalchemy.text("SELECT architecture FROM depcheck;")
298         context["architectures"] = set(row[0] for row in conn.execute(query))
299         context["version"], context["depresult"] = collect_depstate(conn,
300                                                                     source)
301         query = sqlalchemy.text("""
302             SELECT version, architecture, success, starttime, filename
303                 FROM builds WHERE source = :source;""")
304         context["builds"] = list(conn.execute(query, source=source))
305         context["show_bootstrapdn"] = \
306                 any(reason and not reason.startswith("skew ")
307                     for reason in context["depresult"].keys())
308     return flask.render_template_string(src_template, **context)
309
310 @app.route("/build/<path:filename>")
311 def show_log(filename):
312     if filename.endswith(".xz"):
313         return flask.send_from_directory("logs", filename,
314                                          mimetype="application/octet-stream")
315     filename = flask.safe_join("logs", filename + ".xz")
316     try:
317         return flask.send_file(lzma.open(filename, "rb"),
318                                mimetype="text/plain")
319     except FileNotFoundError:
320         raise werkzeug.exceptions.NotFound()
321
322
323 @app.route("/schedule", methods=["POST"])
324 def request_schedule():
325     source = flask.request.form["source"]
326     architecture = flask.request.form["architecture"]
327     with db.engine.connect() as conn:
328         query = sqlalchemy.text("""
329             SELECT 1 FROM depstate WHERE source = :source;""")
330         if not conn.execute(query, source=source).first():
331             raise werkzeug.exceptions.BadRequest()
332         if architecture == "any":
333             architecture = None
334         else:
335             query = sqlalchemy.text("""
336                 SELECT 1 FROM depcheck WHERE architecture = :architecture;""")
337             if not conn.execute(query, architecture=architecture).first():
338                 raise werkzeug.exceptions.BadRequest()
339         query = sqlalchemy.text("""
340             INSERT INTO buildrequests (source, architecture, requesttime)
341                 VALUES (:source, :architecture, datetime('now'));""")
342         conn.execute(query, source=source, architecture=architecture)
343     return flask.render_template_string(schedule_template)