webapp.py

   1 #!/usr/bin/python3
   2 # SPDX-License-Identifier: GPL-2.0+
   3
   4 import datetime
   5 import lzma
   6
   7 import apt_pkg
   8 apt_pkg.init()
   9 version_compare = apt_pkg.version_compare
  10
  11 import flask
  12 import flask_sqlalchemy
  13 import jinja2
  14 import sqlalchemy
  15 import werkzeug
  16
  17 app = flask.Flask("crossqa")
  18 app.config["SQLALCHEMY_DATABASE_URI"] = 'sqlite:///db'
  19 app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
  20 db = flask_sqlalchemy.SQLAlchemy(app)
  21
  22 index_template = """<!DOCTYPE html>
  23 <html>
  24  <head>
  25   <title>Debian cross build quality assurance</title>
  26  </head>
  27  <body>
  28   <header>
  29    <h1>Debian cross build quality assurance</h1>
  30   </header>
  31   <section>
  32    <h3>Recently failed builds</h3>
  33    <table>
  34     <thead>
  35      <tr>
  36       <th>source</th>
  37       <th>version</th>
  38       <th>architecture</th>
  39       <th>started</th>
  40       <th>result log</th>
  41      </tr>
  42     </thead>
  43     <tbody>
  44      {%- for build in builds|sort(attribute='starttime', reverse=true) -%}
  45       <tr>
  46        <td>
  47         <a href="{{ url_for("show_source", source=build.source) }}">
  48          {{- build.source|e -}}
  49         </a>
  50        </td>
  51        <td>{{ build.version|e }}</td>
  52        <td>{{ build.architecture|e }}</td>
  53        <td>
  54         {{- build.starttime|sqltimestamp|formatts -}}
  55        </td>
  56        <td>
  57         <a href="{{ url_for("show_log", filename=build.filename[:-3]) }}">log</a>
  58         <a href="{{ url_for("show_log", filename=build.filename) }}">xz</a>
  59        </td>
  60       </tr>
  61      {%- endfor -%}
  62     </tbody>
  63    </table>
  64   </section>
  65   <footer>
  66    <h3>Details about this service</h3>
  67    <ul>
  68     <li>Maintainer: Helmut Grohne &lt;helmut@subdivi.de&gt;</li>
  69     <li>Source: git://git.subdivi.de/~helmut/crossqa.git</li>
  70    </ul>
  71   </footer>
  72  </body>
  73 </html>
  74 """
  75
  76 src_template = """<!DOCTYPE html>
  77 <html>
  78  <head>
  79   <title>{{ sourcepackage|e }} - Debian cross build</title>
  80   <style>
  81 tr.dep.bad td:nth-child(1) {
  82     background-color: #faa;
  83 }
  84 tr.dep.tempbad td:nth-child(1) {
  85     background-color: #ffa;
  86 }
  87 tr.dep.good td:nth-child(1) {
  88     background-color: #afa;
  89 }
  90 tr.build.bad td:nth-child(4) {
  91     background-color: #faa;
  92 }
  93 tr.build.tempbad td:nth-child(4) {
  94     background-color: #ffa;
  95 }
  96 tr.build.good td:nth-child(4) {
  97     background-color: #afa;
  98 }
  99 th {
 100     padding-left: 1em;
 101     padding-right: 1em;
 102 }
 103 td {
 104     padding-left: 1px;
 105     padding-right: 1em;
 106 }
 107 td:last-child {
 108     padding-right: 1px;
 109 }
 110 footer {
 111     margin-top: 3em;
 112     border-top: 1px solid;
 113 }
 114   </style>
 115  </head>
 116  <body>
 117   <header>
 118    <h1>
 119     <a href="https://tracker.debian.org/pkg/{{ sourcepackage|e }}">
 120      {{- sourcepackage|e -}}
 121     </a>
 122    </h1>
 123   </header>
 124   <section>
 125    <h3>Cross build dependency satisfiability</h3>
 126    <table>
 127     <thead>
 128      <tr>
 129       <th>state</th>
 130       <th>architectures</th>
 131      </tr>
 132     </thead>
 133     <tbody>
 134      {%- set okarchs = depresult.pop(None, None) -%}
 135      {%- for reason, archs in depresult.items()|sort -%}
 136       <tr class="dep {{ "tempbad" if reason.startswith("skew") else "bad" }}">
 137        <td>{{ reason|e }}</td>
 138        <td>{{ archs|arch_format }}</td>
 139       </tr>
 140      {%- endfor -%}
 141      {%- if okarchs -%}
 142       <tr class="dep good">
 143        <td>ok</td>
 144        <td>{{ okarchs|arch_format }}</td>
 145       </tr>
 146      {%- endif -%}
 147     </tbody>
 148    </table>
 149    {%- if show_debcheck -%}
 150     <h5>See also</h5>
 151     <ul>
 152      {%- if show_bootstrapdn -%}
 153       <li>
 154        <a href="https://bootstrap.debian.net/cross_all/{{ sourcepackage|e }}.html">bootstrap.debian.net</a>
 155       </li>
 156      {%- endif -%}
 157      <li>
 158       <a href="https://qa.debian.org/dose/debcheck/cross_unstable_main_amd64/latest/packages/{{ sourcepackage|e }}.html">debcheck</a>
 159      </li>
 160     </ul>
 161    {%- endif -%}
 162   </section>
 163   <section>
 164    <h3>Cross builds</h3>
 165    {%- if builds -%}
 166     <table>
 167      <thead>
 168       <tr>
 169        <th>started</th>
 170        <th>version</th>
 171        <th>architecture</th>
 172        <th>result log</th>
 173       </tr>
 174      </thead>
 175      <tbody>
 176       {%- for build in builds|sort(attribute='starttime', reverse=true) -%}
 177        <tr class="build {{ "good" if build.success else "bad" }}">
 178         <td>
 179          {{- build.starttime|sqltimestamp|formatts -}}
 180         </td>
 181         <td>{{ build.version|e }}</td>
 182         <td>{{ build.architecture|e }}</td>
 183         <td>
 184          <a href="{{ url_for("show_log", filename=build.filename[:-3]) }}">
 185           {{- "ok" if build.success else "failed" -}}
 186          </a>
 187          <a href="{{ url_for("show_log", filename=build.filename) }}">xz</a>
 188         </td>
 189        </tr>
 190       {%- endfor -%}
 191      </tbody>
 192     </table>
 193    {%- else -%}
 194     <p>No build performed yet.</p>
 195    {%- endif -%}
 196    <form method="POST" action="{{ url_for("request_schedule")|e }}">
 197     <input type="submit" name="schedule" value="cross build" />
 198     {{ sourcepackage|e }} for
 199     <input type="hidden" name="source" value="{{ sourcepackage|e }}" />
 200     <select name="architecture">
 201      <option value="any">any</option>
 202      {%- for architecture in architectures|sort -%}
 203       <option value="{{ architecture|e }}">{{ architecture|e }}</option>
 204      {%- endfor -%}
 205     </select>
 206    </form>
 207   </section>
 208   <footer>
 209    <h3>Details about this service</h3>
 210    <ul>
 211     <li>Maintainer: Helmut Grohne &lt;helmut@subdivi.de&gt;</li>
 212     <li>Source: git://git.subdivi.de/~helmut/crossqa.git</li>
 213    </ul>
 214   </footer>
 215  </body>
 216 </html>
 217 """
 218
 219 schedule_template = """<!DOCTYPE html>
 220 <html>
 221  <body>
 222   <p>Scheduled a build of {{ request.form["source"]|e }}
 223    {%- if request.form["architecture"] != "any" %}
 224     for {{ request.form["architecture"]|e -}}
 225    {%- endif %}.
 226   <p>
 227  </body>
 228 </html>
 229 """
 230
 231 @app.template_filter("sqltimestamp")
 232 def sqltimestamp_filter(s):
 233     strptime = datetime.datetime.strptime
 234     try:
 235         return strptime(s, "%Y-%m-%d %H:%M:%S.%f").replace(microsecond=0)
 236     except ValueError:
 237         return strptime(s, "%Y-%m-%d %H:%M:%S")
 238
 239
 240 def formatts(ts):
 241     assert isinstance(ts, datetime.datetime)
 242     dt = datetime.datetime.utcnow() - ts
 243     if dt < datetime.timedelta(seconds=1):
 244         return "now"
 245     if dt < datetime.timedelta(seconds=100):
 246         return "%d s" % dt.seconds
 247     if dt < datetime.timedelta(minutes=100):
 248         return "%d m" % (dt.seconds // 60)
 249     if dt < datetime.timedelta(days=1):
 250         return "%d h" % (dt.seconds // (60 * 60))
 251     return "%d d" % dt.days
 252
 253
 254 @app.template_filter("formatts")
 255 def formatts_filter(ts):
 256     return jinja2.Markup('<time title="%s" datetime="%s">%s</time>' %
 257                          (ts, ts, formatts(ts)))
 258
 259 @app.template_filter('arch_format')
 260 @jinja2.contextfilter
 261 def arch_format_filter(context, some_archs):
 262     if context["architectures"] == some_archs:
 263         return "any"
 264     return ", ".join(sorted(some_archs))
 265
 266 def collect_depstate(conn, source):
 267     version = None
 268     depstate = None
 269     query = sqlalchemy.text("""
 270         SELECT version, architecture, satisfiable, reason
 271             FROM depstate WHERE source = :source;""")
 272     for row in conn.execute(query, source=source):
 273         if version is None or version_compare(version, row.version) > 0:
 274             version = row.version
 275             depstate = {}
 276         depstate[row.architecture] = None if row.satisfiable else row.reason
 277     if version is None:
 278         raise werkzeug.exceptions.NotFound()
 279     depresult = {}
 280     for arch, reason in depstate.items():
 281         depresult.setdefault(reason, set()).add(arch)
 282     return version, depresult
 283
 284 @app.route("/")
 285 def show_index():
 286     with db.engine.connect() as conn:
 287         builds = list(conn.execute("""
 288             SELECT source, version, architecture, starttime, filename
 289                 FROM builds
 290                 WHERE success = 0
 291                 ORDER BY starttime
 292                 DESC LIMIT 10;"""))
 293     return flask.render_template_string(index_template, builds=builds)
 294
 295 @app.route("/src/<source>")
 296 def show_source(source):
 297     context = dict(sourcepackage=source)
 298     with db.engine.connect() as conn:
 299         query = sqlalchemy.text("SELECT architecture FROM depcheck;")
 300         context["architectures"] = set(row[0] for row in conn.execute(query))
 301         context["version"], context["depresult"] = collect_depstate(conn,
 302                                                                     source)
 303         query = sqlalchemy.text("""
 304             SELECT version, architecture, success, starttime, filename
 305                 FROM builds WHERE source = :source;""")
 306         context["builds"] = list(conn.execute(query, source=source))
 307         context["show_bootstrapdn"] = \
 308                 any(reason and not reason.startswith("skew ")
 309                     for reason in context["depresult"].keys())
 310         context["show_debcheck"] = \
 311                 any(context["depresult"].keys())
 312     return flask.render_template_string(src_template, **context)
 313
 314 @app.route("/build/<path:filename>")
 315 def show_log(filename):
 316     if filename.endswith(".xz"):
 317         return flask.send_from_directory("logs", filename,
 318                                          mimetype="application/octet-stream")
 319     filename = flask.safe_join("logs", filename + ".xz")
 320     try:
 321         return flask.send_file(lzma.open(filename, "rb"),
 322                                mimetype="text/plain")
 323     except FileNotFoundError:
 324         raise werkzeug.exceptions.NotFound()
 325
 326
 327 @app.route("/schedule", methods=["POST"])
 328 def request_schedule():
 329     source = flask.request.form["source"]
 330     architecture = flask.request.form["architecture"]
 331     with db.engine.connect() as conn:
 332         query = sqlalchemy.text("""
 333             SELECT 1 FROM depstate WHERE source = :source;""")
 334         if not conn.execute(query, source=source).first():
 335             raise werkzeug.exceptions.BadRequest()
 336         if architecture == "any":
 337             architecture = None
 338         else:
 339             query = sqlalchemy.text("""
 340                 SELECT 1 FROM depcheck WHERE architecture = :architecture;""")
 341             if not conn.execute(query, architecture=architecture).first():
 342                 raise werkzeug.exceptions.BadRequest()
 343         query = sqlalchemy.text("""
 344             INSERT INTO buildrequests (source, architecture, requesttime)
 345                 VALUES (:source, :architecture, datetime('now'));""")
 346         conn.execute(query, source=source, architecture=architecture)
 347     return flask.render_template_string(schedule_template)