webapp.py: improve parameter validation for /schedule
[~helmut/crossqa.git] / build.py
1 #!/usr/bin/python3
2 # SPDX-License-Identifier: GPL-2.0+
3
4 import argparse
5 import collections
6 import contextlib
7 import datetime
8 import lzma
9 import os
10 import os.path
11 import sqlite3
12 import subprocess
13 import time
14
15 from common import decompress_stream, yield_lines, yield_chunks
16
17 def scan_log_status(filelike):
18     it = yield_chunks(filelike)
19     it = decompress_stream(it, lzma.LZMADecompressor())
20     it = yield_lines(it)
21     last_lines = collections.deque(it, 25)
22     status = [l.split(b':', 1)[1].strip()
23               for l in last_lines if l.startswith(b"Status:")]
24     if status:
25         return status[0].decode("ascii")
26     return "unknown"
27
28
29 def do_build(source, version, architecture, server):
30     now = datetime.datetime.utcnow().replace(microsecond=0)
31     logtarget = "%s_%s_%s_%s.log.xz" % (source, version, architecture,
32                                         now.strftime("%Y%m%d%H%M%S"))
33     cmdline = ["ssh", server, "sh", "/dev/stdin", architecture,
34                "%s_%s" % (source, version)]
35     logname = os.path.join("logs", logtarget)
36     with open(logname, "w+b") as output:
37         with open("build.sh", "rb") as instructions:
38             code = subprocess.call(cmdline, stdin=instructions, stdout=output)
39         output.seek(0)
40         status = scan_log_status(output)
41     if code == 255:
42         os.unlink(logname)
43         time.sleep(300)
44         raise RuntimeError("ssh failed")
45     print("status %s code %d" % (status, code))
46     return (now, code == 0, logtarget, status == "given-back")
47
48 def get_build_arch():
49     return subprocess.check_output(["dpkg", "--print-architecture"],
50                                    encoding="ascii").strip()
51
52 def main():
53     argp = argparse.ArgumentParser()
54     argp.add_argument("server", help="machine to build on")
55     argp.add_argument("--buildarch", help="build architecture of the server")
56     args = argp.parse_args()
57     buildarch = args.buildarch or get_build_arch()
58     db = sqlite3.connect("db", detect_types=sqlite3.PARSE_DECLTYPES)
59     with contextlib.closing(db.cursor()) as cur:
60         cur.execute("BEGIN IMMEDIATE;")
61         cur.execute("""
62             SELECT d.source, d.version, d.hostarch, r.id
63                 FROM depstate AS d
64                     JOIN buildrequests AS r
65                         ON d.buildarch = ifnull(r.buildarch, d.buildarch)
66                             AND d.hostarch = ifnull(r.hostarch, d.hostarch)
67                             AND d.source = r.source
68                     JOIN depcheck
69                         ON d.buildarch = depcheck.buildarch
70                             AND d.hostarch = depcheck.hostarch
71                 WHERE d.buildarch = ?
72                     AND d.satisfiable = 1
73                     AND depcheck.giveback = 0
74                     AND NOT EXISTS (SELECT 1 FROM building
75                         WHERE d.source = building.source
76                             OR (d.buildarch = building.buildarch AND
77                                 d.hostarch = building.hostarch))
78                 ORDER BY r.priority DESC, r.requesttime ASC, random()
79                 LIMIT 1;""", (buildarch,))
80         row = cur.fetchone()
81         if not row:
82             cur.execute("""
83                 SELECT source, version, depstate.hostarch, NULL
84                     FROM depstate JOIN depcheck
85                         ON depstate.buildarch = depcheck.buildarch
86                             AND depstate.hostarch = depcheck.hostarch
87                     WHERE depstate.buildarch = ? AND satisfiable = 1
88                         AND giveback = 0
89                     ORDER BY random() LIMIT 1;""", (buildarch,))
90             row = cur.fetchone()
91         if not row:
92             cur.execute("ROLLBACK;")
93             print("no package satisfiable")
94             time.sleep(60)
95             return
96         source, version, hostarch, requestid = row
97         cur.execute("""INSERT INTO building (source, buildarch, hostarch, pid)
98                            VALUES (?, ?, ?, ?);""",
99                     (source, buildarch, hostarch, os.getpid()))
100         cur.execute("COMMIT;")
101     try:
102         print("building %s_%s for %s%s" %
103               (source, version, hostarch,
104                "" if requestid is None else " (request %d)" % requestid))
105         timestamp, success, filename, giveback = \
106             do_build(source, version, hostarch, args.server)
107         with contextlib.closing(db.cursor()) as cur:
108             cur.execute("""INSERT INTO builds
109                             (source, version, buildarch, hostarch, success,
110                              starttime, filename)
111                             VALUES (?, ?, ?, ?, ?, ?, ?);""",
112                         (source, version, buildarch, hostarch, success,
113                          timestamp, filename))
114             if requestid is not None:
115                 cur.execute("DELETE FROM buildrequests WHERE id = ?;",
116                             (requestid,))
117             if giveback:
118                 cur.execute("""UPDATE depcheck SET giveback = 1
119                                 WHERE buildarch = ? AND architecture = ?;""",
120                             (buildarch, hostarch,))
121     finally:
122         with contextlib.closing(db.cursor()) as cur:
123             cur.execute("DELETE FROM building WHERE pid = ?;", (os.getpid(),))
124         db.commit()
125
126 if __name__ == "__main__":
127     main()