add build architecture to schema of building table
[~helmut/crossqa.git] / build.py
1 #!/usr/bin/python3
2 # SPDX-License-Identifier: GPL-2.0+
3
4 import argparse
5 import collections
6 import contextlib
7 import datetime
8 import lzma
9 import os
10 import os.path
11 import sqlite3
12 import subprocess
13 import time
14
15 from common import decompress_stream, yield_lines, yield_chunks
16
17 def scan_log_status(filelike):
18     it = yield_chunks(filelike)
19     it = decompress_stream(it, lzma.LZMADecompressor())
20     it = yield_lines(it)
21     last_lines = collections.deque(it, 25)
22     status = [l.split(b':', 1)[1].strip()
23               for l in last_lines if l.startswith(b"Status:")]
24     if status:
25         return status[0].decode("ascii")
26     return "unknown"
27
28
29 def do_build(source, version, architecture, server):
30     now = datetime.datetime.utcnow().replace(microsecond=0)
31     logtarget = "%s_%s_%s_%s.log.xz" % (source, version, architecture,
32                                         now.strftime("%Y%m%d%H%M%S"))
33     cmdline = ["ssh", server, "sh", "/dev/stdin", architecture,
34                "%s_%s" % (source, version)]
35     logname = os.path.join("logs", logtarget)
36     with open(logname, "w+b") as output:
37         with open("build.sh", "rb") as instructions:
38             code = subprocess.call(cmdline, stdin=instructions, stdout=output)
39         output.seek(0)
40         status = scan_log_status(output)
41     if code == 255:
42         os.unlink(logname)
43         time.sleep(300)
44         raise RuntimeError("ssh failed")
45     print("status %s code %d" % (status, code))
46     return (now, code == 0, logtarget, status == "given-back")
47
48 def main():
49     argp = argparse.ArgumentParser()
50     argp.add_argument("server", help="machine to build on")
51     args = argp.parse_args()
52     db = sqlite3.connect("db", detect_types=sqlite3.PARSE_DECLTYPES)
53     with contextlib.closing(db.cursor()) as cur:
54         cur.execute("BEGIN IMMEDIATE;")
55         cur.execute("""
56             SELECT d.source, d.version, d.architecture, r.id
57                 FROM depstate AS d
58                     JOIN buildrequests AS r
59                         ON d.architecture = ifnull(r.architecture, d.architecture)
60                             AND d.source = r.source
61                     JOIN depcheck
62                         ON d.architecture = depcheck.architecture
63                 WHERE d.satisfiable = 1 AND depcheck.giveback = 0
64                     AND NOT EXISTS (SELECT 1 FROM building
65                         WHERE d.source = building.source
66                             OR ('amd64' = building.buildarch AND
67                                 d.architecture = building.hostarch)
68                 ORDER BY r.priority DESC, r.requesttime ASC, random()
69                 LIMIT 1;""")
70         row = cur.fetchone()
71         if not row:
72             cur.execute("""
73                 SELECT source, version, depstate.architecture, NULL
74                     FROM depstate JOIN depcheck
75                         ON depstate.architecture = depcheck.architecture
76                     WHERE satisfiable = 1 AND giveback = 0
77                     ORDER BY random() LIMIT 1;""")
78             row = cur.fetchone()
79         if not row:
80             cur.execute("ROLLBACK;")
81             print("no package satisfiable")
82             time.sleep(60)
83             return
84         source, version, architecture, requestid = row
85         cur.execute("""INSERT INTO building (source, buildarch, hostarch, pid)
86                            VALUES (?, ?, ?, ?);""",
87                     (source, 'amd64', architecture, os.getpid()))
88         cur.execute("COMMIT;")
89     try:
90         print("building %s_%s for %s%s" %
91               (source, version, architecture,
92                "" if requestid is None else " (request %d)" % requestid))
93         timestamp, success, filename, giveback = \
94             do_build(source, version, architecture, args.server)
95         with contextlib.closing(db.cursor()) as cur:
96             cur.execute("""INSERT INTO builds
97                             (source, version, buildarch, hostarch, success,
98                              starttime, filename)
99                             VALUES (?, ?, ?, ?, ?, ?);""",
100                         (source, version, 'amd64', architecture, success,
101                          timestamp, filename))
102             if requestid is not None:
103                 cur.execute("DELETE FROM buildrequests WHERE id = ?;",
104                             (requestid,))
105             if giveback:
106                 cur.execute("""UPDATE depcheck SET giveback = 1
107                                 WHERE architecture = ?;""",
108                             (architecture,))
109     finally:
110         with contextlib.closing(db.cursor()) as cur:
111             cur.execute("DELETE FROM building WHERE pid = ?;", (os.getpid(),))
112         db.commit()
113
114 if __name__ == "__main__":
115     main()