#!/usr/bin/python3 # SPDX-License-Identifier: GPL-2.0+ import argparse import collections import contextlib import datetime import lzma import os import os.path import sqlite3 import subprocess import time from common import decompress_stream, yield_lines, yield_chunks def scan_log_status(filelike): it = yield_chunks(filelike) it = decompress_stream(it, lzma.LZMADecompressor()) it = yield_lines(it) last_lines = collections.deque(it, 25) status = [l.split(b':', 1)[1].strip() for l in last_lines if l.startswith(b"Status:")] if status: return status[0].decode("ascii") return "unknown" def do_build(source, version, architecture, server): now = datetime.datetime.utcnow().replace(microsecond=0) logtarget = "%s_%s_%s_%s.log.xz" % (source, version, architecture, now.strftime("%Y%m%d%H%M%S")) cmdline = ["ssh", server, "sh", "/dev/stdin", architecture, "%s_%s" % (source, version)] logname = os.path.join("logs", logtarget) with open(logname, "w+b") as output: with open("build.sh", "rb") as instructions: code = subprocess.call(cmdline, stdin=instructions, stdout=output) output.seek(0) status = scan_log_status(output) if code == 255: os.unlink(logname) time.sleep(300) raise RuntimeError("ssh failed") print("status %s code %d" % (status, code)) return (now, code == 0, logtarget, status == "given-back") def get_native_arch() -> str: return subprocess.check_output(["dpkg", "--print-architecture"], encoding="ascii").strip() def main(): argp = argparse.ArgumentParser() argp.add_argument("server", help="machine to build on") argp.add_argument("--buildarch", help="build architecture of the server") args = argp.parse_args() buildarch = args.buildarch or get_native_arch() db = sqlite3.connect("db", detect_types=sqlite3.PARSE_DECLTYPES) with contextlib.closing(db.cursor()) as cur: cur.execute("BEGIN IMMEDIATE;") cur.execute(""" SELECT d.source, d.version, d.hostarch, r.id FROM depstate AS d JOIN buildrequests AS r ON d.buildarch = ifnull(r.buildarch, d.buildarch) AND d.hostarch = ifnull(r.hostarch, d.hostarch) AND d.source = r.source JOIN depcheck ON d.buildarch = depcheck.buildarch AND d.hostarch = depcheck.hostarch WHERE d.buildarch = ? AND d.satisfiable = 1 AND depcheck.giveback = 0 AND NOT EXISTS (SELECT 1 FROM building WHERE d.source = building.source OR (d.buildarch = building.buildarch AND d.hostarch = building.hostarch)) ORDER BY r.priority DESC, r.requesttime ASC, random() LIMIT 1;""", (buildarch,)) row = cur.fetchone() if not row: cur.execute(""" SELECT source, version, depstate.hostarch, NULL FROM depstate JOIN depcheck ON depstate.buildarch = depcheck.buildarch AND depstate.hostarch = depcheck.hostarch WHERE depstate.buildarch = ? AND satisfiable = 1 AND giveback = 0 ORDER BY random() LIMIT 1;""", (buildarch,)) row = cur.fetchone() if not row: cur.execute("ROLLBACK;") print("no package satisfiable") time.sleep(60) return source, version, hostarch, requestid = row cur.execute("""INSERT INTO building (source, buildarch, hostarch, pid) VALUES (?, ?, ?, ?);""", (source, buildarch, hostarch, os.getpid())) cur.execute("COMMIT;") try: print("building %s_%s for %s%s" % (source, version, hostarch, "" if requestid is None else " (request %d)" % requestid)) timestamp, success, filename, giveback = \ do_build(source, version, hostarch, args.server) with contextlib.closing(db.cursor()) as cur: cur.execute("""INSERT INTO builds (source, version, buildarch, hostarch, success, starttime, filename) VALUES (?, ?, ?, ?, ?, ?, ?);""", (source, version, buildarch, hostarch, success, timestamp, filename)) if requestid is not None: cur.execute("DELETE FROM buildrequests WHERE id = ?;", (requestid,)) if giveback: cur.execute("""UPDATE depcheck SET giveback = 1 WHERE buildarch = ? AND hostarch = ?;""", (buildarch, hostarch,)) finally: with contextlib.closing(db.cursor()) as cur: cur.execute("DELETE FROM building WHERE pid = ?;", (os.getpid(),)) db.commit() if __name__ == "__main__": main()