diff options
author | Helmut Grohne <helmut@subdivi.de> | 2019-03-02 16:10:13 +0100 |
---|---|---|
committer | Helmut Grohne <helmut@subdivi.de> | 2019-03-02 16:10:13 +0100 |
commit | 1e1078f6bdf677bbb557291d76ba012a013f81a7 (patch) | |
tree | 603ef7ffac493412786621995052def174d4cee2 /depcheck.py | |
parent | 4d08683201cc37586ffbb3bd5dcf932851c8a913 (diff) | |
download | crossqa-1e1078f6bdf677bbb557291d76ba012a013f81a7.tar.gz |
depcheck: support parallel operation
Diffstat (limited to 'depcheck.py')
-rwxr-xr-x | depcheck.py | 37 |
1 files changed, 24 insertions, 13 deletions
diff --git a/depcheck.py b/depcheck.py index d2c6bf6..3bbe2d7 100755 --- a/depcheck.py +++ b/depcheck.py @@ -4,9 +4,11 @@ import argparse import collections import contextlib import datetime +import functools import hashlib import itertools import lzma +import multiprocessing import os.path import sqlite3 import subprocess @@ -378,6 +380,7 @@ def check_bdsat(mirror, arch): "--deb-emulate-sbuild", ] + result = {} with tempfile.NamedTemporaryFile("w", encoding="utf8") as bintmp, \ tempfile.NamedTemporaryFile("w", encoding="utf8") as srctmp: for p in make_binary_list(mirror, arch): @@ -393,9 +396,8 @@ def check_bdsat(mirror, arch): dose_result = call_dose_builddebcheck(cmd) next(dose_result) # skip header for d in dose_result: - if d["status"] == "ok": - yield (d["package"], d["version"], None) - else: + reason = None + if d["status"] != "ok": r = d["reasons"][0] if "missing" in r: reason = "missing %s" % r["missing"]["pkg"]["unsat-dependency"].split()[0].split(":", 1)[0] @@ -405,14 +407,11 @@ def check_bdsat(mirror, arch): reason += r.split()[0].split(':', 1)[0] else: assert False - yield (d["package"], d["version"], reason) + result[d["package"]] = (d["version"], reason) + return result -def update_depcheck(mirror, db, architecture): - now = datetime.datetime.utcnow() - mirror.update_release() - state = {} - for source, version, reason in check_bdsat(mirror, architecture): - state[source] = (version, reason) + +def update_depcheck(mirror, db, updatetime, architecture, state): with contextlib.closing(db.cursor()) as cur: cur.execute("BEGIN;") cur.execute("SELECT source, version, satisfiable, reason FROM depstate WHERE architecture = ?;", @@ -429,14 +428,21 @@ def update_depcheck(mirror, db, architecture): reason) for source, (version, reason) in state.items())) cur.execute("UPDATE depcheck SET releasetime = ?, updatetime = ?, giveback = 0 WHERE architecture = ?", - (mirror.releasetime, now, architecture)) + (mirror.releasetime, updatetime, architecture)) db.commit() + +def main_docheck(mirror, architecture): + return (architecture, check_bdsat(mirror, architecture)) + + def main(): argp = argparse.ArgumentParser() argp.add_argument('-m', '--mirror', default='http://deb.debian.org/debian', help="debian mirror to use") + argp.add_argument('-p', '--parallel', action="store_true", + help="enable parallel checking") args = argp.parse_args() mirror = DebianMirror(args.mirror) mirror.update_release() @@ -445,9 +451,14 @@ def main(): cur.execute("""SELECT architecture FROM depcheck WHERE giveback = 1 OR releasetime < ?;""", (mirror.releasetime,)) - for architecture, in list(cur.fetchall()): + archs = set(row[0] for row in cur.fetchall()) + print("checking %s" % " ".join(sorted(archs))) + now = datetime.datetime.utcnow() + mapper = multiprocessing.Pool().imap_unordered if args.parallel else map + for architecture, state in mapper(functools.partial(main_docheck, mirror), + archs): print("update %s" % architecture) - update_depcheck(mirror, db, architecture) + update_depcheck(mirror, db, now, architecture, state) if __name__ == "__main__": main() |