depcheck: support parallel operation
authorHelmut Grohne <helmut@subdivi.de>
Sat, 2 Mar 2019 15:10:13 +0000 (16:10 +0100)
committerHelmut Grohne <helmut@subdivi.de>
Sat, 2 Mar 2019 15:10:13 +0000 (16:10 +0100)
depcheck.py

index d2c6bf6..3bbe2d7 100755 (executable)
@@ -4,9 +4,11 @@ import argparse
 import collections
 import contextlib
 import datetime
+import functools
 import hashlib
 import itertools
 import lzma
+import multiprocessing
 import os.path
 import sqlite3
 import subprocess
@@ -378,6 +380,7 @@ def check_bdsat(mirror, arch):
         "--deb-emulate-sbuild",
     ]
 
+    result = {}
     with tempfile.NamedTemporaryFile("w", encoding="utf8") as bintmp, \
             tempfile.NamedTemporaryFile("w", encoding="utf8") as srctmp:
         for p in make_binary_list(mirror, arch):
@@ -393,9 +396,8 @@ def check_bdsat(mirror, arch):
         dose_result = call_dose_builddebcheck(cmd)
         next(dose_result) # skip header
         for d in dose_result:
-            if d["status"] == "ok":
-                yield (d["package"], d["version"], None)
-            else:
+            reason = None
+            if d["status"] != "ok":
                 r = d["reasons"][0]
                 if "missing" in r:
                     reason = "missing %s" % r["missing"]["pkg"]["unsat-dependency"].split()[0].split(":", 1)[0]
@@ -405,14 +407,11 @@ def check_bdsat(mirror, arch):
                     reason += r.split()[0].split(':', 1)[0]
                 else:
                     assert False
-                yield (d["package"], d["version"], reason)
+            result[d["package"]] = (d["version"], reason)
+    return result
 
-def update_depcheck(mirror, db, architecture):
-    now = datetime.datetime.utcnow()
-    mirror.update_release()
-    state = {}
-    for source, version, reason in check_bdsat(mirror, architecture):
-        state[source] = (version, reason)
+
+def update_depcheck(mirror, db, updatetime, architecture, state):
     with contextlib.closing(db.cursor()) as cur:
         cur.execute("BEGIN;")
         cur.execute("SELECT source, version, satisfiable, reason FROM depstate WHERE architecture = ?;",
@@ -429,14 +428,21 @@ def update_depcheck(mirror, db, architecture):
                           reason)
                          for source, (version, reason) in state.items()))
         cur.execute("UPDATE depcheck SET releasetime = ?, updatetime = ?, giveback = 0 WHERE architecture = ?",
-                    (mirror.releasetime, now, architecture))
+                    (mirror.releasetime, updatetime, architecture))
     db.commit()
 
+
+def main_docheck(mirror, architecture):
+    return (architecture, check_bdsat(mirror, architecture))
+
+
 def main():
     argp = argparse.ArgumentParser()
     argp.add_argument('-m', '--mirror',
                       default='http://deb.debian.org/debian',
                       help="debian mirror to use")
+    argp.add_argument('-p', '--parallel', action="store_true",
+                      help="enable parallel checking")
     args = argp.parse_args()
     mirror = DebianMirror(args.mirror)
     mirror.update_release()
@@ -445,9 +451,14 @@ def main():
     cur.execute("""SELECT architecture FROM depcheck
                        WHERE giveback = 1 OR releasetime < ?;""",
                 (mirror.releasetime,))
-    for architecture, in list(cur.fetchall()):
+    archs = set(row[0] for row in cur.fetchall())
+    print("checking %s" % " ".join(sorted(archs)))
+    now = datetime.datetime.utcnow()
+    mapper = multiprocessing.Pool().imap_unordered if args.parallel else map
+    for architecture, state in mapper(functools.partial(main_docheck, mirror),
+                                      archs):
         print("update %s" % architecture)
-        update_depcheck(mirror, db, architecture)
+        update_depcheck(mirror, db, now, architecture, state)
 
 if __name__ == "__main__":
     main()