table buildreqeusts should have an explicit id

[~helmut/crossqa.git] / depcheck.py

diff --git a/depcheck.py b/depcheck.py
index 7f47a58..743a09d 100755 (executable)
--- a/depcheck.py
+++ b/depcheck.py
@@ -1,12 +1,15 @@
 #!/usr/bin/python3
+# SPDX-License-Identifier: GPL-2.0+
 
 import argparse
 import collections
 import contextlib
 import datetime
+import functools
 import hashlib
 import itertools
 import lzma
+import multiprocessing
 import os.path
 import sqlite3
 import subprocess
@@ -378,6 +381,7 @@ def check_bdsat(mirror, arch):
         "--deb-emulate-sbuild",
     ]
 
+    result = {}
     with tempfile.NamedTemporaryFile("w", encoding="utf8") as bintmp, \
             tempfile.NamedTemporaryFile("w", encoding="utf8") as srctmp:
         for p in make_binary_list(mirror, arch):
@@ -393,9 +397,8 @@ def check_bdsat(mirror, arch):
         dose_result = call_dose_builddebcheck(cmd)
         next(dose_result) # skip header
         for d in dose_result:
-            if d["status"] == "ok":
-                yield (d["package"], d["version"], True, None)
-            else:
+            reason = None
+            if d["status"] != "ok":
                 r = d["reasons"][0]
                 if "missing" in r:
                     reason = "missing %s" % r["missing"]["pkg"]["unsat-dependency"].split()[0].split(":", 1)[0]
@@ -405,47 +408,60 @@ def check_bdsat(mirror, arch):
                     reason += r.split()[0].split(':', 1)[0]
                 else:
                     assert False
-                yield (d["package"], d["version"], False, reason)
+            result[d["package"]] = (d["version"], reason)
+    return result
 
-def update_depcheck(mirror, db, architecture):
-    now = datetime.datetime.utcnow()
-    mirror.update_release()
-    state = {}
-    for source, version, satisfiable, reason in check_bdsat(mirror, architecture):
-        state[source] = (version, satisfiable, reason)
+
+def update_depcheck(mirror, db, updatetime, architecture, state):
     with contextlib.closing(db.cursor()) as cur:
         cur.execute("BEGIN;")
         cur.execute("SELECT source, version, satisfiable, reason FROM depstate WHERE architecture = ?;",
                     (architecture,))
         for source, version, satisfiable, reason in list(cur.fetchall()):
-            if state.get(source) == (version, satisfiable, reason):
+            if satisfiable == (reason is None) and \
+               state.get(source) == (version, reason):
                 del state[source]
             else:
                 cur.execute("DELETE FROM depstate WHERE source = ? AND version = ? AND architecture = ?;",
                             (source, version, architecture))
         cur.executemany("INSERT INTO depstate (source, architecture, version, satisfiable, reason) VALUES (?, ?, ?, ?, ?);",
-                        ((source, architecture, version, satisfiable, reason)
-                         for source, (version, satisfiable, reason) in state.items()))
+                        ((source, architecture, version, reason is None,
+                          reason)
+                         for source, (version, reason) in state.items()))
         cur.execute("UPDATE depcheck SET releasetime = ?, updatetime = ?, giveback = 0 WHERE architecture = ?",
-                    (mirror.releasetime, now, architecture))
+                    (mirror.releasetime, updatetime, architecture))
     db.commit()
 
+
+def main_docheck(mirror, architecture):
+    return (architecture, check_bdsat(mirror, architecture))
+
+
 def main():
     argp = argparse.ArgumentParser()
     argp.add_argument('-m', '--mirror',
                       default='http://deb.debian.org/debian',
                       help="debian mirror to use")
+    argp.add_argument('-p', '--parallel', action="store_true",
+                      help="enable parallel checking")
     args = argp.parse_args()
     mirror = DebianMirror(args.mirror)
     mirror.update_release()
-    db = sqlite3.connect("db", detect_types=sqlite3.PARSE_DECLTYPES)
+    db = sqlite3.connect("db")
     cur = db.cursor()
-    cur.execute("SELECT architecture, releasetime, updatetime, giveback FROM depcheck;")
-    lastupdate = datetime.datetime.utcnow() - datetime.timedelta(hours=6)
-    for architecture, releasetime, updatetime, giveback in list(cur.fetchall()):
-        if giveback or updatetime < lastupdate or releasetime < mirror.releasetime:
-            print("update %s" % architecture)
-            update_depcheck(mirror, db, architecture)
+    cur.execute("""SELECT architecture FROM depcheck
+                       WHERE giveback = 1 OR releasetime < ?;""",
+                (mirror.releasetime,))
+    archs = set(row[0] for row in cur.fetchall())
+    if not archs:
+        return
+    print("checking %s" % " ".join(sorted(archs)))
+    now = datetime.datetime.utcnow().replace(microsecond=0)
+    mapper = multiprocessing.Pool().imap_unordered if args.parallel else map
+    for architecture, state in mapper(functools.partial(main_docheck, mirror),
+                                      archs):
+        print("update %s" % architecture)
+        update_depcheck(mirror, db, now, architecture, state)
 
 if __name__ == "__main__":
     main()