summaryrefslogtreecommitdiff
path: root/depcheck.py
diff options
context:
space:
mode:
authorHelmut Grohne <helmut@subdivi.de>2019-03-02 16:10:13 +0100
committerHelmut Grohne <helmut@subdivi.de>2019-03-02 16:10:13 +0100
commit1e1078f6bdf677bbb557291d76ba012a013f81a7 (patch)
tree603ef7ffac493412786621995052def174d4cee2 /depcheck.py
parent4d08683201cc37586ffbb3bd5dcf932851c8a913 (diff)
downloadcrossqa-1e1078f6bdf677bbb557291d76ba012a013f81a7.tar.gz
depcheck: support parallel operation
Diffstat (limited to 'depcheck.py')
-rwxr-xr-xdepcheck.py37
1 files changed, 24 insertions, 13 deletions
diff --git a/depcheck.py b/depcheck.py
index d2c6bf6..3bbe2d7 100755
--- a/depcheck.py
+++ b/depcheck.py
@@ -4,9 +4,11 @@ import argparse
import collections
import contextlib
import datetime
+import functools
import hashlib
import itertools
import lzma
+import multiprocessing
import os.path
import sqlite3
import subprocess
@@ -378,6 +380,7 @@ def check_bdsat(mirror, arch):
"--deb-emulate-sbuild",
]
+ result = {}
with tempfile.NamedTemporaryFile("w", encoding="utf8") as bintmp, \
tempfile.NamedTemporaryFile("w", encoding="utf8") as srctmp:
for p in make_binary_list(mirror, arch):
@@ -393,9 +396,8 @@ def check_bdsat(mirror, arch):
dose_result = call_dose_builddebcheck(cmd)
next(dose_result) # skip header
for d in dose_result:
- if d["status"] == "ok":
- yield (d["package"], d["version"], None)
- else:
+ reason = None
+ if d["status"] != "ok":
r = d["reasons"][0]
if "missing" in r:
reason = "missing %s" % r["missing"]["pkg"]["unsat-dependency"].split()[0].split(":", 1)[0]
@@ -405,14 +407,11 @@ def check_bdsat(mirror, arch):
reason += r.split()[0].split(':', 1)[0]
else:
assert False
- yield (d["package"], d["version"], reason)
+ result[d["package"]] = (d["version"], reason)
+ return result
-def update_depcheck(mirror, db, architecture):
- now = datetime.datetime.utcnow()
- mirror.update_release()
- state = {}
- for source, version, reason in check_bdsat(mirror, architecture):
- state[source] = (version, reason)
+
+def update_depcheck(mirror, db, updatetime, architecture, state):
with contextlib.closing(db.cursor()) as cur:
cur.execute("BEGIN;")
cur.execute("SELECT source, version, satisfiable, reason FROM depstate WHERE architecture = ?;",
@@ -429,14 +428,21 @@ def update_depcheck(mirror, db, architecture):
reason)
for source, (version, reason) in state.items()))
cur.execute("UPDATE depcheck SET releasetime = ?, updatetime = ?, giveback = 0 WHERE architecture = ?",
- (mirror.releasetime, now, architecture))
+ (mirror.releasetime, updatetime, architecture))
db.commit()
+
+def main_docheck(mirror, architecture):
+ return (architecture, check_bdsat(mirror, architecture))
+
+
def main():
argp = argparse.ArgumentParser()
argp.add_argument('-m', '--mirror',
default='http://deb.debian.org/debian',
help="debian mirror to use")
+ argp.add_argument('-p', '--parallel', action="store_true",
+ help="enable parallel checking")
args = argp.parse_args()
mirror = DebianMirror(args.mirror)
mirror.update_release()
@@ -445,9 +451,14 @@ def main():
cur.execute("""SELECT architecture FROM depcheck
WHERE giveback = 1 OR releasetime < ?;""",
(mirror.releasetime,))
- for architecture, in list(cur.fetchall()):
+ archs = set(row[0] for row in cur.fetchall())
+ print("checking %s" % " ".join(sorted(archs)))
+ now = datetime.datetime.utcnow()
+ mapper = multiprocessing.Pool().imap_unordered if args.parallel else map
+ for architecture, state in mapper(functools.partial(main_docheck, mirror),
+ archs):
print("update %s" % architecture)
- update_depcheck(mirror, db, architecture)
+ update_depcheck(mirror, db, now, architecture, state)
if __name__ == "__main__":
main()