diff options
Diffstat (limited to 'autoimport.py')
| -rwxr-xr-x | autoimport.py | 61 |
1 files changed, 32 insertions, 29 deletions
diff --git a/autoimport.py b/autoimport.py index 9c4c73e..5c2c9be 100755 --- a/autoimport.py +++ b/autoimport.py @@ -26,26 +26,27 @@ def process_http(pkgs, url): pkglist = io.BytesIO(pkglist) pkglist = deb822.Packages.iter_paragraphs(pkglist) for pkg in pkglist: - name = pkg["Package"] - if name in pkgs and \ - version_compare(pkgs[name]["version"], pkg["Version"]) > 0: + key = (pkg["Package"], pkg["Architecture"]) + if key in pkgs and \ + version_compare(pkgs[key]["version"], pkg["Version"]) > 0: continue - pkgs[name] = dict(version=pkg["Version"], - filename="%s/%s" % (url, pkg["Filename"]), - sha256hash=pkg["SHA256"]) + pkgs[key] = dict(version=pkg["Version"], + filename="%s/%s" % (url, pkg["Filename"]), + sha256hash=pkg["SHA256"]) def process_file(pkgs, filename): base = os.path.basename(filename) if not base.endswith(".deb"): raise ValueError("filename does not end in .deb") - parts = base.split("_") + parts = base[:-4].split("_") if len(parts) != 3: raise ValueError("filename not in form name_version_arch.deb") - name, version, _ = parts + name, version, architecture = parts + key = (name, architecture) version = urllib.unquote(version) - if name in pkgs and version_compare(pkgs[name]["version"], version) > 0: + if key in pkgs and version_compare(pkgs[key]["version"], version) > 0: return - pkgs[name] = dict(version=version, filename=filename) + pkgs[key] = dict(version=version, filename=filename) def process_dir(pkgs, d): for entry in os.listdir(d): @@ -54,7 +55,7 @@ def process_dir(pkgs, d): except ValueError: pass -def process_pkg(name, pkgdict, outpath): +def process_pkg(key, pkgdict, outpath): filename = pkgdict["filename"] print("importing %s" % filename) importcmd = ["python", "importpkg.py"] @@ -75,7 +76,7 @@ def process_pkg(name, pkgdict, outpath): with open(outpath, "w") as outp: subprocess.check_call(importcmd, stdin=inp, stdout=outp, close_fds=True) - print("preprocessed %s" % name) + print("preprocessed %s:%s" % key) def main(): parser = optparse.OptionParser() @@ -103,42 +104,44 @@ def main(): process_file(pkgs, d) print("reading database") - cur.execute("SELECT name, version FROM package;") - knownpkgs = dict((row[0], row[1]) for row in cur.fetchall()) + cur.execute("SELECT name, architecture, version FROM package;") + knownpkgs = dict(((row[0], row[1]), row[2]) for row in cur.fetchall()) distpkgs = set(pkgs.keys()) if options.new: - for name in distpkgs: - if name in knownpkgs and version_compare(pkgs[name]["version"], - knownpkgs[name]) <= 0: - del pkgs[name] + for key in distpkgs: + if key in knownpkgs and version_compare(pkgs[key]["version"], + knownpkgs[key]) <= 0: + del pkgs[key] knownpkgs = set(knownpkgs) with e: fs = {} - for name, pkg in pkgs.items(): - outpath = os.path.join(tmpdir, name) - fs[e.submit(process_pkg, name, pkg, outpath)] = name + for key, pkg in pkgs.items(): + outpath = os.path.join(tmpdir, "%s_%s" % key) + fs[e.submit(process_pkg, key, pkg, outpath)] = key for f in concurrent.futures.as_completed(fs.keys()): - name = fs[f] + key = fs[f] if f.exception(): - print("%s failed to import: %r" % (name, f.exception())) + print("%s:%s failed to import: %r" % + (key[0], key[1], f.exception())) continue - inf = os.path.join(tmpdir, name) - print("sqlimporting %s" % name) + inf = os.path.join(tmpdir, "%s_%s" % key) + print("sqlimporting %s:%s" % key) with open(inf) as inp: try: readyaml(db, inp) except Exception as exc: - print("%s failed sql with exception %r" % (name, exc)) + print("%s:%s failed sql with exception %r" % + (key[0], key[1], exc)) else: os.unlink(inf) if options.prune: delpkgs = knownpkgs - distpkgs - print("clearing packages %s" % " ".join(delpkgs)) - cur.executemany("DELETE FROM package WHERE name = ?;", - ((pkg,) for pkg in delpkgs)) + print("clearing packages %s" % " ".join(map("%s:%s".__mod__, delpkgs))) + cur.executemany("DELETE FROM package WHERE name = ? AND architecture = ?;", + delpkgs) # Tables content, dependency and sharing will also be pruned # due to ON DELETE CASCADE clauses. db.commit() |