diff options
| -rwxr-xr-x | autoimport.py | 22 |
1 files changed, 16 insertions, 6 deletions
diff --git a/autoimport.py b/autoimport.py index 481a3f8..a0681b3 100755 --- a/autoimport.py +++ b/autoimport.py @@ -4,12 +4,14 @@ packages contained. It has rather strong assumptions on the working directory. """ import gzip +import errno import io import multiprocessing import optparse import os import sqlite3 import subprocess +import tempfile import urllib import concurrent.futures @@ -52,14 +54,14 @@ def process_dir(pkgs, d): except ValueError: pass -def process_pkg(name, pkgdict): +def process_pkg(name, pkgdict, outpath): filename = pkgdict["filename"] print("importing %s" % filename) importcmd = ["python", "importpkg.py"] if "sha256hash" in pkgdict: importcmd.extend(["-H", pkgdict["sha256hash"]]) if filename.startswith("http://"): - with open(os.path.join("tmp", name), "w") as outp: + with open(outpath, "w") as outp: dl = subprocess.Popen(["curl", "-s", filename], stdout=subprocess.PIPE, close_fds=True) imp = subprocess.Popen(importcmd, stdin=dl.stdout, stdout=outp, @@ -70,7 +72,7 @@ def process_pkg(name, pkgdict): raise ValueError("curl failed") else: with open(filename) as inp: - with open(os.path.join("tmp", name), "w") as outp: + with open(outpath, "w") as outp: subprocess.check_call(importcmd, stdin=inp, stdout=outp, close_fds=True) print("preprocessed %s" % name) @@ -82,7 +84,7 @@ def main(): parser.add_option("-p", "--prune", action="store_true", help="prune packages old packages") options, args = parser.parse_args() - subprocess.check_call(["mkdir", "-p", "tmp"]) + tmpdir = tempfile.mkdtemp(prefix=b"debian-dedup") db = sqlite3.connect("test.sqlite3") cur = db.cursor() cur.execute("PRAGMA foreign_keys = ON;") @@ -111,14 +113,15 @@ def main(): with e: fs = {} for name, pkg in pkgs.items(): - fs[e.submit(process_pkg, name, pkg)] = name + outpath = os.path.join(tmpdir, name) + fs[e.submit(process_pkg, name, pkg, outpath)] = name for f in concurrent.futures.as_completed(fs.keys()): name = fs[f] if f.exception(): print("%s failed to import: %r" % (name, f.exception())) continue - inf = os.path.join("tmp", name) + inf = os.path.join(tmpdir, name) print("sqlimporting %s" % name) with open(inf) as inp: try: @@ -136,6 +139,13 @@ def main(): # Tables content, dependency and sharing will also be pruned # due to ON DELETE CASCADE clauses. db.commit() + try: + os.rmdir(tmpdir) + except OSError as err: + if err.errno != errno.ENOTEMPTY: + raise + print("keeping temporary directory %s due to failed packages %s" % + (tmpdir, " ".join(os.listdir(tmpdir)))) if __name__ == "__main__": main() |