From c0cd9df1e6cb63d524939028f5f6a07c2c8c3da5 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Mon, 2 Sep 2013 10:00:44 +0200 Subject: autoimport: avoid hard coded temporary directory --- autoimport.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) (limited to 'autoimport.py') diff --git a/autoimport.py b/autoimport.py index 481a3f8..a0681b3 100755 --- a/autoimport.py +++ b/autoimport.py @@ -4,12 +4,14 @@ packages contained. It has rather strong assumptions on the working directory. """ import gzip +import errno import io import multiprocessing import optparse import os import sqlite3 import subprocess +import tempfile import urllib import concurrent.futures @@ -52,14 +54,14 @@ def process_dir(pkgs, d): except ValueError: pass -def process_pkg(name, pkgdict): +def process_pkg(name, pkgdict, outpath): filename = pkgdict["filename"] print("importing %s" % filename) importcmd = ["python", "importpkg.py"] if "sha256hash" in pkgdict: importcmd.extend(["-H", pkgdict["sha256hash"]]) if filename.startswith("http://"): - with open(os.path.join("tmp", name), "w") as outp: + with open(outpath, "w") as outp: dl = subprocess.Popen(["curl", "-s", filename], stdout=subprocess.PIPE, close_fds=True) imp = subprocess.Popen(importcmd, stdin=dl.stdout, stdout=outp, @@ -70,7 +72,7 @@ def process_pkg(name, pkgdict): raise ValueError("curl failed") else: with open(filename) as inp: - with open(os.path.join("tmp", name), "w") as outp: + with open(outpath, "w") as outp: subprocess.check_call(importcmd, stdin=inp, stdout=outp, close_fds=True) print("preprocessed %s" % name) @@ -82,7 +84,7 @@ def main(): parser.add_option("-p", "--prune", action="store_true", help="prune packages old packages") options, args = parser.parse_args() - subprocess.check_call(["mkdir", "-p", "tmp"]) + tmpdir = tempfile.mkdtemp(prefix=b"debian-dedup") db = sqlite3.connect("test.sqlite3") cur = db.cursor() cur.execute("PRAGMA foreign_keys = ON;") @@ -111,14 +113,15 @@ def main(): with e: fs = {} for name, pkg in pkgs.items(): - fs[e.submit(process_pkg, name, pkg)] = name + outpath = os.path.join(tmpdir, name) + fs[e.submit(process_pkg, name, pkg, outpath)] = name for f in concurrent.futures.as_completed(fs.keys()): name = fs[f] if f.exception(): print("%s failed to import: %r" % (name, f.exception())) continue - inf = os.path.join("tmp", name) + inf = os.path.join(tmpdir, name) print("sqlimporting %s" % name) with open(inf) as inp: try: @@ -136,6 +139,13 @@ def main(): # Tables content, dependency and sharing will also be pruned # due to ON DELETE CASCADE clauses. db.commit() + try: + os.rmdir(tmpdir) + except OSError as err: + if err.errno != errno.ENOTEMPTY: + raise + print("keeping temporary directory %s due to failed packages %s" % + (tmpdir, " ".join(os.listdir(tmpdir)))) if __name__ == "__main__": main() -- cgit v1.2.3 From 022985f098a206c3b7852fe08a798cd31623f10d Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Mon, 2 Sep 2013 18:51:20 +0200 Subject: add option -d --database for db path to all scripts --- autoimport.py | 5 ++++- readyaml.py | 8 +++++++- update_sharing.py | 11 ++++++++--- webapp.py | 8 +++++++- 4 files changed, 26 insertions(+), 6 deletions(-) (limited to 'autoimport.py') diff --git a/autoimport.py b/autoimport.py index a0681b3..d44c012 100755 --- a/autoimport.py +++ b/autoimport.py @@ -83,9 +83,12 @@ def main(): help="avoid reimporting same versions") parser.add_option("-p", "--prune", action="store_true", help="prune packages old packages") + parser.add_option("-d", "--database", action="store", + default="test.sqlite3", + help="path to the sqlite3 database file") options, args = parser.parse_args() tmpdir = tempfile.mkdtemp(prefix=b"debian-dedup") - db = sqlite3.connect("test.sqlite3") + db = sqlite3.connect(options.database) cur = db.cursor() cur.execute("PRAGMA foreign_keys = ON;") e = concurrent.futures.ThreadPoolExecutor(multiprocessing.cpu_count()) diff --git a/readyaml.py b/readyaml.py index 21b1ca1..2ef9a3b 100755 --- a/readyaml.py +++ b/readyaml.py @@ -2,6 +2,7 @@ """This tool reads a yaml file as generated by importpkg.py on stdin and updates the database with the contents.""" +import optparse import sqlite3 import sys @@ -53,7 +54,12 @@ def readyaml(db, stream): raise ValueError("missing commit block") def main(): - db = sqlite3.connect("test.sqlite3") + parser = optparse.OptionParser() + parser.add_option("-d", "--database", action="store", + default="test.sqlite3", + help="path to the sqlite3 database file") + options, args = parser.parse_args() + db = sqlite3.connect(options.database) readyaml(db, sys.stdin) if __name__ == "__main__": diff --git a/update_sharing.py b/update_sharing.py index 5ec6c7b..1ff0fd8 100755 --- a/update_sharing.py +++ b/update_sharing.py @@ -1,5 +1,6 @@ #!/usr/bin/python +import optparse import sqlite3 from dedup.utils import fetchiter @@ -37,8 +38,7 @@ def process_pkgdict(cursor, pkgdict): insert_key = (pid1, pid2, fid1, fid2) add_values(cursor, insert_key, pkgnumfiles, pkgsize) -def main(): - db = sqlite3.connect("test.sqlite3") +def main(db): cur = db.cursor() cur.execute("PRAGMA foreign_keys = ON;") cur.execute("DELETE FROM sharing;") @@ -61,4 +61,9 @@ def main(): db.commit() if __name__ == "__main__": - main() + parser = optparse.OptionParser() + parser.add_option("-d", "--database", action="store", + default="test.sqlite3", + help="path to the sqlite3 database file") + options, args = parser.parse_args() + main(sqlite3.connect(options.database)) diff --git a/webapp.py b/webapp.py index f202c2e..632b485 100755 --- a/webapp.py +++ b/webapp.py @@ -1,6 +1,7 @@ #!/usr/bin/python import datetime +import optparse import sqlite3 from wsgiref.simple_server import make_server @@ -237,7 +238,12 @@ class Application(object): return html_response(source_template.render(params)) def main(): - app = Application(sqlite3.connect("test.sqlite3")) + parser = optparse.OptionParser() + parser.add_option("-d", "--database", action="store", + default="test.sqlite3", + help="path to the sqlite3 database file") + options, args = parser.parse_args() + app = Application(sqlite3.connect(options.database)) app = SharedDataMiddleware(app, {"/": ("dedup", "static")}) make_server("0.0.0.0", 8800, app).serve_forever() -- cgit v1.2.3