Merge branch 'master' into sqlalchemy

In the mean time, the master branch evolved quite a bit and the schema
changed again (eqclass added to function table). The main reason for the
merge is to resolve the large amounts of conflicts once, so development
of the sqlalchemy branch can continue and still benefit from changes in
the master branch such as schema compatibility, adapting the indent
level in web app due to the use of contextlib.closing which resembles
sqlalchemy's "with db.begin() as conn:".

Conflicts:
	autoimport.py
	dedup/utils.py
	readyaml.py
	update_sharing.py
	webapp.py

1 files changed, 19 insertions, 6 deletions

diff --git a/autoimport.py b/autoimport.py
index c1e0da5..8c61a18 100755
--- a/autoimport.py
+++ b/autoimport.py
@@ -4,11 +4,13 @@ packages contained. It has rather strong assumptions on the working directory.
 """
 
 import gzip
+import errno
 import io
 import multiprocessing
 import optparse
 import os
 import subprocess
+import tempfile
 import urllib
 
 import concurrent.futures
@@ -54,14 +56,14 @@ def process_dir(pkgs, d):
         except ValueError:
             pass
 
-def process_pkg(name, pkgdict):
+def process_pkg(name, pkgdict, outpath):
     filename = pkgdict["filename"]
     print("importing %s" % filename)
     importcmd = ["python", "importpkg.py"]
     if "sha256hash" in pkgdict:
         importcmd.extend(["-H", pkgdict["sha256hash"]])
     if filename.startswith("http://"):
-        with open(os.path.join("tmp", name), "w") as outp:
+        with open(outpath, "w") as outp:
             dl = subprocess.Popen(["curl", "-s", filename],
                                   stdout=subprocess.PIPE, close_fds=True)
             imp = subprocess.Popen(importcmd, stdin=dl.stdout, stdout=outp,
@@ -72,7 +74,7 @@ def process_pkg(name, pkgdict):
                 raise ValueError("curl failed")
     else:
         with open(filename) as inp:
-            with open(os.path.join("tmp", name), "w") as outp:
+            with open(outpath, "w") as outp:
                 subprocess.check_call(importcmd, stdin=inp, stdout=outp,
                                       close_fds=True)
     print("preprocessed %s" % name)
@@ -83,8 +85,11 @@ def main():
                       help="avoid reimporting same versions")
     parser.add_option("-p", "--prune", action="store_true",
                       help="prune packages old packages")
+    parser.add_option("-d", "--database", action="store",
+                      default="sqlite:///test.sqlite3",
+                      help="location of the database")
     options, args = parser.parse_args()
-    subprocess.check_call(["mkdir", "-p", "tmp"])
+    tmpdir = tempfile.mkdtemp(prefix=b"debian-dedup")
     db = sqlalchemy.create_engine("sqlite:///test.sqlite3")
     enable_sqlite_foreign_keys(db)
     e = concurrent.futures.ThreadPoolExecutor(multiprocessing.cpu_count())
@@ -113,14 +118,15 @@ def main():
     with e:
         fs = {}
         for name, pkg in pkgs.items():
-            fs[e.submit(process_pkg, name, pkg)] = name
+            outpath = os.path.join(tmpdir, name)
+            fs[e.submit(process_pkg, name, pkg, outpath)] = name
 
         for f in concurrent.futures.as_completed(fs.keys()):
             name = fs[f]
             if f.exception():
                 print("%s failed to import: %r" % (name, f.exception()))
                 continue
-            inf = os.path.join("tmp", name)
+            inf = os.path.join(tmpdir, name)
             print("sqlimporting %s" % name)
             with open(inf) as inp:
                 try:
@@ -140,6 +146,13 @@ def main():
                              [dict(name=pkg) for pkg in delpkgs])
                 # Tables content, dependency and sharing will also be pruned
                 # due to ON DELETE CASCADE clauses.
+    try:
+        os.rmdir(tmpdir)
+    except OSError as err:
+        if err.errno != errno.ENOTEMPTY:
+            raise
+        print("keeping temporary directory %s due to failed packages %s" %
+              (tmpdir, " ".join(os.listdir(tmpdir))))
 
 if __name__ == "__main__":
     main()