permit multiple architectures per package

While the importer can easily cope with this change, the web
presentation still needs fixing. It works somewhat now.

1 files changed, 32 insertions, 29 deletions

diff --git a/autoimport.py b/autoimport.py
index d44c012..c7eb2f5 100755
--- a/autoimport.py
+++ b/autoimport.py
@@ -26,26 +26,27 @@ def process_http(pkgs, url):
     pkglist = io.BytesIO(pkglist)
     pkglist = deb822.Packages.iter_paragraphs(pkglist)
     for pkg in pkglist:
-        name = pkg["Package"]
-        if name in pkgs and \
-                version_compare(pkgs[name]["version"], pkg["Version"]) > 0:
+        key = (pkg["Package"], pkg["Architecture"])
+        if key in pkgs and \
+                version_compare(pkgs[key]["version"], pkg["Version"]) > 0:
             continue
-        pkgs[name] = dict(version=pkg["Version"],
-                          filename="%s/%s" % (url, pkg["Filename"]),
-                          sha256hash=pkg["SHA256"])
+        pkgs[key] = dict(version=pkg["Version"],
+                         filename="%s/%s" % (url, pkg["Filename"]),
+                         sha256hash=pkg["SHA256"])
 
 def process_file(pkgs, filename):
     base = os.path.basename(filename)
     if not base.endswith(".deb"):
         raise ValueError("filename does not end in .deb")
-    parts = base.split("_")
+    parts = base[:-4].split("_")
     if len(parts) != 3:
         raise ValueError("filename not in form name_version_arch.deb")
-    name, version, _ = parts
+    name, version, architecture = parts
+    key = (name, architecture)
     version = urllib.unquote(version)
-    if name in pkgs and version_compare(pkgs[name]["version"], version) > 0:
+    if key in pkgs and version_compare(pkgs[key]["version"], version) > 0:
         return
-    pkgs[name] = dict(version=version, filename=filename)
+    pkgs[key] = dict(version=version, filename=filename)
 
 def process_dir(pkgs, d):
     for entry in os.listdir(d):
@@ -54,7 +55,7 @@ def process_dir(pkgs, d):
         except ValueError:
             pass
 
-def process_pkg(name, pkgdict, outpath):
+def process_pkg(key, pkgdict, outpath):
     filename = pkgdict["filename"]
     print("importing %s" % filename)
     importcmd = ["python", "importpkg.py"]
@@ -75,7 +76,7 @@ def process_pkg(name, pkgdict, outpath):
             with open(outpath, "w") as outp:
                 subprocess.check_call(importcmd, stdin=inp, stdout=outp,
                                       close_fds=True)
-    print("preprocessed %s" % name)
+    print("preprocessed %s:%s" % key)
 
 def main():
     parser = optparse.OptionParser()
@@ -103,42 +104,44 @@ def main():
             process_file(pkgs, d)
 
     print("reading database")
-    cur.execute("SELECT name, version FROM package;")
-    knownpkgs = dict((row[0], row[1]) for row in cur.fetchall())
+    cur.execute("SELECT name, architecture, version FROM package;")
+    knownpkgs = dict(((row[0], row[1]), row[2]) for row in cur.fetchall())
     distpkgs = set(pkgs.keys())
     if options.new:
-        for name in distpkgs:
-            if name in knownpkgs and version_compare(pkgs[name]["version"],
-                    knownpkgs[name]) <= 0:
-                del pkgs[name]
+        for key in distpkgs:
+            if key in knownpkgs and version_compare(pkgs[key]["version"],
+                    knownpkgs[key]) <= 0:
+                del pkgs[key]
     knownpkgs = set(knownpkgs)
 
     with e:
         fs = {}
-        for name, pkg in pkgs.items():
-            outpath = os.path.join(tmpdir, name)
-            fs[e.submit(process_pkg, name, pkg, outpath)] = name
+        for key, pkg in pkgs.items():
+            outpath = os.path.join(tmpdir, "%s_%s" % key)
+            fs[e.submit(process_pkg, key, pkg, outpath)] = key
 
         for f in concurrent.futures.as_completed(fs.keys()):
-            name = fs[f]
+            key = fs[f]
             if f.exception():
-                print("%s failed to import: %r" % (name, f.exception()))
+                print("%s:%s failed to import: %r" %
+                      (key[0], key[1], f.exception()))
                 continue
-            inf = os.path.join(tmpdir, name)
-            print("sqlimporting %s" % name)
+            inf = os.path.join(tmpdir, "%s_%s" % key)
+            print("sqlimporting %s:%s" % key)
             with open(inf) as inp:
                 try:
                     readyaml(db, inp)
                 except Exception as exc:
-                    print("%s failed sql with exception %r" % (name, exc))
+                    print("%s:%s failed sql with exception %r" %
+                          (key[0], key[1], exc))
                 else:
                     os.unlink(inf)
 
     if options.prune:
         delpkgs = knownpkgs - distpkgs
-        print("clearing packages %s" % " ".join(delpkgs))
-        cur.executemany("DELETE FROM package WHERE name = ?;",
-                        ((pkg,) for pkg in delpkgs))
+        print("clearing packages %s" % " ".join(map("%s:%s".__mod__, delpkgs)))
+        cur.executemany("DELETE FROM package WHERE name = ? AND architecture = ?;",
+                        delpkgs)
         # Tables content, dependency and sharing will also be pruned
         # due to ON DELETE CASCADE clauses.
         db.commit()