dedup.utils: uninline helper function iterate_packages

2 files changed, 22 insertions, 15 deletions

diff --git a/autoimport.py b/autoimport.py
index eb610b4..0f518c6 100755
--- a/autoimport.py
+++ b/autoimport.py
@@ -4,7 +4,6 @@ packages contained. It has rather strong assumptions on the working directory.
 """
 
 import argparse
-import contextlib
 import errno
 import multiprocessing
 import pathlib
@@ -14,26 +13,23 @@ import sys
 import tempfile
 import urllib.parse
 import concurrent.futures
-from debian import deb822
 from debian.debian_support import version_compare
 
-from dedup.utils import open_compressed_mirror_url
+from dedup.utils import iterate_packages
 
 from readyaml import readyaml
 
 def process_http(pkgs, url, addhash=True):
-    listurl = url + "/dists/sid/main/binary-amd64/Packages"
-    with contextlib.closing(open_compressed_mirror_url(listurl)) as pkglist:
-        for pkg in deb822.Packages.iter_paragraphs(pkglist):
-            name = pkg["Package"]
-            if name in pkgs and \
-                    version_compare(pkgs[name]["version"], pkg["Version"]) > 0:
-                continue
-            inst = dict(version=pkg["Version"],
-                        filename="%s/%s" % (url, pkg["Filename"]))
-            if addhash:
-                inst["sha256hash"] = pkg["SHA256"]
-            pkgs[name] = inst
+    for pkg in iterate_packages(url, "amd64"):
+        name = pkg["Package"]
+        if name in pkgs and \
+                version_compare(pkgs[name]["version"], pkg["Version"]) > 0:
+            continue
+        inst = dict(version=pkg["Version"],
+                    filename="%s/%s" % (url, pkg["Filename"]))
+        if addhash:
+            inst["sha256hash"] = pkg["SHA256"]
+        pkgs[name] = inst
 
 def process_file(pkgs, filename):
     if filename.suffix != ".deb":
diff --git a/dedup/utils.py b/dedup/utils.py
index d3a27a0..55cdef0 100644
--- a/dedup/utils.py
+++ b/dedup/utils.py
@@ -1,7 +1,10 @@
+import contextlib
 import errno
 import urllib.error
 import urllib.request
 
+import debian.deb822
+
 from dedup.compression import decompress
 
 def fetchiter(cursor):
@@ -30,3 +33,11 @@ def open_compressed_mirror_url(url, extensions=(".xz", ".gz", "")):
         else:
             return decompress(handle, ext)
     raise OSError(errno.ENOENT, "No such file or directory")
+
+def iterate_packages(mirror, architecture, distribution="sid", section="main"):
+    """Download the relevant binary package list and generate
+    debian.deb822.Packages objects per listed package."""
+    url = "%s/dists/%s/%s/binary-%s/Packages" % \
+            (mirror, distribution, section, architecture)
+    with contextlib.closing(open_compressed_mirror_url(url)) as pkglist:
+        yield from debian.deb822.Packages.iter_paragraphs(pkglist)