diff options
Diffstat (limited to 'dedup/utils.py')
| -rw-r--r-- | dedup/utils.py | 36 |
1 files changed, 17 insertions, 19 deletions
diff --git a/dedup/utils.py b/dedup/utils.py index dab6653..55cdef0 100644 --- a/dedup/utils.py +++ b/dedup/utils.py @@ -1,29 +1,19 @@ +import contextlib import errno -try: - from urllib.error import URLError, HTTPError -except ImportError: - from urllib2 import URLError, HTTPError -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen +import urllib.error +import urllib.request -from debian.debian_support import version_compare +import debian.deb822 from dedup.compression import decompress def fetchiter(cursor): rows = cursor.fetchmany() while rows: - for row in rows: - yield row + yield from rows rows = cursor.fetchmany() -def sql_add_version_compare(db): - db.create_collation("debian_version", version_compare) - db.create_function("debian_version_compare", 2, version_compare) - -def open_compressed_mirror_url(url, extensions=(u".xz", u".gz", u"")): +def open_compressed_mirror_url(url, extensions=(".xz", ".gz", "")): """Fetch the given url. Try appending each of the given compression schemes and move on in case it doesn't exist. Decompress the resulting stream on the fly. @@ -31,11 +21,11 @@ def open_compressed_mirror_url(url, extensions=(u".xz", u".gz", u"")): """ for ext in extensions: try: - handle = urlopen(url + ext) - except HTTPError as error: + handle = urllib.request.urlopen(url + ext) + except urllib.error.HTTPError as error: if error.code != 404: raise - except URLError as error: + except urllib.error.URLError as error: if not hasattr(error.reason, "errno"): raise if error.reason.errno != errno.ENOENT: @@ -43,3 +33,11 @@ def open_compressed_mirror_url(url, extensions=(u".xz", u".gz", u"")): else: return decompress(handle, ext) raise OSError(errno.ENOENT, "No such file or directory") + +def iterate_packages(mirror, architecture, distribution="sid", section="main"): + """Download the relevant binary package list and generate + debian.deb822.Packages objects per listed package.""" + url = "%s/dists/%s/%s/binary-%s/Packages" % \ + (mirror, distribution, section, architecture) + with contextlib.closing(open_compressed_mirror_url(url)) as pkglist: + yield from debian.deb822.Packages.iter_paragraphs(pkglist) |