From 420804c25797add9689bacbd3db138a92a623c8e Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Wed, 25 May 2016 19:27:35 +0200 Subject: autoimport: improve fetching package lists Moving the fetching part into dedup.utils. Instead of hard coding the gzip compressed copy, try xz, gz and plain in that order. Also take care to actually close the connection. --- autoimport.py | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) (limited to 'autoimport.py') diff --git a/autoimport.py b/autoimport.py index 9c416c5..5fb0659 100755 --- a/autoimport.py +++ b/autoimport.py @@ -4,6 +4,7 @@ packages contained. It has rather strong assumptions on the working directory. """ import argparse +import contextlib import errno import multiprocessing import os @@ -15,33 +16,28 @@ try: from urllib.parse import unquote except ImportError: from urllib import unquote -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen import concurrent.futures from debian import deb822 from debian.debian_support import version_compare -from dedup.compression import decompress +from dedup.utils import open_compressed_mirror_url from readyaml import readyaml def process_http(pkgs, url, addhash=True): - pkglist = urlopen(url + "/dists/sid/main/binary-amd64/Packages.gz") - pkglist = decompress(pkglist, ".gz") - pkglist = deb822.Packages.iter_paragraphs(pkglist) - for pkg in pkglist: - name = pkg["Package"] - if name in pkgs and \ - version_compare(pkgs[name]["version"], pkg["Version"]) > 0: - continue - inst = dict(version=pkg["Version"], - filename="%s/%s" % (url, pkg["Filename"])) - if addhash: - inst["sharing"] = pkg["SHA256"] - pkgs[name] = inst + url += "/dists/sid/main/binary-amd64/Packages" + with contextlib.closing(open_compressed_mirror_url(url)) as pkglist: + for pkg in deb822.Packages.iter_paragraphs(pkglist): + name = pkg["Package"] + if name in pkgs and \ + version_compare(pkgs[name]["version"], pkg["Version"]) > 0: + continue + inst = dict(version=pkg["Version"], + filename="%s/%s" % (url, pkg["Filename"])) + if addhash: + inst["sharing"] = pkg["SHA256"] + pkgs[name] = inst def process_file(pkgs, filename): base = os.path.basename(filename) -- cgit v1.2.3