From 072f38cdfae031bcfb3c18cd0ada89f9b9d4f94b Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Wed, 20 Feb 2013 21:37:57 +0100 Subject: reduce memory usage of autoimport --- autoimport.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/autoimport.py b/autoimport.py index 23092d7..ab3e45c 100755 --- a/autoimport.py +++ b/autoimport.py @@ -13,18 +13,16 @@ def main(): urlbase = sys.argv[1] db = sqlite3.connect("test.sqlite3") cur = db.cursor() + cur.execute("SELECT package, version FROM package;") + knownpkgs = dict((row[0], row[1]) for row in cur.fetchall()) - cur.execute("SELECT package, version, architecture FROM package;") - knownpkgs = dict((row[0], dict(package=row[0], version=row[1], - architecture=row[2])) - for row in cur.fetchall()) pkglist = urllib.urlopen(urlbase + "/dists/sid/main/binary-amd64/Packages.gz").read() pkglist = gzip.GzipFile(fileobj=io.BytesIO(pkglist)).read() - distpkgs = dict() + distpkgs = set() for pkg in deb822.Packages.iter_paragraphs(io.BytesIO(pkglist)): name = pkg["Package"] - distpkgs[name] = pkg - if pkg["Version"] == knownpkgs.get(name, dict(version=()))["version"]: + distpkgs.add(name) + if pkg["Version"] == knownpkgs.get(name, ()): continue pkgurl = "%s/%s" % (urlbase, pkg["Filename"]) print("importing %s" % name) @@ -37,7 +35,7 @@ def main(): cur.execute("PRAGMA foreign_keys=1;") cur.executemany("DELETE FROM package WHERE package = ?;", - ((pkg,) for pkg in set(knownpkgs) - set(distpkgs))) + ((pkg,) for pkg in set(knownpkgs) - distpkgs)) db.commit() if __name__ == "__main__": -- cgit v1.2.3