From b3d4bbae6fa2810daf3481d8ba99675ffe3bce9e Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 7 Mar 2013 08:24:44 +0100 Subject: integrate the source table into the package table --- schema.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'schema.sql') diff --git a/schema.sql b/schema.sql index 878a5fe..954b359 100644 --- a/schema.sql +++ b/schema.sql @@ -1,7 +1,6 @@ -CREATE TABLE package (package TEXT PRIMARY KEY, version TEXT, architecture TEXT); +CREATE TABLE package (package TEXT PRIMARY KEY, version TEXT, architecture TEXT, source TEXT); CREATE TABLE content (package TEXT, filename TEXT, size INTEGER, function TEXT, hash TEXT, FOREIGN KEY (package) REFERENCES package(package)); CREATE TABLE dependency (package TEXT, required TEXT, FOREIGN KEY (package) REFERENCES package(package), FOREIGN KEY (required) REFERENCES package(package)); -CREATE TABLE source (source TEXT, package TEXT, FOREIGN KEY (package) REFERENCES package(package)); CREATE INDEX content_package_index ON content (package); CREATE INDEX content_hash_index ON content (hash); -- cgit v1.2.3 From 827daf10eb510a00f9ca710a001f962524686520 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 7 Mar 2013 08:28:56 +0100 Subject: schema.sql: annotat foreign keys of sharing --- schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'schema.sql') diff --git a/schema.sql b/schema.sql index 954b359..d0a27cf 100644 --- a/schema.sql +++ b/schema.sql @@ -4,6 +4,6 @@ CREATE TABLE dependency (package TEXT, required TEXT, FOREIGN KEY (package) REFE CREATE INDEX content_package_index ON content (package); CREATE INDEX content_hash_index ON content (hash); -CREATE TABLE sharing (package1 TEXT, package2 TEXT, func1 TEXT, func2 TEXT, files INTEGER, size INTEGER); +CREATE TABLE sharing (package1 TEXT, package2 TEXT, func1 TEXT, func2 TEXT, files INTEGER, size INTEGER, FOREIGN KEY (package1) REFERENCES package(package), FOREIGN KEY (package2) REFERENCES package(package)); CREATE INDEX sharing_insert_index ON sharing (package1, package2, func1, func2); CREATE INDEX sharing_package_index ON sharing (package1); -- cgit v1.2.3 From d6638c18872db382a8d1045b6de1f9438c8a25fa Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 7 Mar 2013 08:41:35 +0100 Subject: schema.sql: remove unsatisfiable foreign key In the dependency table we will insert dependencies on packages which are not tracked. This happens during initial import and for virtual packages. Therefore the "required" column cannot be a foreign key. --- schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'schema.sql') diff --git a/schema.sql b/schema.sql index d0a27cf..72357e3 100644 --- a/schema.sql +++ b/schema.sql @@ -1,6 +1,6 @@ CREATE TABLE package (package TEXT PRIMARY KEY, version TEXT, architecture TEXT, source TEXT); CREATE TABLE content (package TEXT, filename TEXT, size INTEGER, function TEXT, hash TEXT, FOREIGN KEY (package) REFERENCES package(package)); -CREATE TABLE dependency (package TEXT, required TEXT, FOREIGN KEY (package) REFERENCES package(package), FOREIGN KEY (required) REFERENCES package(package)); +CREATE TABLE dependency (package TEXT, required TEXT, FOREIGN KEY (package) REFERENCES package(package)); CREATE INDEX content_package_index ON content (package); CREATE INDEX content_hash_index ON content (hash); -- cgit v1.2.3 From e0dd701f41a874c1c06702d849fe4b860a450e3e Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 7 Mar 2013 09:05:48 +0100 Subject: use "ON DELETE CASCADE" clauses --- autoimport.py | 10 ++-------- schema.sql | 6 +++--- 2 files changed, 5 insertions(+), 11 deletions(-) (limited to 'schema.sql') diff --git a/autoimport.py b/autoimport.py index bc2c6aa..453a839 100755 --- a/autoimport.py +++ b/autoimport.py @@ -38,16 +38,10 @@ def main(): delpkgs = set(knownpkgs) - distpkgs print("clearing packages %s" % " ".join(delpkgs)) - cur.executemany("DELETE FROM sharing WHERE package1 = ?", - ((pkg,) for pkg in delpkgs)) - cur.executemany("DELETE FROM sharing WHERE package2 = ?", - ((pkg,) for pkg in delpkgs)) - cur.executemany("DELETE FROM content WHERE package = ?;", - ((pkg,) for pkg in delpkgs)) - cur.executemany("DELETE FROM dependency WHERE package = ?;", - ((pkg,) for pkg in delpkgs)) cur.executemany("DELETE FROM package WHERE package = ?;", ((pkg,) for pkg in delpkgs)) + # Tables content, dependency and sharing will also be pruned + # due to ON DELETE CASCADE clauses. db.commit() if __name__ == "__main__": diff --git a/schema.sql b/schema.sql index 72357e3..c329465 100644 --- a/schema.sql +++ b/schema.sql @@ -1,9 +1,9 @@ CREATE TABLE package (package TEXT PRIMARY KEY, version TEXT, architecture TEXT, source TEXT); -CREATE TABLE content (package TEXT, filename TEXT, size INTEGER, function TEXT, hash TEXT, FOREIGN KEY (package) REFERENCES package(package)); -CREATE TABLE dependency (package TEXT, required TEXT, FOREIGN KEY (package) REFERENCES package(package)); +CREATE TABLE content (package TEXT, filename TEXT, size INTEGER, function TEXT, hash TEXT, FOREIGN KEY (package) REFERENCES package(package) ON DELETE CASCADE); +CREATE TABLE dependency (package TEXT, required TEXT, FOREIGN KEY (package) REFERENCES package(package) ON DELETE CASCADE); CREATE INDEX content_package_index ON content (package); CREATE INDEX content_hash_index ON content (hash); -CREATE TABLE sharing (package1 TEXT, package2 TEXT, func1 TEXT, func2 TEXT, files INTEGER, size INTEGER, FOREIGN KEY (package1) REFERENCES package(package), FOREIGN KEY (package2) REFERENCES package(package)); +CREATE TABLE sharing (package1 TEXT, package2 TEXT, func1 TEXT, func2 TEXT, files INTEGER, size INTEGER, FOREIGN KEY (package1) REFERENCES package(package) ON DELETE CASCADE, FOREIGN KEY (package2) REFERENCES package(package) ON DELETE CASCADE); CREATE INDEX sharing_insert_index ON sharing (package1, package2, func1, func2); CREATE INDEX sharing_package_index ON sharing (package1); -- cgit v1.2.3 From 5c0dcba3229b8c3e0faf42cf3e07cb82ee1369cd Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Sat, 9 Mar 2013 18:43:47 +0100 Subject: split content table to a hash table In the old content table (package, filename, size) would be the same for multiple hash functions. Now the schema represents that each file has precisely one size, but multiple hashes. --- importpkg.py | 13 +++++++++---- schema.sql | 6 ++++-- update_sharing.py | 4 ++-- webapp.py | 6 +++--- 4 files changed, 18 insertions(+), 11 deletions(-) (limited to 'schema.sql') diff --git a/importpkg.py b/importpkg.py index aae9a7f..5d6a58c 100755 --- a/importpkg.py +++ b/importpkg.py @@ -102,10 +102,12 @@ def get_hashes(tar): continue hasher = MultiHash(sha512_nontrivial(), gziphash(), imagehash()) hasher = hash_file(hasher, tar.extractfile(elem)) + hashes = {} for hashobj in hasher.hashes: hashvalue = hashobj.hexdigest() if hashvalue: - yield (elem.name, elem.size, hashobj.name, hashvalue) + hashes[hashobj.name] = hashvalue + yield (elem.name, elem.size, hashes) def process_package(db, filelike): cur = db.cursor() @@ -169,14 +171,17 @@ def process_package(db, filelike): continue if state != "control_file": raise ValueError("missing control file") - for name, size, function, hexhash in get_hashes(tf): + for name, size, hashes in get_hashes(tf): try: name = name.decode("utf8") except UnicodeDecodeError: print("warning: skipping filename with encoding error") continue # skip files with non-utf8 encoding for now - cur.execute("INSERT INTO content (package, filename, size, function, hash) VALUES (?, ?, ?, ?, ?);", - (package, name, size, function, hexhash)) + cur.execute("INSERT INTO content (package, filename, size) VALUES (?, ?, ?);", + (package, name, size)) + cid = cur.lastrowid + cur.executemany("INSERT INTO hash (cid, function, hash) VALUES (?, ?, ?);", + ((cid, func, hexhash) for func, hexhash in hashes.items())) db.commit() return raise ValueError("data.tar not found") diff --git a/schema.sql b/schema.sql index c329465..a67c807 100644 --- a/schema.sql +++ b/schema.sql @@ -1,8 +1,10 @@ CREATE TABLE package (package TEXT PRIMARY KEY, version TEXT, architecture TEXT, source TEXT); -CREATE TABLE content (package TEXT, filename TEXT, size INTEGER, function TEXT, hash TEXT, FOREIGN KEY (package) REFERENCES package(package) ON DELETE CASCADE); +CREATE TABLE content (id INTEGER PRIMARY KEY, package TEXT, filename TEXT, size INTEGER, FOREIGN KEY (package) REFERENCES package(package) ON DELETE CASCADE); +CREATE TABLE hash (cid INTEGER, function TEXT, hash TEXT, FOREIGN KEY (cid) REFERENCES content(id) ON DELETE CASCADE); CREATE TABLE dependency (package TEXT, required TEXT, FOREIGN KEY (package) REFERENCES package(package) ON DELETE CASCADE); CREATE INDEX content_package_index ON content (package); -CREATE INDEX content_hash_index ON content (hash); +CREATE INDEX hash_cid_index ON hash (cid); +CREATE INDEX hash_hash_index ON hash (hash); CREATE TABLE sharing (package1 TEXT, package2 TEXT, func1 TEXT, func2 TEXT, files INTEGER, size INTEGER, FOREIGN KEY (package1) REFERENCES package(package) ON DELETE CASCADE, FOREIGN KEY (package2) REFERENCES package(package) ON DELETE CASCADE); CREATE INDEX sharing_insert_index ON sharing (package1, package2, func1, func2); diff --git a/update_sharing.py b/update_sharing.py index 2ed532b..b45e40b 100755 --- a/update_sharing.py +++ b/update_sharing.py @@ -43,9 +43,9 @@ def main(): cur.execute("PRAGMA foreign_keys = ON;") cur.execute("DELETE FROM sharing;") readcur = db.cursor() - readcur.execute("SELECT hash FROM content GROUP BY hash HAVING count(*) > 1;") + readcur.execute("SELECT hash FROM hash GROUP BY hash HAVING count(*) > 1;") for hashvalue, in fetchiter(readcur): - cur.execute("SELECT package, filename, size, function FROM content WHERE hash = ?;", + cur.execute("SELECT content.package, content.filename, content.size, hash.function FROM hash JOIN content ON hash.cid = content.id WHERE hash = ?;", (hashvalue,)) rows = cur.fetchall() print("processing hash %s with %d entries" % (hashvalue, len(rows))) diff --git a/webapp.py b/webapp.py index f80b3da..1da987b 100755 --- a/webapp.py +++ b/webapp.py @@ -279,13 +279,13 @@ class Application(object): if package1 == package2: details1 = details2 = self.get_details(package1) - cur.execute("SELECT a.filename, a.size, a.function, b.filename, b.size, b.function, a.hash FROM content AS a JOIN content AS b ON a.hash = b.hash WHERE a.package = ? AND b.package = ? AND a.filename != b.filename ORDER BY a.size DESC, a.filename, b.filename;", + cur.execute("SELECT a.filename, a.size, ha.function, b.filename, b.size, hb.function, ha.hash FROM content AS a JOIN hash AS ha ON a.id = ha.cid JOIN hash AS hb ON ha.hash = hb.hash JOIN content AS b ON b.id = hb.cid WHERE a.package = ? AND b.package = ? AND a.filename != b.filename ORDER BY a.size DESC, a.filename, b.filename;", (package1, package1)) else: details1 = self.get_details(package1) details2 = self.get_details(package2) - cur.execute("SELECT a.filename, a.size, a.function, b.filename, b.size, b.function, a.hash FROM content AS a JOIN content AS b ON a.hash = b.hash WHERE a.package = ? AND b.package = ? ORDER BY a.size DESC, a.filename, b.filename;", + cur.execute("SELECT a.filename, a.size, ha.function, b.filename, b.size, hb.function, ha.hash FROM content AS a JOIN hash AS ha ON a.id = ha.cid JOIN hash AS hb ON ha.hash = hb.hash JOIN content AS b ON b.id = hb.cid WHERE a.package = ? AND b.package = ? ORDER BY a.size DESC, a.filename, b.filename;", (package1, package2)) shared = generate_shared(fetchiter(cur)) # The cursor will be in use until the template is fully rendered. @@ -297,7 +297,7 @@ class Application(object): def show_hash(self, function, hashvalue): cur = self.db.cursor() - cur.execute("SELECT package, filename, size, function FROM content WHERE hash = ?;", + cur.execute("SELECT content.package, content.filename, content.size, hash.function FROM content JOIN hash ON content.id = hash.cid WHERE hash = ?;", (hashvalue,)) entries = [dict(package=package, filename=filename, size=size, function=otherfunc) -- cgit v1.2.3