diff options
| -rw-r--r-- | README | 2 | ||||
| -rwxr-xr-x | autoimport.py | 34 | ||||
| -rw-r--r-- | dedup/utils.py | 18 | ||||
| -rwxr-xr-x | readyaml.py | 64 | ||||
| -rwxr-xr-x | update_sharing.py | 85 | ||||
| -rwxr-xr-x | webapp.py | 127 |
6 files changed, 175 insertions, 155 deletions
@@ -1,7 +1,7 @@ Required packages ----------------- - aptitude install python python-debian python-lzma curl python-jinja2 python-werkzeug sqlite3 python-imaging python-yaml python-concurrent.futures python-pkg-resources + aptitude install python python-debian python-lzma curl python-jinja2 python-werkzeug sqlite3 python-imaging python-yaml python-concurrent.futures python-pkg-resources python-sqlalchemy Create a database ----------------- diff --git a/autoimport.py b/autoimport.py index d44c012..8c61a18 100755 --- a/autoimport.py +++ b/autoimport.py @@ -9,7 +9,6 @@ import io import multiprocessing import optparse import os -import sqlite3 import subprocess import tempfile import urllib @@ -17,6 +16,9 @@ import urllib import concurrent.futures from debian import deb822 from debian.debian_support import version_compare +import sqlalchemy + +from dedup.utils import enable_sqlite_foreign_keys from readyaml import readyaml @@ -84,13 +86,12 @@ def main(): parser.add_option("-p", "--prune", action="store_true", help="prune packages old packages") parser.add_option("-d", "--database", action="store", - default="test.sqlite3", - help="path to the sqlite3 database file") + default="sqlite:///test.sqlite3", + help="location of the database") options, args = parser.parse_args() tmpdir = tempfile.mkdtemp(prefix=b"debian-dedup") - db = sqlite3.connect(options.database) - cur = db.cursor() - cur.execute("PRAGMA foreign_keys = ON;") + db = sqlalchemy.create_engine("sqlite:///test.sqlite3") + enable_sqlite_foreign_keys(db) e = concurrent.futures.ThreadPoolExecutor(multiprocessing.cpu_count()) pkgs = {} for d in args: @@ -103,8 +104,9 @@ def main(): process_file(pkgs, d) print("reading database") - cur.execute("SELECT name, version FROM package;") - knownpkgs = dict((row[0], row[1]) for row in cur.fetchall()) + with db.begin() as conn: + cur = conn.execute("SELECT name, version FROM package;") + knownpkgs = dict((row[0], row[1]) for row in cur.fetchall()) distpkgs = set(pkgs.keys()) if options.new: for name in distpkgs: @@ -128,7 +130,8 @@ def main(): print("sqlimporting %s" % name) with open(inf) as inp: try: - readyaml(db, inp) + with db.begin() as conn: + readyaml(conn, inp) except Exception as exc: print("%s failed sql with exception %r" % (name, exc)) else: @@ -136,12 +139,13 @@ def main(): if options.prune: delpkgs = knownpkgs - distpkgs - print("clearing packages %s" % " ".join(delpkgs)) - cur.executemany("DELETE FROM package WHERE name = ?;", - ((pkg,) for pkg in delpkgs)) - # Tables content, dependency and sharing will also be pruned - # due to ON DELETE CASCADE clauses. - db.commit() + if delpkgs: + print("clearing packages %s" % " ".join(delpkgs)) + with db.begin() as conn: + conn.execute(sqlalchemy.text("DELETE FROM package WHERE name = :name;"), + [dict(name=pkg) for pkg in delpkgs]) + # Tables content, dependency and sharing will also be pruned + # due to ON DELETE CASCADE clauses. try: os.rmdir(tmpdir) except OSError as err: diff --git a/dedup/utils.py b/dedup/utils.py index 6fb233b..fd30378 100644 --- a/dedup/utils.py +++ b/dedup/utils.py @@ -1,12 +1,20 @@ from debian.debian_support import version_compare +import sqlalchemy.event def fetchiter(cursor): - rows = cursor.fetchmany() + rows = cursor.fetchmany(1024) while rows: for row in rows: yield row - rows = cursor.fetchmany() + rows = cursor.fetchmany(1024) -def sql_add_version_compare(db): - db.create_collation("debian_version", version_compare) - db.create_function("debian_version_compare", 2, version_compare) +def enable_sqlite_foreign_keys(engine): + @sqlalchemy.event.listens_for(engine, "connect") + def pragma_foreign_keys(connection, _): + connection.execute("PRAGMA foreign_keys=ON;") + +def sqlite_add_version_compare(engine): + @sqlalchemy.event.listens_for(engine, "connect") + def add_version_compare(connection, _): + connection.create_collation("debian_version", version_compare) + connection.create_function("debian_version_compare", 2, version_compare) diff --git a/readyaml.py b/readyaml.py index 2ef9a3b..15cfcb3 100755 --- a/readyaml.py +++ b/readyaml.py @@ -3,21 +3,20 @@ updates the database with the contents.""" import optparse -import sqlite3 import sys from debian.debian_support import version_compare +import sqlalchemy import yaml -def readyaml(db, stream): - cur = db.cursor() - cur.execute("PRAGMA foreign_keys = ON;") +from dedup.utils import enable_sqlite_foreign_keys + +def readyaml(conn, stream): gen = yaml.safe_load_all(stream) metadata = next(gen) package = metadata["package"] - cur.execute("SELECT id, version FROM package WHERE name = ?;", - (package,)) - row = cur.fetchone() + row = conn.execute(sqlalchemy.text("SELECT id, version FROM package WHERE name = :name;"), + name=package).fetchone() if row: pid, version = row if version_compare(version, metadata["version"]) > 0: @@ -25,42 +24,49 @@ def readyaml(db, stream): else: pid = None - cur.execute("BEGIN;") - cur.execute("SELECT name, id FROM function;") + cur = conn.execute("SELECT name, id FROM function;") funcmapping = dict(cur.fetchall()) if pid is not None: - cur.execute("DELETE FROM content WHERE pid = ?;", (pid,)) - cur.execute("DELETE FROM dependency WHERE pid = ?;", (pid,)) - cur.execute("UPDATE package SET version = ?, architecture = ?, source = ? WHERE id = ?;", - (metadata["version"], metadata["architecture"], metadata["source"], pid)) + conn.execute(sqlalchemy.text("DELETE FROM content WHERE pid = :pid;"), + pid=pid) + conn.execute(sqlalchemy.text("DELETE FROM dependency WHERE pid = :pid;"), + pid=pid) + conn.execute(sqlalchemy.text("UPDATE package SET version = :version, architecture = :architecture, source = :source WHERE id = :pid;"), + version=metadata["version"], + architecture=metadata["architecture"], + source=metadata["source"], pid=pid) else: - cur.execute("INSERT INTO package (name, version, architecture, source) VALUES (?, ?, ?, ?);", - (package, metadata["version"], metadata["architecture"], - metadata["source"])) - pid = cur.lastrowid - cur.executemany("INSERT INTO dependency (pid, required) VALUES (?, ?);", - ((pid, dep) for dep in metadata["depends"])) + pid = conn.execute(sqlalchemy.text("INSERT INTO package (name, version, architecture, source) VALUES (:name, :version, :architecture, :source);"), + name=package, version=metadata["version"], + architecture=metadata["architecture"], + source=metadata["source"]).lastrowid + if metadata["depends"]: + conn.execute(sqlalchemy.text("INSERT INTO dependency (pid, required) VALUES (:pid, :required);"), + [dict(pid=pid, required=dep) + for dep in metadata["depends"]]) for entry in gen: if entry == "commit": - db.commit() return - cur.execute("INSERT INTO content (pid, filename, size) VALUES (?, ?, ?);", - (pid, entry["name"], entry["size"])) + cur = conn.execute(sqlalchemy.text("INSERT INTO content (pid, filename, size) VALUES (:pid, :filename, :size);"), + pid=pid, filename=entry["name"], size=entry["size"]) cid = cur.lastrowid - cur.executemany("INSERT INTO hash (cid, fid, hash) VALUES (?, ?, ?);", - ((cid, funcmapping[func], hexhash) - for func, hexhash in entry["hashes"].items())) + if entry["hashes"]: + conn.execute(sqlalchemy.text("INSERT INTO hash (cid, fid, hash) VALUES (:cid, :fid, :hash);"), + [dict(cid=cid, fid=funcmapping[func], hash=hexhash) + for func, hexhash in entry["hashes"].items()]) raise ValueError("missing commit block") def main(): parser = optparse.OptionParser() parser.add_option("-d", "--database", action="store", - default="test.sqlite3", - help="path to the sqlite3 database file") + default="sqlite:///test.sqlite3", + help="location of the database") options, args = parser.parse_args() - db = sqlite3.connect(options.database) - readyaml(db, sys.stdin) + db = sqlalchemy.create_engine(options.database) + enable_sqlite_foreign_keys(db) + with db.begin() as conn: + readyaml(conn, sys.stdin) if __name__ == "__main__": main() diff --git a/update_sharing.py b/update_sharing.py index ca6890b..450bfc7 100755 --- a/update_sharing.py +++ b/update_sharing.py @@ -1,17 +1,20 @@ #!/usr/bin/python import optparse -import sqlite3 -from dedup.utils import fetchiter +import sqlalchemy -def add_values(cursor, insert_key, files, size): - cursor.execute("UPDATE sharing SET files = files + ?, size = size + ? WHERE pid1 = ? AND pid2 = ? AND fid1 = ? AND fid2 = ?;", - (files, size) + insert_key) - if cursor.rowcount > 0: +from dedup.utils import fetchiter, enable_sqlite_foreign_keys + +def add_values(conn, insert_key, files, size): + params = dict(files=files, size=size, pid1=insert_key[0], + pid2=insert_key[1], fid1=insert_key[2], fid2=insert_key[3]) + rows = conn.execute(sqlalchemy.text("UPDATE sharing SET files = files + :files, size = size + :size WHERE pid1 = :pid1 AND pid2 = :pid2 AND fid1 = :fid1 AND fid2 = :fid2;"), + **params) + if rows.rowcount > 0: return - cursor.execute("INSERT INTO sharing (pid1, pid2, fid1, fid2, files, size) VALUES (?, ?, ?, ?, ?, ?);", - insert_key + (files, size)) + conn.execute(sqlalchemy.text("INSERT INTO sharing (pid1, pid2, fid1, fid2, files, size) VALUES (:pid1, :pid2, :fid1, :fid2, :files, :size);"), + **params) def compute_pkgdict(rows): pkgdict = dict() @@ -20,7 +23,7 @@ def compute_pkgdict(rows): funcdict.setdefault(fid, []).append((size, filename)) return pkgdict -def process_pkgdict(cursor, pkgdict): +def process_pkgdict(conn, pkgdict): for pid1, funcdict1 in pkgdict.items(): for fid1, files in funcdict1.items(): numfiles = len(files) @@ -36,40 +39,44 @@ def process_pkgdict(cursor, pkgdict): pkgsize = size for fid2 in funcdict2.keys(): insert_key = (pid1, pid2, fid1, fid2) - add_values(cursor, insert_key, pkgnumfiles, pkgsize) + add_values(conn, insert_key, pkgnumfiles, pkgsize) def main(db): - cur = db.cursor() - cur.execute("PRAGMA foreign_keys = ON;") - cur.execute("DELETE FROM sharing;") - cur.execute("DELETE FROM duplicate;") - cur.execute("DELETE FROM issue;") - readcur = db.cursor() - readcur.execute("SELECT hash FROM hash GROUP BY hash HAVING count(*) > 1;") - for hashvalue, in fetchiter(readcur): - cur.execute("SELECT function.eqclass, content.pid, content.id, content.filename, content.size, hash.fid FROM hash JOIN content ON hash.cid = content.id JOIN function ON hash.fid = function.id AND function.eqclass IS NOT NULL WHERE hash = ?;", - (hashvalue,)) - rowdict = dict() - for row in cur.fetchall(): - rowdict.setdefault(row[0], []).append(row[1:]) - for eqclass, rows in rowdict.items(): - if len(rows) < 2: - print("skipping hash %s class %d with too few entries" % (hashvalue, eqclass)) - continue - print("processing hash %s class %d with %d entries" % (hashvalue, eqclass, len(rows))) - pkgdict = compute_pkgdict(rows) - cur.executemany("INSERT OR IGNORE INTO duplicate (cid) VALUES (?);", - [(row[1],) for row in rows]) - process_pkgdict(cur, pkgdict) - cur.execute("INSERT INTO issue (cid, issue) SELECT content.id, 'file named something.gz is not a valid gzip file' FROM content WHERE content.filename LIKE '%.gz' AND NOT EXISTS (SELECT 1 FROM hash JOIN function ON hash.fid = function.id WHERE hash.cid = content.id AND function.name = 'gzip_sha512');") - cur.execute("INSERT INTO issue (cid, issue) SELECT content.id, 'png image not named something.png' FROM content JOIN hash ON content.id = hash.cid JOIN function ON hash.fid = function.id WHERE function.name = 'png_sha512' AND lower(filename) NOT LIKE '%.png';") - cur.execute("INSERT INTO issue (cid, issue) SELECT content.id, 'gif image not named something.gif' FROM content JOIN hash ON content.id = hash.cid JOIN function ON hash.fid = function.id WHERE function.name = 'gif_sha512' AND lower(filename) NOT LIKE '%.gif';") - db.commit() + with db.begin() as conn: + conn.execute("DELETE FROM sharing;") + conn.execute("DELETE FROM duplicate;") + conn.execute("DELETE FROM issue;") + readcur = conn.execute("SELECT hash FROM hash GROUP BY hash HAVING count(*) > 1;") + for hashvalue, in fetchiter(readcur): + rows = conn.execute(sqlalchemy.text("SELECT function.eqclass, content.pid, content.id, content.filename, content.size, hash.fid FROM hash JOIN content ON hash.cid = content.id JOIN function ON hash.fid = function.id AND function.eqclass IS NOT NULL WHERE hash = :hashvalue;"), + hashvalue=hashvalue).fetchall() + rowdict = dict() + for row in rows: + rowdict.setdefault(row[0], []).append(row[1:]) + for eqclass, rows in rowdict.items(): + if len(rows) < 2: + print("skipping hash %s class %d with too few entries" % (hashvalue, eqclass)) + continue + print("processing hash %s class %d with %d entries" % (hashvalue, eqclass, len(rows))) + pkgdict = compute_pkgdict(rows) + for row in rows: + cid = row[1] + already = conn.scalar(sqlalchemy.text("SELECT cid FROM duplicate WHERE cid = :cid;"), + cid=cid) + if not already: + conn.execute(sqlalchemy.text("INSERT INTO duplicate (cid) VALUES (:cid);"), + cid=cid) + process_pkgdict(conn, pkgdict) + conn.execute("INSERT INTO issue (cid, issue) SELECT content.id, 'file named something.gz is not a valid gzip file' FROM content WHERE content.filename LIKE '%.gz' AND NOT EXISTS (SELECT 1 FROM hash JOIN function ON hash.fid = function.id WHERE hash.cid = content.id AND function.name = 'gzip_sha512');") + conn.execute("INSERT INTO issue (cid, issue) SELECT content.id, 'png image not named something.png' FROM content JOIN hash ON content.id = hash.cid JOIN function ON hash.fid = function.id WHERE function.name = 'png_sha512' AND lower(filename) NOT LIKE '%.png';") + conn.execute("INSERT INTO issue (cid, issue) SELECT content.id, 'gif image not named something.gif' FROM content JOIN hash ON content.id = hash.cid JOIN function ON hash.fid = function.id WHERE function.name = 'gif_sha512' AND lower(filename) NOT LIKE '%.gif';") if __name__ == "__main__": parser = optparse.OptionParser() parser.add_option("-d", "--database", action="store", - default="test.sqlite3", - help="path to the sqlite3 database file") + default="sqlite:///test.sqlite3", + help="location of the database") options, args = parser.parse_args() - main(sqlite3.connect(options.database)) + db = sqlalchemy.create_engine(options.database) + enable_sqlite_foreign_keys(db) + main(db) @@ -1,12 +1,11 @@ #!/usr/bin/python -import contextlib import datetime import optparse -import sqlite3 from wsgiref.simple_server import make_server import jinja2 +import sqlalchemy from werkzeug.exceptions import HTTPException, NotFound from werkzeug.routing import Map, Rule, RequestRedirect from werkzeug.wrappers import Request, Response @@ -99,36 +98,35 @@ class Application(object): return e def get_details(self, package): - with contextlib.closing(self.db.cursor()) as cur: - cur.execute("SELECT id, version, architecture FROM package WHERE name = ?;", - (package,)) - row = cur.fetchone() + with self.db.begin() as conn: + row = conn.execute(sqlalchemy.text("SELECT id, version, architecture FROM package WHERE name = :name;"), + name=package).fetchone() if not row: raise NotFound() pid, version, architecture = row - details = dict(pid=pid, - package=package, - version=version, - architecture=architecture) - cur.execute("SELECT count(filename), sum(size) FROM content WHERE pid = ?;", - (pid,)) - num_files, total_size = cur.fetchone() + row = conn.execute(sqlalchemy.text("SELECT count(filename), sum(size) FROM content WHERE pid = :pid;"), + pid=pid).fetchone() + num_files, total_size = row if total_size is None: total_size = 0 - details.update(dict(num_files=num_files, total_size=total_size)) - return details + return dict(pid=pid, + package=package, + version=version, + architecture=architecture, + num_files=num_files, + total_size=total_size) def get_dependencies(self, pid): - with contextlib.closing(self.db.cursor()) as cur: - cur.execute("SELECT required FROM dependency WHERE pid = ?;", - (pid,)) + with self.db.begin() as conn: + cur = conn.execute(sqlalchemy.text("SELECT required FROM dependency WHERE pid = :pid;"), + pid=pid) return set(row[0] for row in fetchiter(cur)) def cached_sharedstats(self, pid): sharedstats = {} - with contextlib.closing(self.db.cursor()) as cur: - cur.execute("SELECT pid2, package.name, f1.name, f2.name, files, size FROM sharing JOIN package ON sharing.pid2 = package.id JOIN function AS f1 ON sharing.fid1 = f1.id JOIN function AS f2 ON sharing.fid2 = f2.id WHERE pid1 = ? AND f1.eqclass = f2.eqclass;", - (pid,)) + with self.db.begin() as conn: + cur = conn.execute(sqlalchemy.text("SELECT pid2, package.name, f1.name, f2.name, files, size FROM sharing JOIN package ON sharing.pid2 = package.id JOIN function AS f1 ON sharing.fid1 = f1.id JOIN function AS f2 ON sharing.fid2 = f2.id WHERE pid1 = :pid AND f1.eqclass = f2.eqclass;"), + pid=pid) for pid2, package2, func1, func2, files, size in fetchiter(cur): curstats = sharedstats.setdefault( function_combination(func1, func2), list()) @@ -143,11 +141,10 @@ class Application(object): params["dependencies"] = self.get_dependencies(params["pid"]) params["shared"] = self.cached_sharedstats(params["pid"]) params["urlroot"] = ".." - cur = self.db.cursor() - cur.execute("SELECT content.filename, issue.issue FROM content JOIN issue ON content.id = issue.cid WHERE content.pid = ?;", - (params["pid"],)) - params["issues"] = dict(cur.fetchall()) - cur.close() + with self.db.begin() as conn: + cur = conn.execute(sqlalchemy.text("SELECT content.filename, issue.issue FROM content JOIN issue ON content.id = issue.cid WHERE content.pid = :pid;"), + pid=params["pid"]) + params["issues"] = dict(cur.fetchall()) return html_response(package_template.render(params)) def compute_comparison(self, pid1, pid2): @@ -160,35 +157,32 @@ class Application(object): * matches: A mapping from filenames in package 2 (pid2) to a mapping from hash function pairs to hash values. """ - cur = self.db.cursor() - cur.execute("SELECT content.id, content.filename, content.size, hash.hash FROM content JOIN hash ON content.id = hash.cid JOIN duplicate ON content.id = duplicate.cid JOIN function ON hash.fid = function.id WHERE pid = ? AND function.name = 'sha512' ORDER BY size DESC;", - (pid1,)) - cursize = -1 - files = dict() - minmatch = 2 if pid1 == pid2 else 1 - cur2 = self.db.cursor() - for cid, filename, size, hashvalue in fetchiter(cur): - if cursize != size: - for entry in files.values(): - if len(entry["matches"]) >= minmatch: - yield entry - files.clear() - cursize = size - - if hashvalue in files: - files[hashvalue]["filenames"].add(filename) - continue - - entry = dict(filenames=set((filename,)), size=size, matches={}) - files[hashvalue] = entry - - cur2.execute("SELECT fa.name, ha.hash, fb.name, filename FROM hash AS ha JOIN hash AS hb ON ha.hash = hb.hash JOIN content ON hb.cid = content.id JOIN function AS fa ON ha.fid = fa.id JOIN function AS fb ON hb.fid = fb.id WHERE ha.cid = ? AND pid = ? AND fa.eqclass = fb.eqclass;", - (cid, pid2)) - for func1, hashvalue, func2, filename in fetchiter(cur2): - entry["matches"].setdefault(filename, {})[func1, func2] = \ - hashvalue - cur2.close() - cur.close() + with self.db.begin() as conn: + cur = conn.execute(sqlalchemy.text("SELECT content.id, content.filename, content.size, hash.hash FROM content JOIN hash ON content.id = hash.cid JOIN duplicate ON content.id = duplicate.cid JOIN function ON hash.fid = function.id WHERE pid = :pid AND function.name = 'sha512' ORDER BY size DESC;"), + pid=pid1) + cursize = -1 + files = dict() + minmatch = 2 if pid1 == pid2 else 1 + for cid, filename, size, hashvalue in fetchiter(cur): + if cursize != size: + for entry in files.values(): + if len(entry["matches"]) >= minmatch: + yield entry + files.clear() + cursize = size + + if hashvalue in files: + files[hashvalue]["filenames"].add(filename) + continue + + entry = dict(filenames=set((filename,)), size=size, matches={}) + files[hashvalue] = entry + + cur2 = conn.execute(sqlalchemy.text("SELECT fa.name, ha.hash, fb.name, filename FROM hash AS ha JOIN hash AS hb ON ha.hash = hb.hash JOIN content ON hb.cid = content.id JOIN function AS fa ON ha.fid = fa.id JOIN function AS fb ON hb.fid = fb.id WHERE ha.cid = :cid AND pid = :pid AND fa.eqclass = fb.eqclass;"), + cid=cid, pid=pid2) + for func1, hashvalue, func2, filename in fetchiter(cur2): + entry["matches"].setdefault(filename, {})[func1, func2] = \ + hashvalue for entry in files.values(): if len(entry["matches"]) >= minmatch: @@ -208,9 +202,9 @@ class Application(object): return html_response(detail_template.stream(params)) def show_hash(self, function, hashvalue): - with contextlib.closing(self.db.cursor()) as cur: - cur.execute("SELECT package.name, content.filename, content.size, f2.name FROM hash JOIN content ON hash.cid = content.id JOIN package ON content.pid = package.id JOIN function AS f2 ON hash.fid = f2.id JOIN function AS f1 ON f2.eqclass = f1.eqclass WHERE f1.name = ? AND hash = ?;", - (function, hashvalue,)) + with self.db.begin() as conn: + cur = conn.execute(sqlalchemy.text("SELECT package.name, content.filename, content.size, f2.name FROM hash JOIN content ON hash.cid = content.id JOIN package ON content.pid = package.id JOIN function AS f2 ON hash.fid = f2.id JOIN function AS f1 ON f2.eqclass = f1.eqclass WHERE f1.name = :function AND hash = :hashvalue;"), + function=function, hashvalue=hashvalue) entries = [dict(package=package, filename=filename, size=size, function=otherfunc) for package, filename, size, otherfunc in fetchiter(cur)] @@ -221,14 +215,14 @@ class Application(object): return html_response(hash_template.render(params)) def show_source(self, package): - with contextlib.closing(self.db.cursor()) as cur: - cur.execute("SELECT name FROM package WHERE source = ?;", - (package,)) + with self.db.begin() as conn: + cur = conn.execute(sqlalchemy.text("SELECT name FROM package WHERE source = :source;"), + source=package) binpkgs = dict.fromkeys(pkg for pkg, in fetchiter(cur)) if not binpkgs: raise NotFound - cur.execute("SELECT p1.name, p2.name, f1.name, f2.name, sharing.files, sharing.size FROM sharing JOIN package AS p1 ON sharing.pid1 = p1.id JOIN package AS p2 ON sharing.pid2 = p2.id JOIN function AS f1 ON sharing.fid1 = f1.id JOIN function AS f2 ON sharing.fid2 = f2.id WHERE p1.source = ?;", - (package,)) + cur = conn.execute(sqlalchemy.text("SELECT p1.name, p2.name, f1.name, f2.name, sharing.files, sharing.size FROM sharing JOIN package AS p1 ON sharing.pid1 = p1.id JOIN package AS p2 ON sharing.pid2 = p2.id JOIN function AS f1 ON sharing.fid1 = f1.id JOIN function AS f2 ON sharing.fid2 = f2.id WHERE p1.source = :source;"), + source=package) for binary, otherbin, func1, func2, files, size in fetchiter(cur): entry = dict(package=otherbin, funccomb=function_combination(func1, func2), @@ -242,10 +236,11 @@ class Application(object): def main(): parser = optparse.OptionParser() parser.add_option("-d", "--database", action="store", - default="test.sqlite3", - help="path to the sqlite3 database file") + default="sqlite:///test.sqlite3", + help="location of the database") options, args = parser.parse_args() - app = Application(sqlite3.connect(options.database)) + db = sqlalchemy.create_engine(options.database) + app = Application(db) app = SharedDataMiddleware(app, {"/static": ("dedup", "static")}) make_server("0.0.0.0", 8800, app).serve_forever() |