diff options
| -rwxr-xr-x | autoimport.py | 30 | ||||
| -rwxr-xr-x | readyaml.py | 60 |
2 files changed, 50 insertions, 40 deletions
diff --git a/autoimport.py b/autoimport.py index 481a3f8..c1e0da5 100755 --- a/autoimport.py +++ b/autoimport.py @@ -8,13 +8,15 @@ import io import multiprocessing import optparse import os -import sqlite3 import subprocess import urllib import concurrent.futures from debian import deb822 from debian.debian_support import version_compare +import sqlalchemy + +from dedup.utils import enable_sqlite_foreign_keys from readyaml import readyaml @@ -83,9 +85,8 @@ def main(): help="prune packages old packages") options, args = parser.parse_args() subprocess.check_call(["mkdir", "-p", "tmp"]) - db = sqlite3.connect("test.sqlite3") - cur = db.cursor() - cur.execute("PRAGMA foreign_keys = ON;") + db = sqlalchemy.create_engine("sqlite:///test.sqlite3") + enable_sqlite_foreign_keys(db) e = concurrent.futures.ThreadPoolExecutor(multiprocessing.cpu_count()) pkgs = {} for d in args: @@ -98,8 +99,9 @@ def main(): process_file(pkgs, d) print("reading database") - cur.execute("SELECT name, version FROM package;") - knownpkgs = dict((row[0], row[1]) for row in cur.fetchall()) + with db.begin() as conn: + cur = conn.execute("SELECT name, version FROM package;") + knownpkgs = dict((row[0], row[1]) for row in cur.fetchall()) distpkgs = set(pkgs.keys()) if options.new: for name in distpkgs: @@ -122,7 +124,8 @@ def main(): print("sqlimporting %s" % name) with open(inf) as inp: try: - readyaml(db, inp) + with db.begin() as conn: + readyaml(conn, inp) except Exception as exc: print("%s failed sql with exception %r" % (name, exc)) else: @@ -130,12 +133,13 @@ def main(): if options.prune: delpkgs = knownpkgs - distpkgs - print("clearing packages %s" % " ".join(delpkgs)) - cur.executemany("DELETE FROM package WHERE name = ?;", - ((pkg,) for pkg in delpkgs)) - # Tables content, dependency and sharing will also be pruned - # due to ON DELETE CASCADE clauses. - db.commit() + if delpkgs: + print("clearing packages %s" % " ".join(delpkgs)) + with db.begin() as conn: + conn.execute(sqlalchemy.text("DELETE FROM package WHERE name = :name;"), + [dict(name=pkg) for pkg in delpkgs]) + # Tables content, dependency and sharing will also be pruned + # due to ON DELETE CASCADE clauses. if __name__ == "__main__": main() diff --git a/readyaml.py b/readyaml.py index 21b1ca1..835967d 100755 --- a/readyaml.py +++ b/readyaml.py @@ -2,21 +2,20 @@ """This tool reads a yaml file as generated by importpkg.py on stdin and updates the database with the contents.""" -import sqlite3 import sys from debian.debian_support import version_compare +import sqlalchemy import yaml -def readyaml(db, stream): - cur = db.cursor() - cur.execute("PRAGMA foreign_keys = ON;") +from dedup.utils import enable_sqlite_foreign_keys + +def readyaml(conn, stream): gen = yaml.safe_load_all(stream) metadata = next(gen) package = metadata["package"] - cur.execute("SELECT id, version FROM package WHERE name = ?;", - (package,)) - row = cur.fetchone() + row = conn.execute(sqlalchemy.text("SELECT id, version FROM package WHERE name = :name;"), + name=package).fetchone() if row: pid, version = row if version_compare(version, metadata["version"]) > 0: @@ -24,37 +23,44 @@ def readyaml(db, stream): else: pid = None - cur.execute("BEGIN;") - cur.execute("SELECT name, id FROM function;") + cur = conn.execute("SELECT name, id FROM function;") funcmapping = dict(cur.fetchall()) if pid is not None: - cur.execute("DELETE FROM content WHERE pid = ?;", (pid,)) - cur.execute("DELETE FROM dependency WHERE pid = ?;", (pid,)) - cur.execute("UPDATE package SET version = ?, architecture = ?, source = ? WHERE id = ?;", - (metadata["version"], metadata["architecture"], metadata["source"], pid)) + conn.execute(sqlalchemy.text("DELETE FROM content WHERE pid = :pid;"), + pid=pid) + conn.execute(sqlalchemy.text("DELETE FROM dependency WHERE pid = :pid;"), + pid=pid) + conn.execute(sqlalchemy.text("UPDATE package SET version = :version, architecture = :architecture, source = :source WHERE id = :pid;"), + version=metadata["version"], + architecture=metadata["architecture"], + source=metadata["source"], pid=pid) else: - cur.execute("INSERT INTO package (name, version, architecture, source) VALUES (?, ?, ?, ?);", - (package, metadata["version"], metadata["architecture"], - metadata["source"])) - pid = cur.lastrowid - cur.executemany("INSERT INTO dependency (pid, required) VALUES (?, ?);", - ((pid, dep) for dep in metadata["depends"])) + pid = conn.execute(sqlalchemy.text("INSERT INTO package (name, version, architecture, source) VALUES (:name, :version, :architecture, :source);"), + name=package, version=metadata["version"], + architecture=metadata["architecture"], + source=metadata["source"]).lastrowid + if metadata["depends"]: + conn.execute(sqlalchemy.text("INSERT INTO dependency (pid, required) VALUES (:pid, :required);"), + [dict(pid=pid, required=dep) + for dep in metadata["depends"]]) for entry in gen: if entry == "commit": - db.commit() return - cur.execute("INSERT INTO content (pid, filename, size) VALUES (?, ?, ?);", - (pid, entry["name"], entry["size"])) + cur = conn.execute(sqlalchemy.text("INSERT INTO content (pid, filename, size) VALUES (:pid, :filename, :size);"), + pid=pid, filename=entry["name"], size=entry["size"]) cid = cur.lastrowid - cur.executemany("INSERT INTO hash (cid, fid, hash) VALUES (?, ?, ?);", - ((cid, funcmapping[func], hexhash) - for func, hexhash in entry["hashes"].items())) + if entry["hashes"]: + conn.execute(sqlalchemy.text("INSERT INTO hash (cid, fid, hash) VALUES (:cid, :fid, :hash);"), + [dict(cid=cid, fid=funcmapping[func], hash=hexhash) + for func, hexhash in entry["hashes"].items()]) raise ValueError("missing commit block") def main(): - db = sqlite3.connect("test.sqlite3") - readyaml(db, sys.stdin) + db = sqlalchemy.create_engine("sqlite:///test.sqlite3") + enable_sqlite_foreign_keys(db) + with db.begin() as conn: + readyaml(conn, sys.stdin) if __name__ == "__main__": main() |