From 060979cd835d4e221d54b38ce57790befc056007 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Thu, 21 Feb 2013 08:42:44 +0100 Subject: rename test.py to importpkg.py --- test.py | 319 ---------------------------------------------------------------- 1 file changed, 319 deletions(-) delete mode 100755 test.py (limited to 'test.py') diff --git a/test.py b/test.py deleted file mode 100755 index a45720a..0000000 --- a/test.py +++ /dev/null @@ -1,319 +0,0 @@ -#!/usr/bin/python -""" -CREATE TABLE package (package TEXT PRIMARY KEY, version TEXT, architecture TEXT); -CREATE TABLE content (package TEXT, filename TEXT, size INTEGER, function TEXT, hash TEXT, FOREIGN KEY (package) REFERENCES package(package)); -CREATE TABLE dependency (package TEXT, required TEXT, FOREIGN KEY (package) REFERENCES package(package), FOREIGN KEY (required) REFERENCES package(package)); -CREATE INDEX content_package_index ON content (package); -CREATE INDEX content_hash_index ON content (hash); -""" - -import hashlib -import sqlite3 -import struct -import sys -import tarfile -import zlib - -from debian.debian_support import version_compare -from debian import deb822 -import lzma - -class ArReader(object): - global_magic = b"!\n" - file_magic = b"`\n" - - def __init__(self, fileobj): - self.fileobj = fileobj - self.remaining = None - self.padding = 0 - - def skip(self, length): - while length: - data = self.fileobj.read(min(4096, length)) - if not data: - raise ValueError("archive truncated") - length -= len(data) - - def read_magic(self): - data = self.fileobj.read(len(self.global_magic)) - if data != self.global_magic: - raise ValueError("ar global header not found") - self.remaining = 0 - - def read_entry(self): - self.skip_current_entry() - if self.padding: - if self.fileobj.read(1) != '\n': - raise ValueError("missing ar padding") - self.padding = 0 - file_header = self.fileobj.read(60) - if not file_header: - raise EOFError("end of archive found") - parts = struct.unpack("16s 12s 6s 6s 8s 10s 2s", file_header) - parts = [p.rstrip(" ") for p in parts] - if parts.pop() != self.file_magic: - raise ValueError("ar file header not found") - self.remaining = int(parts[5]) - self.padding = self.remaining % 2 - return parts[0] # name - - def skip_current_entry(self): - self.skip(self.remaining) - self.remaining = 0 - - def read(self, length=None): - if length is None: - length = self.remaining - else: - length = min(self.remaining, length) - data = self.fileobj.read(length) - self.remaining -= len(data) - return data - -class XzStream(object): - blocksize = 65536 - - def __init__(self, fileobj): - self.fileobj = fileobj - self.decomp = lzma.LZMADecompressor() - self.buff = b"" - - def read(self, length): - data = True - while True: - if len(self.buff) >= length: - ret = self.buff[:length] - self.buff = self.buff[length:] - return ret - elif not data: # read EOF in last iteration - ret = self.buff - self.buff = b"" - return ret - data = self.fileobj.read(self.blocksize) - if data: - self.buff += self.decomp.decompress(data) - else: - self.buff += self.decomp.flush() - -class MultiHash(object): - def __init__(self, *hashes): - self.hashes = hashes - - def update(self, data): - for hasher in self.hashes: - hasher.update(data) - -class HashBlacklist(object): - def __init__(self, hasher, blacklist=set()): - self.hasher = hasher - self.blacklist = blacklist - self.update = self.hasher.update - self.name = hasher.name - - def hexdigest(self): - digest = self.hasher.hexdigest() - if digest in self.blacklist: - return None - return digest - -class GzipDecompressor(object): - def __init__(self): - self.inbuffer = b"" - self.decompressor = None # zlib.decompressobj(-zlib.MAX_WBITS) - - def decompress(self, data): - if self.decompressor: - data = self.decompressor.decompress(data) - if not self.decompressor.unused_data: - return data - unused_data = self.decompressor.unused_data - self.decompressor = None - return data + self.decompress(unused_data) - self.inbuffer += data - skip = 10 - if len(self.inbuffer) < skip: - return b"" - if not self.inbuffer.startswith(b"\037\213\010"): - raise ValueError("gzip magic not found") - flag = ord(self.inbuffer[3]) - if flag & 4: - if len(self.inbuffer) < skip + 2: - return b"" - length, = struct.unpack(" 0: - return # already seen a newer package - - cur.execute("DELETE FROM package WHERE package = ?;", - (package,)) - cur.execute("DELETE FROM content WHERE package = ?;", - (package,)) - cur.execute("INSERT INTO package (package, version, architecture) VALUES (?, ?, ?);", - (package, version, architecture)) - depends = control.relations.get("depends", []) - depends = set(dep[0]["name"].encode("ascii") - for dep in depends if len(dep) == 1) - cur.execute("DELETE FROM dependency WHERE package = ?;", - (package,)) - cur.executemany("INSERT INTO dependency (package, required) VALUES (?, ?);", - ((package, dep) for dep in depends)) - break - continue - elif name == "data.tar.gz": - tf = tarfile.open(fileobj=af, mode="r|gz") - elif name == "data.tar.bz2": - tf = tarfile.open(fileobj=af, mode="r|bz2") - elif name == "data.tar.xz": - zf = XzStream(af) - tf = tarfile.open(fileobj=zf, mode="r|") - else: - continue - if state != "control_file": - raise ValueError("missing control file") - for name, size, function, hexhash in get_hashes(tf): - cur.execute("INSERT INTO content (package, filename, size, function, hash) VALUES (?, ?, ?, ?, ?);", - (package, name.decode("utf8"), size, function, hexhash)) - db.commit() - return - raise ValueError("data.tar not found") - -def main(): - db = sqlite3.connect("test.sqlite3") - process_package(db, sys.stdin) - -if __name__ == "__main__": - main() -- cgit v1.2.3