#!/usr/bin/python
"""This scrip takes a directory or a http base url to a mirror and imports all
packages contained. It has rather strong assumptions on the working directory.
"""

import argparse
import errno
import multiprocessing
import os
import sqlite3
import subprocess
import sys
import tempfile
try:
    from urllib.parse import unquote
except ImportError:
    from urllib import unquote
try:
    from urllib.request import urlopen
except ImportError:
    from urllib import urlopen

import concurrent.futures
from debian import deb822
from debian.debian_support import version_compare

from dedup.compression import decompress

from readyaml import readyaml

def process_http(pkgs, url, addhash=True):
    pkglist = urlopen(url + "/dists/sid/main/binary-amd64/Packages.gz")
    pkglist = decompress(pkglist, ".gz")
    pkglist = deb822.Packages.iter_paragraphs(pkglist)
    for pkg in pkglist:
        name = pkg["Package"]
        if name in pkgs and \
                version_compare(pkgs[name]["version"], pkg["Version"]) > 0:
            continue
        inst = dict(version=pkg["Version"],
                    filename="%s/%s" % (url, pkg["Filename"]))
        if addhash:
            inst["sharing"] = pkg["SHA256"]
        pkgs[name] = inst

def process_file(pkgs, filename):
    base = os.path.basename(filename)
    if not base.endswith(".deb"):
        raise ValueError("filename does not end in .deb")
    parts = base.split("_")
    if len(parts) != 3:
        raise ValueError("filename not in form name_version_arch.deb")
    name, version, _ = parts
    version = unquote(version)
    if name in pkgs and version_compare(pkgs[name]["version"], version) > 0:
        return
    pkgs[name] = dict(version=version, filename=filename)

def process_dir(pkgs, d):
    for entry in os.listdir(d):
        try:
            process_file(pkgs, os.path.join(d, entry))
        except ValueError:
            pass

def process_pkg(name, pkgdict, outpath):
    filename = pkgdict["filename"]
    print("importing %s" % filename)
    importcmd = [sys.executable, "importpkg.py"]
    if "sha256hash" in pkgdict:
        importcmd.extend(["-H", pkgdict["sha256hash"]])
    if filename.startswith(("http://", "https://", "ftp://", "file://")):
        importcmd.append(filename)
        with open(outpath, "w") as outp:
            subprocess.check_call(importcmd, stdout=outp, close_fds=True)
    else:
        with open(filename) as inp:
            with open(outpath, "w") as outp:
                subprocess.check_call(importcmd, stdin=inp, stdout=outp,
                                      close_fds=True)
    print("preprocessed %s" % name)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-n", "--new", action="store_true",
                        help="avoid reimporting same versions")
    parser.add_argument("-p", "--prune", action="store_true",
                        help="prune packages old packages")
    parser.add_argument("-d", "--database", action="store",
                        default="test.sqlite3",
                        help="path to the sqlite3 database file")
    parser.add_argument("--noverify", action="store_true",
                        help="do not verify binary package hashes")
    parser.add_argument("files", nargs='+',
                        help="files or directories or repository urls")
    args = parser.parse_args()
    tmpdir = tempfile.mkdtemp(prefix="debian-dedup")
    db = sqlite3.connect(args.database)
    cur = db.cursor()
    cur.execute("PRAGMA foreign_keys = ON;")
    e = concurrent.futures.ThreadPoolExecutor(multiprocessing.cpu_count())
    pkgs = {}
    for d in args.files:
        print("processing %s" % d)
        if d.startswith(("http://", "https://", "ftp://", "file://")):
            process_http(pkgs, d, not args.noverify)
        elif os.path.isdir(d):
            process_dir(pkgs, d)
        else:
            process_file(pkgs, d)

    print("reading database")
    cur.execute("SELECT name, version FROM package;")
    knownpkgs = dict((row[0], row[1]) for row in cur.fetchall())
    distpkgs = set(pkgs.keys())
    if args.new:
        for name in distpkgs:
            if name in knownpkgs and version_compare(pkgs[name]["version"],
                    knownpkgs[name]) <= 0:
                del pkgs[name]
    knownpkgs = set(knownpkgs)

    with e:
        fs = {}
        for name, pkg in pkgs.items():
            outpath = os.path.join(tmpdir, name)
            fs[e.submit(process_pkg, name, pkg, outpath)] = name

        for f in concurrent.futures.as_completed(fs.keys()):
            name = fs[f]
            if f.exception():
                print("%s failed to import: %r" % (name, f.exception()))
                continue
            inf = os.path.join(tmpdir, name)
            print("sqlimporting %s" % name)
            with open(inf) as inp:
                try:
                    readyaml(db, inp)
                except Exception as exc:
                    print("%s failed sql with exception %r" % (name, exc))
                else:
                    os.unlink(inf)

    if args.prune:
        delpkgs = knownpkgs - distpkgs
        print("clearing packages %s" % " ".join(delpkgs))
        cur.executemany("DELETE FROM package WHERE name = ?;",
                        ((pkg,) for pkg in delpkgs))
        # Tables content, dependency and sharing will also be pruned
        # due to ON DELETE CASCADE clauses.
        db.commit()
    try:
        os.rmdir(tmpdir)
    except OSError as err:
        if err.errno != errno.ENOTEMPTY:
            raise
        print("keeping temporary directory %s due to failed packages %s" %
              (tmpdir, " ".join(os.listdir(tmpdir))))

if __name__ == "__main__":
    main()