path: root/webapp.py

#!/usr/bin/python3

import argparse
import contextlib
import datetime
import io
import sqlite3
from wsgiref.simple_server import make_server

import jinja2
from werkzeug.exceptions import HTTPException, NotFound
from werkzeug.routing import Map, Rule
from werkzeug.utils import redirect
from werkzeug.wrappers import Request, Response
try:
    from werkzeug.middleware.shared_data import SharedDataMiddleware
except ImportError:
    from werkzeug.wsgi import SharedDataMiddleware

from dedup.utils import fetchiter

jinjaenv = jinja2.Environment(loader=jinja2.PackageLoader("dedup", "templates"))

def format_size(size):
    sizef = float(size)
    fmt = "%d B"
    if sizef >= 1024:
        sizef /= 1024
        fmt = "%.1f KB"
    if sizef >= 1024:
        sizef /= 1024
        fmt = "%.1f MB"
    if sizef >= 1024:
        sizef /= 1024
        fmt = "%.1f GB"
    return fmt % sizef

def function_combination(function1, function2):
    if function1 == function2:
        return function1
    return "%s -> %s" % (function1, function2)

# Workaround for jinja bug #59 (broken filesizeformat)
jinjaenv.filters["filesizeformat"] = format_size

base_template = jinjaenv.get_template("base.html")
package_template = jinjaenv.get_template("binary.html")
detail_template = jinjaenv.get_template("compare.html")
hash_template = jinjaenv.get_template("hash.html")
index_template = jinjaenv.get_template("index.html")
source_template = jinjaenv.get_template("source.html")

def encode_and_buffer(stream):
    stream.enable_buffering(16)
    buff = io.BytesIO()
    for elem in stream:
        buff.write(elem.encode("utf8"))
        if buff.tell() >= 2048:
            yield buff.getvalue()
            buff = io.BytesIO()
    if buff.tell() > 0:
        yield buff.getvalue()

def html_response(unicode_iterator, max_age=24 * 60 * 60):
    resp = Response(encode_and_buffer(unicode_iterator), mimetype="text/html")
    resp.cache_control.max_age = max_age
    resp.expires = datetime.datetime.now() + datetime.timedelta(seconds=max_age)
    return resp

class InternalRedirect(Exception):
    def __init__(self, target, code=301):
        Exception.__init__(self)
        self.target = target
        self.code = code

class Application:
    def __init__(self, db):
        self.db = db
        self.routingmap = Map([
            Rule("/", methods=("GET",), endpoint="index"),
            Rule("/binary/<package>", methods=("GET",), endpoint="package"),
            Rule("/compare/<package1>/<package2>", methods=("GET",), endpoint="detail"),
            Rule("/hash/<function>/<hashvalue>", methods=("GET",), endpoint="hash"),
            Rule("/source/<package>", methods=("GET",), endpoint="source"),
        ])

    def cursor(self):
        return contextlib.closing(self.db.cursor())

    @Request.application
    def __call__(self, request):
        mapadapter = self.routingmap.bind_to_environ(request.environ)
        try:
            endpoint, args = mapadapter.match()
            if endpoint == "index" and not request.environ["PATH_INFO"]:
                raise InternalRedirect("/")
            method = getattr(self, "show_" + endpoint, None)
            if method is None:
                return NotFound()
            return method(**args)
        except InternalRedirect as r:
            return redirect(request.environ["SCRIPT_NAME"] + r.target, r.code)
        except HTTPException as e:
            return e

    def get_details(self, package):
        with self.cursor() as cur:
            cur.execute("SELECT id, version, architecture FROM package WHERE name = ?;",
                        (package,))
            row = cur.fetchone()
            if not row:
                raise NotFound()
            pid, version, architecture = row
            details = dict(pid=pid,
                           package=package,
                           version=version,
                           architecture=architecture)
            cur.execute("SELECT count(filename), sum(size) FROM content WHERE pid = ?;",
                        (pid,))
            num_files, total_size = cur.fetchone()
        if total_size is None:
            total_size = 0
        details.update(dict(num_files=num_files, total_size=total_size))
        return details

    def get_dependencies(self, pid):
        with self.cursor() as cur:
            cur.execute("SELECT required FROM dependency WHERE pid = ?;",
                        (pid,))
            return set(row[0] for row in fetchiter(cur))

    def cached_sharedstats(self, pid):
        sharedstats = {}
        with self.cursor() as cur:
            cur.execute("SELECT pid2, package.name, f1.name, f2.name, files, size FROM sharing JOIN package ON sharing.pid2 = package.id JOIN function AS f1 ON sharing.fid1 = f1.id JOIN function AS f2 ON sharing.fid2 = f2.id WHERE pid1 = ? AND f1.eqclass = f2.eqclass;",
                        (pid,))
            for pid2, package2, func1, func2, files, size in fetchiter(cur):
                curstats = sharedstats.setdefault(
                        function_combination(func1, func2), list())
                if pid2 == pid:
                    package2 = None
                curstats.append(dict(package=package2, duplicate=files,
                                     savable=size))
        return sharedstats

    def show_index(self):
        return html_response(index_template.stream(dict(urlroot="")))

    def show_package(self, *, package):
        params = self.get_details(package)
        params["dependencies"] = self.get_dependencies(params["pid"])
        params["shared"] = self.cached_sharedstats(params["pid"])
        params["urlroot"] = ".."
        with self.cursor() as cur:
            cur.execute("SELECT content.filename, issue.issue FROM content JOIN issue ON content.id = issue.cid WHERE content.pid = ?;",
                        (params["pid"],))
            params["issues"] = dict(cur.fetchall())
        return html_response(package_template.stream(params))

    def compute_comparison(self, pid1, pid2):
        """Compute a sequence of comparison objects ordered by the size of the
        object in the first package. Each element of the sequence is a dict
        defining the following keys:
         * filenames: A set of filenames in package 1 (pid1) all referring to
           the same object.
         * size: Size of the object in bytes.
         * matches: A mapping from filenames in package 2 (pid2) to a mapping
           from hash function pairs to hash values.
        """
        with self.cursor() as cur, self.cursor() as cur2:
            cur.execute("SELECT content.id, content.filename, content.size, hash.hash FROM content JOIN hash ON content.id = hash.cid JOIN duplicate ON content.id = duplicate.cid JOIN function ON hash.fid = function.id WHERE pid = ? AND function.name = 'sha512' ORDER BY size DESC;",
                        (pid1,))
            cursize = -1
            files = dict()
            minmatch = 2 if pid1 == pid2 else 1
            cur2 = self.db.cursor()
            for cid, filename, size, hashvalue in fetchiter(cur):
                if cursize != size:
                    for entry in files.values():
                        if len(entry["matches"]) >= minmatch:
                            yield entry
                    files.clear()
                    cursize = size

                if hashvalue in files:
                    files[hashvalue]["filenames"].add(filename)
                    continue

                entry = dict(filenames=set((filename,)), size=size, matches={})
                files[hashvalue] = entry

                cur2.execute("SELECT fa.name, ha.hash, fb.name, filename FROM hash AS ha JOIN hash AS hb ON ha.hash = hb.hash JOIN content ON hb.cid = content.id JOIN function AS fa ON ha.fid = fa.id JOIN function AS fb ON hb.fid = fb.id WHERE ha.cid = ? AND pid = ? AND fa.eqclass = fb.eqclass;",
                             (cid, pid2))
                for func1, hashvalue, func2, filename in fetchiter(cur2):
                    entry["matches"].setdefault(filename, {})[func1, func2] = \
                            hashvalue

        for entry in files.values():
            if len(entry["matches"]) >= minmatch:
                yield entry

    def show_detail(self, *, package1, package2):
        details1 = details2 = self.get_details(package1)
        if package1 != package2:
            details2 = self.get_details(package2)

        shared = self.compute_comparison(details1["pid"], details2["pid"])
        params = dict(
            details1=details1,
            details2=details2,
            urlroot="../..",
            shared=shared)
        return html_response(detail_template.stream(params))

    def show_hash(self, *, function, hashvalue):
        if function == "image_sha512":
            # backwards compatibility
            raise InternalRedirect("/hash/png_sha512/%s" % hashvalue)

        with self.cursor() as cur:
            cur.execute("SELECT package.name, content.filename, content.size, f2.name FROM hash JOIN content ON hash.cid = content.id JOIN package ON content.pid = package.id JOIN function AS f2 ON hash.fid = f2.id JOIN function AS f1 ON f2.eqclass = f1.eqclass WHERE f1.name = ? AND hash = ?;",
                        (function, hashvalue,))
            entries = [dict(package=package, filename=filename, size=size,
                            function=otherfunc)
                       for package, filename, size, otherfunc in fetchiter(cur)]
            if not entries:
                # Assumption: '~' serves as an infinite character larger than
                # any other character in the hash column.
                cur.execute("SELECT DISTINCT hash.hash FROM hash JOIN function ON hash.fid = function.id WHERE function.name = ? AND hash.hash >= ? AND hash.hash <= ? LIMIT 2;",
                            (function, hashvalue, hashvalue + '~'))
                values = cur.fetchall()
                if len(values) == 1:
                    raise InternalRedirect("/hash/%s/%s" %
                                           (function, values[0][0]), 302)
                raise NotFound()
        params = dict(function=function, hashvalue=hashvalue, entries=entries,
                      urlroot="../..")
        return html_response(hash_template.stream(params))

    def show_source(self, *, package):
        with self.cursor() as cur:
            cur.execute("SELECT name FROM package WHERE source = ?;",
                        (package,))
            binpkgs = dict.fromkeys(pkg for pkg, in fetchiter(cur))
            if not binpkgs:
                raise NotFound
            cur.execute("SELECT p1.name, p2.name, f1.name, f2.name, sharing.files, sharing.size FROM sharing JOIN package AS p1 ON sharing.pid1 = p1.id JOIN package AS p2 ON sharing.pid2 = p2.id JOIN function AS f1 ON sharing.fid1 = f1.id JOIN function AS f2 ON sharing.fid2 = f2.id WHERE p1.source = ?;",
                        (package,))
            for binary, otherbin, func1, func2, files, size in fetchiter(cur):
                entry = dict(package=otherbin,
                             funccomb=function_combination(func1, func2),
                             duplicate=files, savable=size)
                oldentry = binpkgs.get(binary)
                if not (oldentry and oldentry["savable"] >= size):
                    binpkgs[binary] = entry
        params = dict(source=package, packages=binpkgs, urlroot="..")
        return html_response(source_template.stream(params))

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-d", "--database", action="store",
                        default="test.sqlite3",
                        help="path to the sqlite3 database file")
    args = parser.parse_args()
    app = Application(sqlite3.connect(args.database))
    app = SharedDataMiddleware(app, {"/static": ("dedup", "static")})
    make_server("0.0.0.0", 8800, app).serve_forever()

if __name__ == "__main__":
    main()