#!/usr/bin/python
import datetime
import optparse
from wsgiref.simple_server import make_server
import jinja2
import sqlalchemy
from werkzeug.exceptions import HTTPException, NotFound
from werkzeug.routing import Map, Rule, RequestRedirect
from werkzeug.wrappers import Request, Response
from werkzeug.wsgi import SharedDataMiddleware
from dedup.utils import fetchiter
jinjaenv = jinja2.Environment(loader=jinja2.PackageLoader("dedup", "templates"))
def format_size(size):
size = float(size)
fmt = "%d B"
if size >= 1024:
size /= 1024
fmt = "%.1f KB"
if size >= 1024:
size /= 1024
fmt = "%.1f MB"
if size >= 1024:
size /= 1024
fmt = "%.1f GB"
return fmt % size
def function_combination(function1, function2):
if function1 == function2:
return function1
return "%s -> %s" % (function1, function2)
# Workaround for jinja bug #59 (broken filesizeformat)
jinjaenv.filters["filesizeformat"] = format_size
base_template = jinjaenv.get_template("base.html")
package_template = jinjaenv.get_template("binary.html")
detail_template = jinjaenv.get_template("compare.html")
hash_template = jinjaenv.get_template("hash.html")
index_template = jinjaenv.get_template("index.html")
source_template = jinjaenv.get_template("source.html")
def encode_and_buffer(iterator):
buff = b""
for elem in iterator:
buff += elem.encode("utf8")
if len(buff) >= 2048:
yield buff
buff = b""
if buff:
yield buff
def html_response(unicode_iterator, max_age=24 * 60 * 60):
resp = Response(encode_and_buffer(unicode_iterator), mimetype="text/html")
resp.cache_control.max_age = max_age
resp.expires = datetime.datetime.now() + datetime.timedelta(seconds=max_age)
return resp
class Application(object):
def __init__(self, db):
self.db = db
self.routingmap = Map([
Rule("/", methods=("GET",), endpoint="index"),
Rule("/binary/<package>", methods=("GET",), endpoint="package"),
Rule("/compare/<package1>/<package2>", methods=("GET",), endpoint="detail"),
Rule("/hash/<function>/<hashvalue>", methods=("GET",), endpoint="hash"),
Rule("/source/<package>", methods=("GET",), endpoint="source"),
])
@Request.application
def __call__(self, request):
mapadapter = self.routingmap.bind_to_environ(request.environ)
try:
endpoint, args = mapadapter.match()
if endpoint == "package":
return self.show_package(args["package"])
elif endpoint == "detail":
return self.show_detail(args["package1"], args["package2"])
elif endpoint == "hash":
if args["function"] == "image_sha512":
# backwards compatibility
raise RequestRedirect("%s/hash/png_sha512/%s" %
(request.environ["SCRIPT_NAME"],
args["hashvalue"]))
return self.show_hash(args["function"], args["hashvalue"])
elif endpoint == "index":
if not request.environ["PATH_INFO"]:
raise RequestRedirect(request.environ["SCRIPT_NAME"] + "/")
return html_response(index_template.render(dict(urlroot="")))
elif endpoint == "source":
return self.show_source(args["package"])
raise NotFound()
except HTTPException as e:
return e
def get_details(self, package):
with self.db.begin() as conn:
row = conn.execute(sqlalchemy.text("SELECT id, version, architecture FROM package WHERE name = :name;"),
name=package).fetchone()
if not row:
raise NotFound()
pid, version, architecture = row
row = conn.execute(sqlalchemy.text("SELECT count(filename), sum(size) FROM content WHERE pid = :pid;"),
pid=pid).fetchone()
num_files, total_size = row
if total_size is None:
total_size = 0
return dict(pid=pid,
package=package,
version=version,
architecture=architecture,
num_files=num_files,
total_size=total_size)
def get_dependencies(self, pid):
with self.db.begin() as conn:
cur = conn.execute(sqlalchemy.text("SELECT required FROM dependency WHERE pid = :pid;"),
pid=pid)
return set(row[0] for row in fetchiter(cur))
def cached_sharedstats(self, pid):
sharedstats = {}
with self.db.begin() as conn:
cur = conn.execute(sqlalchemy.text("SELECT pid2, package.name, f1.name, f2.name, files, size FROM sharing JOIN package ON sharing.pid2 = package.id JOIN function AS f1 ON sharing.fid1 = f1.id JOIN function AS f2 ON sharing.fid2 = f2.id WHERE pid1 = :pid AND f1.eqclass = f2.eqclass;"),
pid=pid)
for pid2, package2, func1, func2, files, size in fetchiter(cur):
curstats = sharedstats.setdefault(
function_combination(func1, func2), list())
if pid2 == pid:
package2 = None
curstats.append(dict(package=package2, duplicate=files,
savable=size))
return sharedstats
def show_package(self, package):
params = self.get_details(package)
params["dependencies"] = self.get_dependencies(params["pid"])
params["shared"] = self.cached_sharedstats(params["pid"])
params["urlroot"] = ".."
with self.db.begin() as conn:
cur = conn.execute(sqlalchemy.text("SELECT content.filename, issue.issue FROM content JOIN issue ON content.id = issue.cid WHERE content.pid = :pid;"),
pid=params["pid"])
params["issues"] = dict(cur.fetchall())
return html_response(package_template.render(params))
def compute_comparison(self, pid1, pid2):
"""Compute a sequence of comparison objects ordered by the size of the
object in the first package. Each element of the sequence is a dict
defining the following keys:
* filenames: A set of filenames in package 1 (pid1) all referring to
the same object.
* size: Size of the object in bytes.
* matches: A mapping from filenames in package 2 (pid2) to a mapping
from hash function pairs to hash values.
"""
with self.db.begin() as conn:
cur = conn.execute(sqlalchemy.text("SELECT content.id, content.filename, content.size, hash.hash FROM content JOIN hash ON content.id = hash.cid JOIN duplicate ON content.id = duplicate.cid JOIN function ON hash.fid = function.id WHERE pid = :pid AND function.name = 'sha512' ORDER BY size DESC;"),
pid=pid1)
cursize = -1
files = dict()
minmatch = 2 if pid1 == pid2 else 1
for cid, filename, size, hashvalue in fetchiter(cur):
if cursize != size:
for entry in files.values():
if len(entry["matches"]) >= minmatch:
yield entry
files.clear()
cursize = size
if hashvalue in files:
files[hashvalue]["filenames"].add(filename)
continue
entry = dict(filenames=set((filename,)), size=size, matches={})
files[hashvalue] = entry
cur2 = conn.execute(sqlalchemy.text("SELECT fa.name, ha.hash, fb.name, filename FROM hash AS ha JOIN hash AS hb ON ha.hash = hb.hash JOIN content ON hb.cid = content.id JOIN function AS fa ON ha.fid = fa.id JOIN function AS fb ON hb.fid = fb.id WHERE ha.cid = :cid AND pid = :pid AND fa.eqclass = fb.eqclass;"),
cid=cid, pid=pid2)
for func1, hashvalue, func2, filename in fetchiter(cur2):
entry["matches"].setdefault(filename, {})[func1, func2] = \
hashvalue
for entry in files.values():
if len(entry["matches"]) >= minmatch:
yield entry
def show_detail(self, package1, package2):
details1 = details2 = self.get_details(package1)
if package1 != package2:
details2 = self.get_details(package2)
shared = self.compute_comparison(details1["pid"], details2["pid"])
params = dict(
details1=details1,
details2=details2,
urlroot="../..",
shared=shared)
return html_response(detail_template.stream(params))
def show_hash(self, function, hashvalue):
with self.db.begin() as conn:
cur = conn.execute(sqlalchemy.text("SELECT package.name, content.filename, content.size, f2.name FROM hash JOIN content ON hash.cid = content.id JOIN package ON content.pid = package.id JOIN function AS f2 ON hash.fid = f2.id JOIN function AS f1 ON f2.eqclass = f1.eqclass WHERE f1.name = :function AND hash = :hashvalue;"),
function=function, hashvalue=hashvalue)
entries = [dict(package=package, filename=filename, size=size,
function=otherfunc)
for package, filename, size, otherfunc in fetchiter(cur)]
if not entries:
raise NotFound()
params = dict(function=function, hashvalue=hashvalue, entries=entries,
urlroot="../..")
return html_response(hash_template.render(params))
def show_source(self, package):
with self.db.begin() as conn:
cur = conn.execute(sqlalchemy.text("SELECT name FROM package WHERE source = :source;"),
source=package)
binpkgs = dict.fromkeys(pkg for pkg, in fetchiter(cur))
if not binpkgs:
raise NotFound
cur = conn.execute(sqlalchemy.text("SELECT p1.name, p2.name, f1.name, f2.name, sharing.files, sharing.size FROM sharing JOIN package AS p1 ON sharing.pid1 = p1.id JOIN package AS p2 ON sharing.pid2 = p2.id JOIN function AS f1 ON sharing.fid1 = f1.id JOIN function AS f2 ON sharing.fid2 = f2.id WHERE p1.source = :source;"),
source=package)
for binary, otherbin, func1, func2, files, size in fetchiter(cur):
entry = dict(package=otherbin,
funccomb=function_combination(func1, func2),
duplicate=files, savable=size)
oldentry = binpkgs.get(binary)
if not (oldentry and oldentry["savable"] >= size):
binpkgs[binary] = entry
params = dict(source=package, packages=binpkgs, urlroot="..")
return html_response(source_template.render(params))
def main():
parser = optparse.OptionParser()
parser.add_option("-d", "--database", action="store",
default="sqlite:///test.sqlite3",
help="location of the database")
options, args = parser.parse_args()
db = sqlalchemy.create_engine(options.database)
app = Application(db)
app = SharedDataMiddleware(app, {"/static": ("dedup", "static")})
make_server("0.0.0.0", 8800, app).serve_forever()
if __name__ == "__main__":
main()