Merge branch 'master' into sqlalchemy

In the mean time, the master branch evolved quite a bit and the schema
changed again (eqclass added to function table). The main reason for the
merge is to resolve the large amounts of conflicts once, so development
of the sqlalchemy branch can continue and still benefit from changes in
the master branch such as schema compatibility, adapting the indent
level in web app due to the use of contextlib.closing which resembles
sqlalchemy's "with db.begin() as conn:".

Conflicts:
	autoimport.py
	dedup/utils.py
	readyaml.py
	update_sharing.py
	webapp.py

1 files changed, 15 insertions, 22 deletions

diff --git a/webapp.py b/webapp.py
index bbc45e1..f1a0df3 100755
--- a/webapp.py
+++ b/webapp.py
@@ -1,6 +1,7 @@
 #!/usr/bin/python
 
 import datetime
+import optparse
 from wsgiref.simple_server import make_server
 
 import jinja2
@@ -12,16 +13,6 @@ from werkzeug.wsgi import SharedDataMiddleware
 
 from dedup.utils import fetchiter
 
-hash_functions = [
-        ("sha512", "sha512"),
-        ("png_sha512", "png_sha512"),
-        ("png_sha512", "gif_sha512"),
-        ("gif_sha512", "png_sha512"),
-        ("gif_sha512", "gif_sha512"),
-        ("gzip_sha512", "gzip_sha512"),
-        ("sha512", "gzip_sha512"),
-        ("gzip_sha512", "sha512")]
-
 jinjaenv = jinja2.Environment(loader=jinja2.PackageLoader("dedup", "templates"))
 
 def format_size(size):
@@ -134,11 +125,9 @@ class Application(object):
     def cached_sharedstats(self, pid):
         sharedstats = {}
         with self.db.begin() as conn:
-            cur = conn.execute(sqlalchemy.text("SELECT pid2, package.name, f1.name, f2.name, files, size FROM sharing JOIN package ON sharing.pid2 = package.id JOIN function AS f1 ON sharing.fid1 = f1.id JOIN function AS f2 ON sharing.fid2 = f2.id WHERE pid1 = :pid;"),
+            cur = conn.execute(sqlalchemy.text("SELECT pid2, package.name, f1.name, f2.name, files, size FROM sharing JOIN package ON sharing.pid2 = package.id JOIN function AS f1 ON sharing.fid1 = f1.id JOIN function AS f2 ON sharing.fid2 = f2.id WHERE pid1 = :pid AND f1.eqclass = f2.eqclass;"),
                                pid=pid)
             for pid2, package2, func1, func2, files, size in fetchiter(cur):
-                if (func1, func2) not in hash_functions:
-                    continue
                 curstats = sharedstats.setdefault(
                         function_combination(func1, func2), list())
                 if pid2 == pid:
@@ -159,7 +148,7 @@ class Application(object):
         return html_response(package_template.render(params))
 
     def compute_comparison(self, pid1, pid2):
-        """Compute a sequence of comparison objects ordery by the size of the
+        """Compute a sequence of comparison objects ordered by the size of the
         object in the first package. Each element of the sequence is a dict
         defining the following keys:
          * filenames: A set of filenames in package 1 (pid1) all referring to
@@ -189,9 +178,9 @@ class Application(object):
                 entry = dict(filenames=set((filename,)), size=size, matches={})
                 files[hashvalue] = entry
 
-                cur = conn.execute(sqlalchemy.text("SELECT fa.name, ha.hash, fb.name, filename FROM hash AS ha JOIN hash AS hb ON ha.hash = hb.hash JOIN content ON hb.cid = content.id JOIN function AS fa ON ha.fid = fa.id JOIN function AS fb ON hb.fid = fb.id WHERE ha.cid = :cid AND pid = :pid;"),
+                cur2 = conn.execute(sqlalchemy.text("SELECT fa.name, ha.hash, fb.name, filename FROM hash AS ha JOIN hash AS hb ON ha.hash = hb.hash JOIN content ON hb.cid = content.id JOIN function AS fa ON ha.fid = fa.id JOIN function AS fb ON hb.fid = fb.id WHERE ha.cid = :cid AND pid = :pid AND fa.eqclass = fb.eqclass;"),
                                    cid=cid, pid=pid2)
-                for func1, hashvalue, func2, filename in fetchiter(cur):
+                for func1, hashvalue, func2, filename in fetchiter(cur2):
                     entry["matches"].setdefault(filename, {})[func1, func2] = \
                             hashvalue
 
@@ -214,12 +203,11 @@ class Application(object):
 
     def show_hash(self, function, hashvalue):
         with self.db.begin() as conn:
-            cur = conn.execute(sqlalchemy.text("SELECT package.name, content.filename, content.size, function.name FROM hash JOIN content ON hash.cid = content.id JOIN package ON content.pid = package.id JOIN function ON hash.fid = function.id WHERE hash = :hashvalue;"),
-                               hashvalue=hashvalue)
+            cur = conn.execute(sqlalchemy.text("SELECT package.name, content.filename, content.size, f2.name FROM hash JOIN content ON hash.cid = content.id JOIN package ON content.pid = package.id JOIN function AS f2 ON hash.fid = f2.id JOIN function AS f1 ON f2.eqclass = f1.eqclass WHERE f1.name = :function AND hash = :hashvalue;"),
+                               function=function, hashvalue=hashvalue)
             entries = [dict(package=package, filename=filename, size=size,
                             function=otherfunc)
-                       for package, filename, size, otherfunc in fetchiter(cur)
-                       if (function, otherfunc) in hash_functions]
+                       for package, filename, size, otherfunc in fetchiter(cur)]
         if not entries:
             raise NotFound()
         params = dict(function=function, hashvalue=hashvalue, entries=entries,
@@ -246,9 +234,14 @@ class Application(object):
         return html_response(source_template.render(params))
 
 def main():
-    db = sqlalchemy.create_engine("sqlite:///test.sqlite3")
+    parser = optparse.OptionParser()
+    parser.add_option("-d", "--database", action="store",
+                      default="sqlite:///test.sqlite3",
+                      help="location of the database")
+    options, args = parser.parse_args()
+    db = sqlalchemy.create_engine(options.database)
     app = Application(db)
-    app = SharedDataMiddleware(app, {"/": ("dedup", "static")})
+    app = SharedDataMiddleware(app, {"/static": ("dedup", "static")})
     make_server("0.0.0.0", 8800, app).serve_forever()
 
 if __name__ == "__main__":