From 5df1185e5fa0830b546b4ef6af3cdadc655c16c8 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Sat, 23 Sep 2017 10:33:43 +0200 Subject: add module dedup.filemagic This module is not used anywhere and thus its dependency on python3-magic is not recorded in the README. It can be used to guess the file type by looking at the contents using file magic. It is not a typical hash function, but it can be used for repurposing dedup for other analysers. --- dedup/filemagic.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 dedup/filemagic.py (limited to 'dedup/filemagic.py') diff --git a/dedup/filemagic.py b/dedup/filemagic.py new file mode 100644 index 0000000..4cc9357 --- /dev/null +++ b/dedup/filemagic.py @@ -0,0 +1,43 @@ +"""A very strange "hash" that uses the magic module (python3-magic) to guess +the file type.""" + +import magic + +class FileDigester(object): + """A hashlib-like class to guess a filetype using the magic module.""" + FILE_BYTES_MAX = 1024 * 1024 # copied from file source + + def __init__(self): + self.buff = b"" + self.identification = None + + def _compute_identification(self): + try: + return magic.none_magic.buffer(self.buff) + except UnicodeDecodeError: + return "magic identification is not valid UTF-8" + + def update(self, buff): + if self.identification: + return + self.buff += buff + if len(self.buff) >= self.FILE_BYTES_MAX: + self.identification = self._compute_identification() + self.buff = None + + def identify(self): + """Return the guessed file magic identification.""" + if self.identification: + return self.identification + return self._compute_identification() + + def hexdigest(self): + """Compatibility with hashlib. An alias of identify. Doesn't return + hex.""" + return self.identify() + + def copy(self): + new = FileDigester() + new.buff = self.buff + new.identification = self.identification + return new -- cgit v1.2.3 From e77a1ebf8bda10494088bb6c72873d8ef214e0f3 Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Mon, 25 Jun 2018 21:07:41 +0200 Subject: adapt to python3-magic/2:0.4.15-1 API --- dedup/filemagic.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'dedup/filemagic.py') diff --git a/dedup/filemagic.py b/dedup/filemagic.py index 4cc9357..c5a6357 100644 --- a/dedup/filemagic.py +++ b/dedup/filemagic.py @@ -3,6 +3,12 @@ the file type.""" import magic +# It changed API a few times... +try: + _magic_identify = magic.from_buffer +except AttributeError: + _magic_identify = magic.none_magic.buffer + class FileDigester(object): """A hashlib-like class to guess a filetype using the magic module.""" FILE_BYTES_MAX = 1024 * 1024 # copied from file source @@ -13,7 +19,7 @@ class FileDigester(object): def _compute_identification(self): try: - return magic.none_magic.buffer(self.buff) + return _magic_identify(self.buff) except UnicodeDecodeError: return "magic identification is not valid UTF-8" -- cgit v1.2.3 From 27b95909f061ae3ecb3ba1b8d46adfef98ca5e6f Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Sun, 16 Feb 2020 08:21:20 +0100 Subject: drop support for Python 2.x --- README | 2 +- autoimport.py | 10 +++------- dedup/arreader.py | 2 +- dedup/compression.py | 17 +++++++---------- dedup/debpkg.py | 32 +++++--------------------------- dedup/filemagic.py | 2 +- dedup/hashing.py | 14 +++++--------- dedup/image.py | 2 +- dedup/utils.py | 21 +++++++-------------- importpkg.py | 22 ++++++++-------------- readyaml.py | 2 +- update_sharing.py | 2 +- webapp.py | 4 ++-- 13 files changed, 43 insertions(+), 89 deletions(-) (limited to 'dedup/filemagic.py') diff --git a/README b/README index 5329bd8..d0a488c 100644 --- a/README +++ b/README @@ -1,7 +1,7 @@ Required packages ----------------- - aptitude install python python-debian python-lzma python-jinja2 python-werkzeug sqlite3 python-imaging python-yaml python-concurrent.futures python-pkg-resources + aptitude install python3 python3-debian python3-lzma python3-jinja2 python3-werkzeug sqlite3 python3-imaging python3-yaml python3-concurrent.futures python3-pkg-resources Create a database ----------------- diff --git a/autoimport.py b/autoimport.py index e51d052..ec47db0 100755 --- a/autoimport.py +++ b/autoimport.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 """This scrip takes a directory or a http base url to a mirror and imports all packages contained. It has rather strong assumptions on the working directory. """ @@ -12,11 +12,7 @@ import sqlite3 import subprocess import sys import tempfile -try: - from urllib.parse import unquote -except ImportError: - from urllib import unquote - +import urllib.parse import concurrent.futures from debian import deb822 from debian.debian_support import version_compare @@ -47,7 +43,7 @@ def process_file(pkgs, filename): if len(parts) != 3: raise ValueError("filename not in form name_version_arch.deb") name, version, _ = parts - version = unquote(version) + version = urllib.parse.unquote(version) if name in pkgs and version_compare(pkgs[name]["version"], version) > 0: return pkgs[name] = dict(version=version, filename=filename) diff --git a/dedup/arreader.py b/dedup/arreader.py index e53efd9..8b14ff9 100644 --- a/dedup/arreader.py +++ b/dedup/arreader.py @@ -1,6 +1,6 @@ import struct -class ArReader(object): +class ArReader: """Streaming AR file reader. After constructing an object, you usually call read_magic once. Then you call read_entry in a loop and use the ArReader object as file-like only providing read() to read the respective diff --git a/dedup/compression.py b/dedup/compression.py index 8d1912b..161eda2 100644 --- a/dedup/compression.py +++ b/dedup/compression.py @@ -1,13 +1,10 @@ import bz2 import struct -import sys import zlib import lzma -crc32_type = "L" if sys.version_info.major >= 3 else "l" - -class GzipDecompressor(object): +class GzipDecompressor: """An interface to gzip which is similar to bz2.BZ2Decompressor and lzma.LZMADecompressor.""" def __init__(self): @@ -66,7 +63,7 @@ class GzipDecompressor(object): elif not self.sawheader: return self.inbuffer else: - expect = struct.pack("<" + crc32_type + "L", self.crc, self.size) + expect = struct.pack("= 3: - def opentar(filelike): - return tarfile.open(fileobj=filelike, mode="r|", encoding="utf8", - errors="surrogateescape") +def opentar(filelike): + return tarfile.open(fileobj=filelike, mode="r|", encoding="utf8", + errors="surrogateescape") - def decodetarname(name): - """Decoded name of a tarinfo. - @raises UnicodeDecodeError: - """ - try: - name.encode("utf8", "strict") - except UnicodeEncodeError as e: - if e.reason == "surrogates not allowed": - name.encode("utf8", "surrogateescape").decode("utf8", "strict") - return name -else: - def opentar(filelike): - return tarfile.open(fileobj=filelike, mode="r|") - - def decodetarname(name): - """Decoded name of a tarinfo. - @raises UnicodeDecodeError: - """ - return name.decode("utf8") - -class DebExtractor(object): +class DebExtractor: "Base class for extracting desired features from a Debian package." def __init__(self): diff --git a/dedup/filemagic.py b/dedup/filemagic.py index c5a6357..b71c276 100644 --- a/dedup/filemagic.py +++ b/dedup/filemagic.py @@ -9,7 +9,7 @@ try: except AttributeError: _magic_identify = magic.none_magic.buffer -class FileDigester(object): +class FileDigester: """A hashlib-like class to guess a filetype using the magic module.""" FILE_BYTES_MAX = 1024 * 1024 # copied from file source diff --git a/dedup/hashing.py b/dedup/hashing.py index c91fb64..21f14ea 100644 --- a/dedup/hashing.py +++ b/dedup/hashing.py @@ -1,10 +1,6 @@ import itertools -try: - from itertools import imap as map -except ImportError: - pass # in python3 map is already imap -class HashBlacklist(object): +class HashBlacklist: """Turn a hashlib-like object into a hash that returns None for some blacklisted hashes instead of the real hash value. @@ -35,7 +31,7 @@ class HashBlacklist(object): def copy(self): return HashBlacklist(self.hashobj.copy(), self.blacklist) -class HashBlacklistContent(object): +class HashBlacklistContent: """Turn a hashlib-like object into a hash that returns None for some blacklisted content instead of the real hash value. Unlike HashBlacklist, not the output of the hash is considered, but its input.""" @@ -85,7 +81,7 @@ class HashBlacklistContent(object): new.stored = self.stored return new -class DecompressedHash(object): +class DecompressedHash: """Apply a decompression function before the hash. This class provides the hashlib interface (update, hexdigest, copy) excluding digest and name.""" def __init__(self, decompressor, hashobj): @@ -119,7 +115,7 @@ class DecompressedHash(object): def copy(self): return DecompressedHash(self.decompressor.copy(), self.hashobj.copy()) -class SuppressingHash(object): +class SuppressingHash: """A hash that silences exceptions from the update and hexdigest methods of a hashlib-like object. If an exception has occurred, hexdigest always returns None.""" @@ -167,7 +163,7 @@ def hash_file(hashobj, filelike, blocksize=65536): data = filelike.read(blocksize) return hashobj -class HashedStream(object): +class HashedStream: """A file-like object, that supports sequential reading and hashes the contents on the fly.""" def __init__(self, filelike, hashobj): diff --git a/dedup/image.py b/dedup/image.py index 314eb44..2e64e6b 100644 --- a/dedup/image.py +++ b/dedup/image.py @@ -3,7 +3,7 @@ import struct import PIL.Image -class ImageHash(object): +class ImageHash: """A hash on the contents of an image data type supported by PIL. This disregards mode, depth and meta information. Note that due to limitations in PIL and the image format (interlacing) the full contents are stored and diff --git a/dedup/utils.py b/dedup/utils.py index dab6653..46f8e64 100644 --- a/dedup/utils.py +++ b/dedup/utils.py @@ -1,12 +1,6 @@ import errno -try: - from urllib.error import URLError, HTTPError -except ImportError: - from urllib2 import URLError, HTTPError -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen +import urllib.error +import urllib.request from debian.debian_support import version_compare @@ -15,15 +9,14 @@ from dedup.compression import decompress def fetchiter(cursor): rows = cursor.fetchmany() while rows: - for row in rows: - yield row + yield from rows rows = cursor.fetchmany() def sql_add_version_compare(db): db.create_collation("debian_version", version_compare) db.create_function("debian_version_compare", 2, version_compare) -def open_compressed_mirror_url(url, extensions=(u".xz", u".gz", u"")): +def open_compressed_mirror_url(url, extensions=(".xz", ".gz", "")): """Fetch the given url. Try appending each of the given compression schemes and move on in case it doesn't exist. Decompress the resulting stream on the fly. @@ -31,11 +24,11 @@ def open_compressed_mirror_url(url, extensions=(u".xz", u".gz", u"")): """ for ext in extensions: try: - handle = urlopen(url + ext) - except HTTPError as error: + handle = urllib.request.urlopen(url + ext) + except urllib.error.HTTPError as error: if error.code != 404: raise - except URLError as error: + except urllib.error.URLError as error: if not hasattr(error.reason, "errno"): raise if error.reason.errno != errno.ENOENT: diff --git a/importpkg.py b/importpkg.py index ce4a446..4693401 100755 --- a/importpkg.py +++ b/importpkg.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 """This tool reads a Debian package from stdin and emits a yaml stream on stdout. It does not access a database. Therefore it can be run in parallel and on multiple machines. The generated yaml contains multiple documents. The first @@ -8,15 +8,12 @@ And finally a document consisting of the string "commit" is emitted.""" import argparse import hashlib import sys +import urllib.request import zlib -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen import yaml -from dedup.debpkg import DebExtractor, decodetarname, get_tar_hashes +from dedup.debpkg import DebExtractor, get_tar_hashes from dedup.hashing import DecompressedHash, SuppressingHash, HashedStream, \ HashBlacklistContent from dedup.compression import GzipDecompressor @@ -63,7 +60,7 @@ class ImportpkgExtractor(DebExtractor): # deb822 currently returns :any dependencies raw. see #670679 deprelations = info.relations.get("depends", []) + \ info.relations.get("pre-depends", []) - depends = set(dep[0]["name"].split(u':', 1)[0] + depends = set(dep[0]["name"].split(':', 1)[0] for dep in deprelations if len(dep) == 1) self.callback(dict(package=info["package"], source=source, version=info["version"], @@ -73,22 +70,19 @@ class ImportpkgExtractor(DebExtractor): for name, size, hashes in get_tar_hashes(tarfileobj, self.hash_functions): try: - name = decodetarname(name) - except UnicodeDecodeError: + name.encode("utf8", "strict") + except UnicodeEncodeError: print("warning: skipping filename with encoding error") continue # skip files with non-utf8 encoding for now self.callback(dict(name=name, size=size, hashes=hashes)) raise ProcessingFinished() def main(): - try: - stdin = sys.stdin.buffer - except AttributeError: # python2 - stdin = sys.stdin parser = argparse.ArgumentParser() parser.add_argument("-H", "--hash", action="store", help="verify that stdin hash given sha256 hash") - parser.add_argument("input", nargs='?', default=stdin, type=urlopen, + parser.add_argument("input", nargs='?', default=sys.stdin.buffer, + type=urllib.request.urlopen, help="read from this location instead of stdin") args = parser.parse_args() dumper = yaml.SafeDumper(sys.stdout) diff --git a/readyaml.py b/readyaml.py index b6f7316..a4837cf 100755 --- a/readyaml.py +++ b/readyaml.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 """This tool reads a yaml file as generated by importpkg.py on stdin and updates the database with the contents.""" diff --git a/update_sharing.py b/update_sharing.py index ac6c945..78e6171 100755 --- a/update_sharing.py +++ b/update_sharing.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import argparse import sqlite3 diff --git a/webapp.py b/webapp.py index f9e667e..69e9df8 100755 --- a/webapp.py +++ b/webapp.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import argparse import contextlib @@ -68,7 +68,7 @@ class InternalRedirect(Exception): self.target = target self.code = code -class Application(object): +class Application: def __init__(self, db): self.db = db self.routingmap = Map([ -- cgit v1.2.3