From 12c89ebaf67b5a55ea23af3ed47038968870f309 Mon Sep 17 00:00:00 2001
From: Helmut Grohne <helmut@subdivi.de>
Date: Thu, 21 Feb 2013 17:33:27 +0100
Subject: move compression functions to module dedup.compression
---
dedup/compression.py | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 103 insertions(+)
create mode 100644 dedup/compression.py
(limited to 'dedup')
diff --git a/dedup/compression.py b/dedup/compression.py
new file mode 100644
index 0000000..869c49f
--- /dev/null
+++ b/dedup/compression.py
@@ -0,0 +1,103 @@
+import struct
+import zlib
+
+class GzipDecompressor(object):
+ """An interface to gzip which is similar to bz2.BZ2Decompressor and
+ lzma.LZMADecompressor."""
+ def __init__(self):
+ self.inbuffer = b""
+ self.decompressor = None
+
+ def decompress(self, data):
+ """
+ @raises ValueError: if no gzip magic is found
+ @raises zlib.error: from zlib invocations
+ """
+ while True:
+ if self.decompressor:
+ data = self.decompressor.decompress(data)
+ unused_data = self.decompressor.unused_data
+ if not unused_data:
+ return data
+ self.decompressor = None
+ return data + self.decompress(unused_data)
+ self.inbuffer += data
+ skip = 10
+ if len(self.inbuffer) < skip:
+ return b""
+ if not self.inbuffer.startswith(b"\037\213\010"):
+ raise ValueError("gzip magic not found")
+ flag = ord(self.inbuffer[3])
+ if flag & 4:
+ if len(self.inbuffer) < skip + 2:
+ return b""
+ length, = struct.unpack("<H", self.inbuffer[skip:skip+2])
+ skip += 2 + length
+ for field in (8, 16):
+ if flag & field:
+ length = self.inbuffer.find(b"\0", skip)
+ if length < 0:
+ return b""
+ skip = length + 1
+ if flag & 2:
+ skip += 2
+ if len(self.inbuffer) < skip:
+ return b""
+ data = self.inbuffer[skip:]
+ self.inbuffer = b""
+ self.decompressor = zlib.decompressobj(-zlib.MAX_WBITS)
+
+ @property
+ def unused_data(self):
+ if self.decompressor:
+ return self.decompressor.unused_data
+ else:
+ return self.inbuffer
+
+ def flush(self):
+ """
+ @raises zlib.error: from zlib invocations
+ """
+ if not self.decompressor:
+ return b""
+ return self.decompressor.flush()
+
+ def copy(self):
+ new = GzipDecompressor()
+ new.inbuffer = self.inbuffer
+ if self.decompressor:
+ new.decompressor = self.decompressor.copy()
+ return new
+
+class DecompressedStream(object):
+ """Turn a readable file-like into a decompressed file-like. Te only part
+ of being file-like consists of the read(size) method in both cases."""
+ blocksize = 65536
+
+ def __init__(self, fileobj, decompressor):
+ """
+ @param fileobj: a file-like object providing read(size)
+ @param decompressor: a bz2.BZ2Decompressor or lzma.LZMADecompressor
+ like object providing methods decompress and flush and an
+ attribute unused_data
+ """
+ self.fileobj = fileobj
+ self.decompressor = decompressor
+ self.buff = b""
+
+ def read(self, length=None):
+ data = True
+ while True:
+ if length is not None and len(self.buff) >= length:
+ ret = self.buff[:length]
+ self.buff = self.buff[length:]
+ return ret
+ elif not data: # read EOF in last iteration
+ ret = self.buff
+ self.buff = b""
+ return ret
+ data = self.fileobj.read(self.blocksize)
+ if data:
+ self.buff += self.decompressor.decompress(data)
+ else:
+ self.buff += self.decompressor.flush()
--
cgit v1.2.3