initial checkin

1 files changed, 23 insertions, 0 deletions

diff --git a/common.py b/common.py
new file mode 100644
index 0000000..bb8b7a4
--- /dev/null
+++ b/common.py
@@ -0,0 +1,23 @@
+def yield_lines(iterable):
+    """Converts an arbitrary bytes iterable into an iterable that yields whole
+    lines. The final byte of each returned value (except possibly the last one)
+    is a newline or carriage return character. The concatenation of the input
+    iterable equals the concatenation of the output iterable."""
+    buff = b""
+    for data in iterable:
+        buff += data
+        parts = buff.splitlines(True)
+        buff = parts.pop()
+        yield from parts
+    if buff:
+        yield buff
+
+def decompress_stream(iterable, decompressor):
+    """Decompress an iterable of bytes using the given decompressor into
+    another (decompressed) iterable of bytes. The decompressor can be a
+    bz2.BZ2Decompressor or lzma.LZMADecompressor instance."""
+    for data in iterable:
+        data = decompressor.decompress(data)
+        yield data
+    if hasattr(decompressor, "flush"):
+        yield decompressor.flush()