From a3cc49725febb2cca1c915ef768604831563954f Mon Sep 17 00:00:00 2001 From: Helmut Grohne Date: Sun, 3 Feb 2019 12:42:23 +0100 Subject: initial checkin --- common.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 common.py (limited to 'common.py') diff --git a/common.py b/common.py new file mode 100644 index 0000000..bb8b7a4 --- /dev/null +++ b/common.py @@ -0,0 +1,23 @@ +def yield_lines(iterable): + """Converts an arbitrary bytes iterable into an iterable that yields whole + lines. The final byte of each returned value (except possibly the last one) + is a newline or carriage return character. The concatenation of the input + iterable equals the concatenation of the output iterable.""" + buff = b"" + for data in iterable: + buff += data + parts = buff.splitlines(True) + buff = parts.pop() + yield from parts + if buff: + yield buff + +def decompress_stream(iterable, decompressor): + """Decompress an iterable of bytes using the given decompressor into + another (decompressed) iterable of bytes. The decompressor can be a + bz2.BZ2Decompressor or lzma.LZMADecompressor instance.""" + for data in iterable: + data = decompressor.decompress(data) + yield data + if hasattr(decompressor, "flush"): + yield decompressor.flush() -- cgit v1.2.3