def yield_lines(iterable): """Converts an arbitrary bytes iterable into an iterable that yields whole lines. The final byte of each returned value (except possibly the last one) is a newline or carriage return character. The concatenation of the input iterable equals the concatenation of the output iterable.""" buff = b"" for data in iterable: buff += data parts = buff.splitlines(True) buff = parts.pop() yield from parts if buff: yield buff def decompress_stream(iterable, decompressor): """Decompress an iterable of bytes using the given decompressor into another (decompressed) iterable of bytes. The decompressor can be a bz2.BZ2Decompressor or lzma.LZMADecompressor instance.""" for data in iterable: data = decompressor.decompress(data) yield data if hasattr(decompressor, "flush"): yield decompressor.flush()