Buffer compression writers

src/xopen/__init__.py

@@ -732,10 +732,21 @@ def xopen(
         detected_format = _detect_format_from_content(filename)
 
     if detected_format == "gz":
-        return _open_gz(filename, mode, compresslevel, threads)
+        opened_file = _open_gz(filename, mode, compresslevel, threads)
     elif detected_format == "xz":
-        return _open_xz(filename, mode)
+        opened_file = _open_xz(filename, mode)
     elif detected_format == "bz2":
-        return _open_bz2(filename, mode, threads)
+        opened_file = _open_bz2(filename, mode, threads)
     else:
-        return open(filename, mode)
+        opened_file = open(filename, mode)
+
+    # The "write" method for GzipFile is very costly. Lots of python calls are
+    # made. To a lesser extent this is true for LzmaFile and BZ2File. By
+    # putting a buffer in between, the expensive write method is called much
+    # less. The effect is very noticeable when writing small units such as
+    # lines or FASTQ records.
+    if (isinstance(opened_file, gzip.GzipFile) or
+            isinstance(opened_file, bz2.BZ2File) or
+            isinstance(opened_file, lzma.LZMAFile)) and "w" in mode:
+        opened_file = io.BufferedWriter(opened_file)  # type: ignore
+    return opened_file

tests/test_xopen.py

@@ -532,14 +532,16 @@ def test_write_no_threads(tmpdir, ext):
     klass = klasses[ext]
     path = str(tmpdir.join(f"out.{ext}"))
     with xopen(path, "wb", threads=0) as f:
-        assert isinstance(f, klass), f
+        assert isinstance(f, io.BufferedWriter)
+        if ext:
+            assert isinstance(f.raw, klass), f
 
 
 def test_write_gzip_no_threads_no_isal(tmpdir, xopen_without_igzip):
     import gzip
     path = str(tmpdir.join("out.gz"))
     with xopen_without_igzip(path, "wb", threads=0) as f:
-        assert isinstance(f, gzip.GzipFile), f
+        assert isinstance(f.raw, gzip.GzipFile), f
 
 
 def test_write_stdout():