diff --git a/fsspec/implementations/local.py b/fsspec/implementations/local.py index b549b93e2..8b180c95c 100644 --- a/fsspec/implementations/local.py +++ b/fsspec/implementations/local.py @@ -437,6 +437,11 @@ def __getstate__(self): def commit(self): if self.autocommit: raise RuntimeError("Can only commit if not already set to autocommit") + if not self.f.closed: + # a compression wrapper (e.g., GzipFile) does not close the file + # object it was given, so buffered bytes and any trailer may still + # be pending here. Windows also refuses to rename an open file. + self.f.close() try: shutil.move(self.temp, self.path) except PermissionError as e: diff --git a/fsspec/implementations/tests/test_local.py b/fsspec/implementations/tests/test_local.py index d74b168d9..52d11a980 100644 --- a/fsspec/implementations/tests/test_local.py +++ b/fsspec/implementations/tests/test_local.py @@ -509,6 +509,19 @@ def test_commit_discard(tmpdir): assert not fs.exists(tmpdir + "/bfile") +def test_transaction_with_compression(tmpdir): + path = str(tmpdir / "afile.gz") + fs = LocalFileSystem() + + with fs.transaction: + with fs.open(path, "wt", compression="gzip") as f: + f.write("data") + assert not fs.exists(path) + + with gzip.open(path, "rt") as f: + assert f.read() == "data" + + def test_same_permissions_with_and_without_transaction(tmpdir): tmpdir = str(tmpdir) diff --git a/fsspec/spec.py b/fsspec/spec.py index a390bf060..429b73cd9 100644 --- a/fsspec/spec.py +++ b/fsspec/spec.py @@ -1363,6 +1363,8 @@ def open( cache_options=cache_options, **kwargs, ) + if not ac and "r" not in mode: + self.transaction.files.append(f) if compression is not None: from fsspec.compression import compr from fsspec.core import get_compression @@ -1370,9 +1372,6 @@ def open( compression = get_compression(path, compression) compress = compr[compression] f = compress(f, mode=mode[0]) - - if not ac and "r" not in mode: - self.transaction.files.append(f) return f def touch(self, path, truncate=True, **kwargs):