GH-36983: [Python] Different get_file_info behaviour between pyarrow.…

…fs.S3FileSystem and s3fs (#37768) ### What changes are included in this PR? Update `test_get_file_info_with_selector ` to check that the base directory is not included in the `S3FileSystem` or `s3fs` filesystem. Also remove old comments and flexible checks. * Closes: #36983 Lead-authored-by: AlenkaF <[email protected]> Co-authored-by: Joris Van den Bossche <[email protected]> Signed-off-by: Joris Van den Bossche <[email protected]>
apache · Oct 10, 2023 · 94b7258 · 94b7258
1 parent 9afa848
commit 94b7258
Showing 1 changed file with 3 additions and 12 deletions.
diff --git a/python/pyarrow/tests/test_fs.py b/python/pyarrow/tests/test_fs.py
@@ -699,10 +699,8 @@ def test_get_file_info_with_selector(fs, pathfn):
 
         infos = fs.get_file_info(selector)
         if fs.type_name == "py::fsspec+s3":
-            # s3fs only lists directories if they are not empty, but depending
-            # on the s3fs/fsspec version combo, it includes the base_dir
-            # (https://github.com/dask/s3fs/issues/393)
-            assert (len(infos) == 4) or (len(infos) == 5)
+            # s3fs only lists directories if they are not empty
+            len(infos) == 4
         else:
             assert len(infos) == 5
 
@@ -713,10 +711,6 @@ def test_get_file_info_with_selector(fs, pathfn):
             elif (info.path.rstrip("/").endswith(dir_a) or
                   info.path.rstrip("/").endswith(dir_b)):
                 assert info.type == FileType.Directory
-            elif (fs.type_name == "py::fsspec+s3" and
-                  info.path.rstrip("/").endswith("selector-dir")):
-                # s3fs can include base dir, see above
-                assert info.type == FileType.Directory
             else:
                 raise ValueError('unexpected path {}'.format(info.path))
             check_mtime_or_absent(info)
@@ -727,10 +721,7 @@ def test_get_file_info_with_selector(fs, pathfn):
         infos = fs.get_file_info(selector)
         if fs.type_name == "py::fsspec+s3":
             # s3fs only lists directories if they are not empty
-            # + for s3fs 0.5.2 all directories are dropped because of buggy
-            # side-effect of previous find() call
-            # (https://github.com/dask/s3fs/issues/410)
-            assert (len(infos) == 3) or (len(infos) == 2)
+            assert len(infos) == 3
         else:
             assert len(infos) == 4