Consistent glob behaviour for "**" patterns (#143)

* tests: parametrize glob tests

* upath: fix glob behavior

* upath.core: move fsspec glob check to rglob

upath/core.py

@@ -29,6 +29,19 @@
     "UPath",
 ]
 
+_FSSPEC_HAS_WORKING_GLOB = None
+
+
+def _check_fsspec_has_working_glob():
+    global _FSSPEC_HAS_WORKING_GLOB
+    from fsspec.implementations.memory import MemoryFileSystem
+
+    m = type("_M", (MemoryFileSystem,), {"store": {}, "pseudo_dirs": [""]})()
+    m.touch("a.txt")
+    m.touch("f/b.txt")
+    g = _FSSPEC_HAS_WORKING_GLOB = len(m.glob("**/*.txt")) == 2
+    return g
+
 
 class _FSSpecAccessor:
     __slots__ = ("_fs",)
@@ -377,14 +390,29 @@ def glob(self: PT, pattern: str) -> Generator[PT, None, None]:
             yield self._make_child(name)
 
     def rglob(self: PT, pattern: str) -> Generator[PT, None, None]:
-        path_pattern = self.joinpath(pattern)
-        r_path_pattern = self.joinpath("**", pattern)
-        for p in (path_pattern, r_path_pattern):
-            for name in self._accessor.glob(self, p):
+        if _FSSPEC_HAS_WORKING_GLOB is None:
+            _check_fsspec_has_working_glob()
+
+        if _FSSPEC_HAS_WORKING_GLOB:
+            r_path_pattern = self.joinpath("**", pattern)
+            for name in self._accessor.glob(self, r_path_pattern):
                 name = self._sub_path(name)
                 name = name.split(self._flavour.sep)
                 yield self._make_child(name)
 
+        else:
+            path_pattern = self.joinpath(pattern)
+            r_path_pattern = self.joinpath("**", pattern)
+            seen = set()
+            for p in (path_pattern, r_path_pattern):
+                for name in self._accessor.glob(self, p):
+                    name = self._sub_path(name)
+                    name = name.split(self._flavour.sep)
+                    pth = self._make_child(name)
+                    if pth.parts not in seen:
+                        yield pth
+                        seen.add(pth.parts)
+
     def _sub_path(self, name):
         # only want the path name with iterdir
         sp = self._path

upath/tests/cases.py

@@ -4,7 +4,9 @@
 from pathlib import Path
 
 import pytest
+from fsspec import __version__ as fsspec_version
 from fsspec import filesystem
+from packaging.version import Version
 
 from upath import UPath
 
@@ -41,9 +43,22 @@ def test_expanduser(self):
         with pytest.raises(NotImplementedError):
             self.path.expanduser()
 
-    def test_glob(self, pathlib_base):
-        mock_glob = list(self.path.glob("**.txt"))
-        path_glob = list(pathlib_base.glob("**/*.txt"))
+    @pytest.mark.parametrize(
+        "pattern",
+        (
+            "*.txt",
+            "*",
+            pytest.param(
+                "**/*.txt",
+                marks=pytest.mark.xfail()
+                if Version(fsspec_version) < Version("2023.9.0")
+                else (),
+            ),
+        ),
+    )
+    def test_glob(self, pathlib_base, pattern):
+        mock_glob = list(self.path.glob(pattern))
+        path_glob = list(pathlib_base.glob(pattern))
 
         _mock_start = len(self.path.parts)
         mock_glob_normalized = sorted([a.parts[_mock_start:] for a in mock_glob])