Skip to content

Commit

Permalink
use single cloud_prefix
Browse files Browse the repository at this point in the history
  • Loading branch information
@kujenga
kujenga committed Sep 3, 2024
1 parent 54c37b2 commit 39aeb09
Show file tree
Hide file tree
Showing 7 changed files with 18 additions and 25 deletions.
10 changes: 4 additions & 6 deletions cloudpathlib/azure/azblobclient.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,6 @@ class AzureBlobClient(Client):
authentication options.
"""

cloud_prefix: str = "az://"

def __init__(
self,
account_url: Optional[str] = None,
Expand Down Expand Up @@ -278,13 +276,13 @@ def _list_dir(
) -> Iterable[Tuple[AzureBlobPath, bool]]:
if not cloud_path.container:
for container in self.service_client.list_containers():
yield self.CloudPath(f"{self.cloud_prefix}{container.name}"), True
yield self.CloudPath(f"{cloud_path.cloud_prefix}{container.name}"), True

if not recursive:
continue

yield from self._list_dir(

Check warning on line 284 in cloudpathlib/azure/azblobclient.py

View check run for this annotation

Codecov / codecov/patch

cloudpathlib/azure/azblobclient.py#L284 

Added line #L284 was not covered by tests
self.CloudPath(f"{self.cloud_prefix}{container.name}"), recursive=True
self.CloudPath(f"{cloud_path.cloud_prefix}{container.name}"), recursive=True
)
return

Expand All @@ -300,7 +298,7 @@ def _list_dir(

for path in paths:
yield self.CloudPath(
f"{self.cloud_prefix}{cloud_path.container}/{path.name}"
f"{cloud_path.cloud_prefix}{cloud_path.container}/{path.name}"
), path.is_directory

else:
Expand All @@ -313,7 +311,7 @@ def _list_dir(
# walk_blobs returns folders with a trailing slash
blob_path = blob.name.rstrip("/")
blob_cloud_path = self.CloudPath(
f"{self.cloud_prefix}{cloud_path.container}/{blob_path}"
f"{cloud_path.cloud_prefix}{cloud_path.container}/{blob_path}"
)

yield blob_cloud_path, (
Expand Down
2 changes: 0 additions & 2 deletions cloudpathlib/client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@ class Client(abc.ABC, Generic[BoundedCloudPath]):
_cloud_meta: CloudImplementation
_default_client = None

cloud_prefix: str

def __init__(
self,
file_cache_mode: Optional[Union[str, FileCacheMode]] = None,
Expand Down
12 changes: 5 additions & 7 deletions cloudpathlib/gs/gsclient.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,6 @@ class GSClient(Client):
options.
"""

cloud_prefix: str = "gs://"

def __init__(
self,
application_credentials: Optional[Union[str, os.PathLike]] = None,
Expand Down Expand Up @@ -185,7 +183,7 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat
)

yield from (
(self.CloudPath(f"{self.cloud_prefix}{str(b)}"), True)
(self.CloudPath(f"{cloud_path.cloud_prefix}{str(b)}"), True)
for b in self.client.list_buckets()
)
return
Expand All @@ -204,13 +202,13 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat
if parent not in yielded_dirs and str(parent) != ".":
yield (
self.CloudPath(
f"{self.cloud_prefix}{cloud_path.bucket}/{prefix}{parent}"
f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{prefix}{parent}"
),
True, # is a directory
)
yielded_dirs.add(parent)
yield (
self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{o.name}"),
self.CloudPath(f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{o.name}"),
False,
) # is a file
else:
Expand All @@ -220,13 +218,13 @@ def _list_dir(self, cloud_path: GSPath, recursive=False) -> Iterable[Tuple[GSPat
# see: https://github.com/googleapis/python-storage/issues/863
for file in iterator:
yield (
self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{file.name}"),
self.CloudPath(f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{file.name}"),
False, # is a file
)

for directory in iterator.prefixes:
yield (
self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{directory}"),
self.CloudPath(f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{directory}"),
True, # is a directory
)

Expand Down
1 change: 0 additions & 1 deletion cloudpathlib/local/implementations/azure.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ class LocalAzureBlobClient(LocalClient):
substitute when writing tests.
"""

cloud_prefix: str = "az://"
_cloud_meta = local_azure_blob_implementation

def __init__(self, *args, **kwargs):
Expand Down
1 change: 0 additions & 1 deletion cloudpathlib/local/implementations/gs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ class LocalGSClient(LocalClient):
substitute when writing tests.
"""

cloud_prefix: str = "gs://"
_cloud_meta = local_gs_implementation


Expand Down
1 change: 0 additions & 1 deletion cloudpathlib/local/implementations/s3.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ class LocalS3Client(LocalClient):
substitute when writing tests.
"""

cloud_prefix: str = "s3://"
_cloud_meta = local_s3_implementation


Expand Down
16 changes: 9 additions & 7 deletions cloudpathlib/s3/s3client.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,6 @@ class S3Client(Client):
instances. See documentation for the [`__init__` method][cloudpathlib.s3.s3client.S3Client.__init__]
for detailed authentication options."""

cloud_prefix: str = "s3://"

def __init__(
self,
aws_access_key_id: Optional[str] = None,
Expand Down Expand Up @@ -219,7 +217,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
)

yield from (
(self.CloudPath(f"{self.cloud_prefix}{b['Name']}"), True)
(self.CloudPath(f"{cloud_path.cloud_prefix}{b['Name']}"), True)
for b in self.client.list_buckets().get("Buckets", [])
)
return
Expand All @@ -243,7 +241,9 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
canonical = result_prefix.get("Prefix").rstrip("/") # keep a canonical form
if canonical not in yielded_dirs:
yield (
self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{canonical}"),
self.CloudPath(
f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{canonical}"
),
True,
)
yielded_dirs.add(canonical)
Expand All @@ -257,7 +257,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
if parent_canonical not in yielded_dirs and str(parent) != ".":
yield (
self.CloudPath(
f"{self.cloud_prefix}{cloud_path.bucket}/{parent_canonical}"
f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{parent_canonical}"
),
True,
)
Expand All @@ -271,7 +271,9 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
# s3 fake directories have 0 size and end with "/"
if result_key.get("Key").endswith("/") and result_key.get("Size") == 0:
yield (
self.CloudPath(f"{self.cloud_prefix}{cloud_path.bucket}/{canonical}"),
self.CloudPath(
f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{canonical}"
),
True,
)
yielded_dirs.add(canonical)
Expand All @@ -280,7 +282,7 @@ def _list_dir(self, cloud_path: S3Path, recursive=False) -> Iterable[Tuple[S3Pat
else:
yield (
self.CloudPath(
f"{self.cloud_prefix}{cloud_path.bucket}/{result_key.get('Key')}"
f"{cloud_path.cloud_prefix}{cloud_path.bucket}/{result_key.get('Key')}"
),
False,
)
Expand Down

0 comments on commit 39aeb09

Please sign in to comment.