Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update flavours #285

Merged
merged 3 commits into from
Sep 9, 2024
Merged

Update flavours #285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e80a9bf
update requirements
ap-- Sep 9, 2024
8a1657b
fix xrootd integration
ap-- Sep 9, 2024
6fedc47
update sources
ap-- Sep 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions dev/fsspec_inspector/generate_flavours.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,10 +169,11 @@ def _fix_oss_file_system(x: str) -> str:

def _fix_xrootd_file_system(x: str) -> str:
x = re.sub(
r"client.URL",
"urlsplit",
r"return client[.]URL\(path\)[.]path_with_params",
"x = urlsplit(path); return (x.path + f'?{x.query}' if x.query else '')",
x,
)
x = re.sub(r"client[.]URL\(u\)", "urlsplit(u)", x)
return re.sub(
"url.hostid",
"url.netloc",
Expand Down
8 changes: 4 additions & 4 deletions dev/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
fsspec[git,hdfs,dask,http,sftp,smb]==2024.6.1
fsspec[git,hdfs,dask,http,sftp,smb]==2024.9.0

# these dependencies define their own filesystems
adlfs==2024.7.0
boxfs==0.3.0
dropboxdrivefs==1.4.1
gcsfs==2024.6.1
s3fs==2024.6.1
gcsfs==2024.9.0.post1
s3fs==2024.9.0
ocifs==1.3.1
webdav4[fsspec]==0.10.0
# gfrivefs @ git+https://github.com/fsspec/gdrivefs@master broken ...
Expand All @@ -14,5 +14,5 @@ dvc==3.55.2
huggingface_hub==0.24.6
lakefs-spec==0.10.0
ossfs==2023.12.0
fsspec-xrootd==0.3.0
fsspec-xrootd==0.4.0
wandbfs==0.0.2
46 changes: 23 additions & 23 deletions upath/_flavour_sources.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ def __init_subclass__(cls: Any, **kwargs):

class AbstractFileSystemFlavour(FileSystemFlavourBase):
__orig_class__ = 'fsspec.spec.AbstractFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol: str | tuple[str, ...] = 'abstract'
root_marker: Literal['', '/'] = ''
sep: Literal['/'] = '/'
Expand Down Expand Up @@ -312,7 +312,7 @@ def _strip_protocol(cls, path) -> str:

class DaskWorkerFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.dask.DaskWorkerFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('dask',)
root_marker = ''
sep = '/'
Expand All @@ -328,15 +328,15 @@ def _get_kwargs_from_urls(path):

class DataFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.data.DataFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('data',)
root_marker = ''
sep = '/'


class DatabricksFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.dbfs.DatabricksFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('dbfs',)
root_marker = ''
sep = '/'
Expand Down Expand Up @@ -369,7 +369,7 @@ class DropboxDriveFileSystemFlavour(AbstractFileSystemFlavour):

class FTPFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.ftp.FTPFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('ftp',)
root_marker = '/'
sep = '/'
Expand All @@ -388,7 +388,7 @@ def _get_kwargs_from_urls(urlpath):

class GCSFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'gcsfs.core.GCSFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0post1'
protocol = ('gs', 'gcs')
root_marker = ''
sep = '/'
Expand Down Expand Up @@ -465,7 +465,7 @@ def _split_path(cls, path, version_aware=False):

class GitFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.git.GitFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('git',)
root_marker = ''
sep = '/'
Expand Down Expand Up @@ -493,7 +493,7 @@ def _get_kwargs_from_urls(path):

class GithubFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.github.GithubFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('github',)
root_marker = ''
sep = '/'
Expand All @@ -518,7 +518,7 @@ def _get_kwargs_from_urls(path):

class HTTPFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.http.HTTPFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('http', 'https')
root_marker = ''
sep = '/'
Expand All @@ -539,7 +539,7 @@ def _parent(cls, path):

class HadoopFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.arrow.HadoopFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('hdfs', 'arrow_hdfs')
root_marker = '/'
sep = '/'
Expand Down Expand Up @@ -580,7 +580,7 @@ class HfFileSystemFlavour(AbstractFileSystemFlavour):

class JupyterFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.jupyter.JupyterFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('jupyter', 'jlab')
root_marker = ''
sep = '/'
Expand All @@ -606,7 +606,7 @@ def _strip_protocol(cls, path):

class LibArchiveFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.libarchive.LibArchiveFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('libarchive',)
root_marker = ''
sep = '/'
Expand All @@ -619,7 +619,7 @@ def _strip_protocol(cls, path):

class LocalFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.local.LocalFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('file', 'local')
root_marker = '/'
sep = '/'
Expand Down Expand Up @@ -697,7 +697,7 @@ def _strip_protocol(cls, path):

class MemoryFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.memory.MemoryFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('memory',)
root_marker = '/'
sep = '/'
Expand Down Expand Up @@ -796,15 +796,15 @@ class OverlayFileSystemFlavour(AbstractFileSystemFlavour):

class ReferenceFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.reference.ReferenceFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('reference',)
root_marker = ''
sep = '/'


class S3FileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 's3fs.core.S3FileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('s3', 's3a')
root_marker = ''
sep = '/'
Expand All @@ -831,7 +831,7 @@ def _get_kwargs_from_urls(urlpath):

class SFTPFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.sftp.SFTPFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('sftp', 'ssh')
root_marker = ''
sep = '/'
Expand All @@ -850,7 +850,7 @@ def _get_kwargs_from_urls(urlpath):

class SMBFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.smb.SMBFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('smb',)
root_marker = ''
sep = '/'
Expand All @@ -870,7 +870,7 @@ def _get_kwargs_from_urls(path):

class TarFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.tar.TarFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('tar',)
root_marker = ''
sep = '/'
Expand All @@ -886,7 +886,7 @@ class WandbFSFlavour(AbstractFileSystemFlavour):

class WebHDFSFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.webhdfs.WebHDFS'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('webhdfs', 'webHDFS')
root_marker = ''
sep = '/'
Expand Down Expand Up @@ -921,7 +921,7 @@ def _strip_protocol(cls, path: str) -> str:

class XRootDFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec_xrootd.xrootd.XRootDFileSystem'
__orig_version__ = '0.3.0'
__orig_version__ = '0.4.0'
protocol = ('root',)
root_marker = '/'
sep = '/'
Expand All @@ -930,7 +930,7 @@ class XRootDFileSystemFlavour(AbstractFileSystemFlavour):
def _strip_protocol(cls, path: str | list[str]) -> Any:
if isinstance(path, str):
if path.startswith(cls.protocol):
return urlsplit(path).path.rstrip("/") or cls.root_marker
x = urlsplit(path); return (x.path + f'?{x.query}' if x.query else '').rstrip("/") or cls.root_marker
# assume already stripped
return path.rstrip("/") or cls.root_marker
elif isinstance(path, list):
Expand All @@ -947,7 +947,7 @@ def _get_kwargs_from_urls(u: str) -> dict[Any, Any]:

class ZipFileSystemFlavour(AbstractFileSystemFlavour):
__orig_class__ = 'fsspec.implementations.zip.ZipFileSystem'
__orig_version__ = '2024.6.1'
__orig_version__ = '2024.9.0'
protocol = ('zip',)
root_marker = ''
sep = '/'
Expand Down