From 309711f12e6ab07114a8d7e9c8210798aebae861 Mon Sep 17 00:00:00 2001 From: Cyril Date: Thu, 14 Nov 2024 15:30:18 +0100 Subject: [PATCH 1/6] Add tree method to display tree-like structure of the filesystem MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit >>> fs.tree(path='/start/folder', display_size=True) /start/folder ├── folder1 │ ├── file1.txt (1.234MB) │ └── file2.txt (0.567MB) └── folder2 └── file3.txt (2.345MB) --- fsspec/spec.py | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/fsspec/spec.py b/fsspec/spec.py index 9659f2e98..2c3b48820 100644 --- a/fsspec/spec.py +++ b/fsspec/spec.py @@ -1567,6 +1567,79 @@ def modified(self, path): """Return the modified timestamp of a file as a datetime.datetime""" raise NotImplementedError + def tree( + self, + path: str = '/', + recursion_limit: int = 2, + max_display: int = 25, + display_size: bool = False, + prefix: str = "", + is_last: bool = True, + first: bool = True + ): + """ + Display a tree-like structure of the filesystem starting from the given path. + + Parameters + ---------- + path: Root path to start traversal from + recursion_limit: Maximum depth of directory traversal + max_display: Maximum number of items to display per directory + display_size: Whether to display file sizes + prefix: Current line prefix for visual tree structure + is_last: Whether current item is last in its level + first: Whether this is the first call (displays root path) + + Example + ------- + >>> fs.tree(path='/start/folder', display_size=True) + + /start/folder + ├── folder1 + │ ├── file1.txt (1.234MB) + │ └── file2.txt (0.567MB) + └── folder2 + └── file3.txt (2.345MB) + """ + if first: + print(path) + if recursion_limit: + try: + contents = self.ls(path, detail=True) + contents.sort(key=lambda x: (not x.get('type') == 'directory', x.get('name', ''))) + + if max_display is not None and len(contents) > max_display: + displayed_contents = contents[:max_display] + remaining_count = len(contents) - max_display + else: + displayed_contents = contents + remaining_count = 0 + + for i, item in enumerate(displayed_contents): + is_last_item = (i == len(displayed_contents) - 1) and (remaining_count == 0) + + branch = "└── " if is_last_item else "├── " + new_prefix = prefix + (" " if is_last_item else "│ ") + + name = os.path.basename(item.get('name', '')) + size = f" ({item.get('size', 0) / 2**20:.3f}Mb)" if display_size and item.get('type') == 'file' else "" + + print(f"{prefix}{branch}{name}{size}") + + if item.get('type') == 'directory' and recursion_limit > 0: + self.tree(item.get('name', ''), recursion_limit - 1, max_display, new_prefix, is_last_item, display_size=display_size, first=False) + + if remaining_count > 0: + more_message = f"{remaining_count} more item(s) not displayed." + print(f"{prefix}{'└── ' if is_last else '├── '}{more_message}") + + except FileNotFoundError: + print(f"{prefix}Error: Path not found - {path}") + except PermissionError: + print(f"{prefix}Error: Permission denied - {path}") + except Exception as e: + print(f"{prefix}Unexpected error: {str(e)}") + # ------------------------------------------------------------------------ # Aliases From ca16769f4fbd4409dec9ff7ce179ec0f24b18270 Mon Sep 17 00:00:00 2001 From: Cyril Date: Thu, 14 Nov 2024 17:28:05 +0100 Subject: [PATCH 2/6] indent + smart unit + remove try except --- fsspec/spec.py | 86 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 51 insertions(+), 35 deletions(-) diff --git a/fsspec/spec.py b/fsspec/spec.py index 2c3b48820..1977a3d7c 100644 --- a/fsspec/spec.py +++ b/fsspec/spec.py @@ -1575,7 +1575,8 @@ def tree( display_size: bool = False, prefix: str = "", is_last: bool = True, - first: bool = True + first: bool = True, + indent_size: int = 4 ): """ Display a tree-like structure of the filesystem starting from the given path. @@ -1589,6 +1590,7 @@ def tree( prefix: Current line prefix for visual tree structure is_last: Whether current item is last in its level first: Whether this is the first call (displays root path) + indent_size: Number of spaces by indent Example ------- @@ -1601,44 +1603,58 @@ def tree( └── folder2 └── file3.txt (2.345MB) """ + def format_bytes(n: int) -> str: + """Format bytes as text.""" + for prefix, k in ( + ("P", 2**50), + ("T", 2**40), + ("G", 2**30), + ("M", 2**20), + ("k", 2**10), + ): + if n >= 0.9*k: + return f"{n / k:.2f} {prefix}b" + return f"{n}B" + if first: print(path) if recursion_limit: - try: - contents = self.ls(path, detail=True) - contents.sort(key=lambda x: (not x.get('type') == 'directory', x.get('name', ''))) - - if max_display is not None and len(contents) > max_display: - displayed_contents = contents[:max_display] - remaining_count = len(contents) - max_display - else: - displayed_contents = contents - remaining_count = 0 - - for i, item in enumerate(displayed_contents): - is_last_item = (i == len(displayed_contents) - 1) and (remaining_count == 0) - - branch = "└── " if is_last_item else "├── " - new_prefix = prefix + (" " if is_last_item else "│ ") - - name = os.path.basename(item.get('name', '')) - size = f" ({item.get('size', 0) / 2**20:.3f}Mb)" if display_size and item.get('type') == 'file' else "" + indent = " " * indent_size + contents = self.ls(path, detail=True) + contents.sort(key=lambda x: (not x.get('type') == 'directory', x.get('name', ''))) - print(f"{prefix}{branch}{name}{size}") - - if item.get('type') == 'directory' and recursion_limit > 0: - self.tree(item.get('name', ''), recursion_limit - 1, max_display, new_prefix, is_last_item, display_size=display_size, first=False) - - if remaining_count > 0: - more_message = f"{remaining_count} more item(s) not displayed." - print(f"{prefix}{'└── ' if is_last else '├── '}{more_message}") - - except FileNotFoundError: - print(f"{prefix}Error: Path not found - {path}") - except PermissionError: - print(f"{prefix}Error: Permission denied - {path}") - except Exception as e: - print(f"{prefix}Unexpected error: {str(e)}") + if max_display is not None and len(contents) > max_display: + displayed_contents = contents[:max_display] + remaining_count = len(contents) - max_display + else: + displayed_contents = contents + remaining_count = 0 + + for i, item in enumerate(displayed_contents): + is_last_item = (i == len(displayed_contents) - 1) and (remaining_count == 0) + + branch = "└"+('─'*(indent_size-2)) if is_last_item else "├"+('─'*(indent_size-2)) + branch += ' ' + new_prefix = prefix + (indent if is_last_item else "│" + " " * (indent_size - 1)) + + name = os.path.basename(item.get('name', '')) + size = f" ({format_bytes(item.get('size', 0))})" if display_size and item.get('type') == 'file' else "" + + print(f"{prefix}{branch}{name}{size}") + + if item.get('type') == 'directory' and recursion_limit > 0: + self.tree(path=item.get('name', ''), + recursion_limit=recursion_limit - 1, + max_display=max_display, + display_size=display_size, + prefix=new_prefix, + is_last=is_last_item, + first=False, + indent_size=indent_size) + + if remaining_count > 0: + more_message = f"{remaining_count} more item(s) not displayed." + print(f"{prefix}{'└── ' if is_last else '├── '}{more_message}") # ------------------------------------------------------------------------ # Aliases From a8d449b672b9c6a151858f3dcbab2ffb44510f68 Mon Sep 17 00:00:00 2001 From: Cyril Date: Fri, 15 Nov 2024 12:20:25 +0100 Subject: [PATCH 3/6] str output --- fsspec/spec.py | 68 ++++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/fsspec/spec.py b/fsspec/spec.py index 1977a3d7c..a063a9413 100644 --- a/fsspec/spec.py +++ b/fsspec/spec.py @@ -1577,10 +1577,10 @@ def tree( is_last: bool = True, first: bool = True, indent_size: int = 4 - ): + ) -> str: """ - Display a tree-like structure of the filesystem starting from the given path. - + Return a tree-like structure of the filesystem starting from the given path as a string. + Parameters ---------- path: Root path to start traversal from @@ -1591,11 +1591,16 @@ def tree( is_last: Whether current item is last in its level first: Whether this is the first call (displays root path) indent_size: Number of spaces by indent - + + Returns + ------- + str: A string representing the tree structure. + Example ------- - >>> fs.tree(path='/start/folder', display_size=True) - + >>> tree = fs.tree(path='/start/folder', display_size=True) + >>> print(tree) + /start/folder ├── folder1 │ ├── file1.txt (1.234MB) @@ -1612,49 +1617,58 @@ def format_bytes(n: int) -> str: ("M", 2**20), ("k", 2**10), ): - if n >= 0.9*k: + if n >= 0.9 * k: return f"{n / k:.2f} {prefix}b" return f"{n}B" - + + result = [] + if first: - print(path) + result.append(path) + if recursion_limit: indent = " " * indent_size contents = self.ls(path, detail=True) contents.sort(key=lambda x: (not x.get('type') == 'directory', x.get('name', ''))) - + if max_display is not None and len(contents) > max_display: displayed_contents = contents[:max_display] remaining_count = len(contents) - max_display else: displayed_contents = contents remaining_count = 0 - + for i, item in enumerate(displayed_contents): is_last_item = (i == len(displayed_contents) - 1) and (remaining_count == 0) - - branch = "└"+('─'*(indent_size-2)) if is_last_item else "├"+('─'*(indent_size-2)) + + branch = "└" + ('─' * (indent_size - 2)) if is_last_item else "├" + ('─' * (indent_size - 2)) branch += ' ' new_prefix = prefix + (indent if is_last_item else "│" + " " * (indent_size - 1)) - + name = os.path.basename(item.get('name', '')) size = f" ({format_bytes(item.get('size', 0))})" if display_size and item.get('type') == 'file' else "" - - print(f"{prefix}{branch}{name}{size}") - + + result.append(f"{prefix}{branch}{name}{size}") + if item.get('type') == 'directory' and recursion_limit > 0: - self.tree(path=item.get('name', ''), - recursion_limit=recursion_limit - 1, - max_display=max_display, - display_size=display_size, - prefix=new_prefix, - is_last=is_last_item, - first=False, - indent_size=indent_size) - + result.append( + self.tree( + path=item.get('name', ''), + recursion_limit=recursion_limit - 1, + max_display=max_display, + display_size=display_size, + prefix=new_prefix, + is_last=is_last_item, + first=False, + indent_size=indent_size + ) + ) + if remaining_count > 0: more_message = f"{remaining_count} more item(s) not displayed." - print(f"{prefix}{'└── ' if is_last else '├── '}{more_message}") + result.append(f"{prefix}{'└── ' if is_last else '├── '}{more_message}") + + return "\n".join((_ for _ in result if _)) # ------------------------------------------------------------------------ # Aliases From 23807edc96e6def5af4551450ed0c0815e99be95 Mon Sep 17 00:00:00 2001 From: Cyril Date: Fri, 15 Nov 2024 15:43:40 +0100 Subject: [PATCH 4/6] number of immediate children --- fsspec/spec.py | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/fsspec/spec.py b/fsspec/spec.py index a063a9413..9a69b1633 100644 --- a/fsspec/spec.py +++ b/fsspec/spec.py @@ -1595,18 +1595,14 @@ def tree( Returns ------- str: A string representing the tree structure. - + Example ------- - >>> tree = fs.tree(path='/start/folder', display_size=True) + >>> from fsspec import filesystem + + >>> fs = filesystem('ftp', host='test.rebex.net', user='demo', password='password') + >>> tree = fs.tree(display_size=True, recursion_limit=3, indent_size=8, max_display=10) >>> print(tree) - - /start/folder - ├── folder1 - │ ├── file1.txt (1.234MB) - │ └── file2.txt (0.567MB) - └── folder2 - └── file3.txt (2.345MB) """ def format_bytes(n: int) -> str: """Format bytes as text.""" @@ -1622,10 +1618,10 @@ def format_bytes(n: int) -> str: return f"{n}B" result = [] - + if first: result.append(path) - + if recursion_limit: indent = " " * indent_size contents = self.ls(path, detail=True) @@ -1646,7 +1642,24 @@ def format_bytes(n: int) -> str: new_prefix = prefix + (indent if is_last_item else "│" + " " * (indent_size - 1)) name = os.path.basename(item.get('name', '')) - size = f" ({format_bytes(item.get('size', 0))})" if display_size and item.get('type') == 'file' else "" + + if display_size and item.get('type') == 'directory': + sub_contents = self.ls(item.get('name', ''), detail=True) + num_files = sum(1 for sub_item in sub_contents if sub_item.get('type') == 'file') + num_folders = sum(1 for sub_item in sub_contents if sub_item.get('type') == 'directory') + + if num_files == 0 and num_folders == 0: + size = " (empty folder)" + elif num_files == 0: + size = f" ({num_folders} subfolder{'s' if num_folders > 1 else ''})" + elif num_folders == 0: + size = f" ({num_files} file{'s' if num_files > 1 else ''})" + else: + size = f" ({num_files} file{'s' if num_files > 1 else ''}, {num_folders} subfolder{'s' if num_folders > 1 else ''})" + elif display_size and item.get('type') == 'file': + size = f" ({format_bytes(item.get('size', 0))})" + else: + size = "" result.append(f"{prefix}{branch}{name}{size}") @@ -1666,7 +1679,7 @@ def format_bytes(n: int) -> str: if remaining_count > 0: more_message = f"{remaining_count} more item(s) not displayed." - result.append(f"{prefix}{'└── ' if is_last else '├── '}{more_message}") + result.append(f"{prefix}{"└" + ('─' * (indent_size - 2))} {more_message}") return "\n".join((_ for _ in result if _)) From 9d25232850a1ddb370e0d62a8ce8e4aa0d89a99a Mon Sep 17 00:00:00 2001 From: Martin Durant Date: Fri, 22 Nov 2024 09:58:15 -0500 Subject: [PATCH 5/6] Update fsspec/spec.py --- fsspec/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fsspec/spec.py b/fsspec/spec.py index 80068496e..a4351d10b 100644 --- a/fsspec/spec.py +++ b/fsspec/spec.py @@ -1688,7 +1688,7 @@ def format_bytes(n: int) -> str: if remaining_count > 0: more_message = f"{remaining_count} more item(s) not displayed." - result.append(f"{prefix}{"└" + ('─' * (indent_size - 2))} {more_message}") + result.append(f"{prefix}{'└' + ('─' * (indent_size - 2))} {more_message}") return "\n".join((_ for _ in result if _)) From 240b04f8d0670a7816709a37bec2b14e3d3a0fc0 Mon Sep 17 00:00:00 2001 From: CyrilJl Date: Fri, 22 Nov 2024 16:52:33 +0100 Subject: [PATCH 6/6] pre-commit --- fsspec/spec.py | 101 +++++++++++++++++++++++++++++-------------------- 1 file changed, 60 insertions(+), 41 deletions(-) diff --git a/fsspec/spec.py b/fsspec/spec.py index a4351d10b..921268df3 100644 --- a/fsspec/spec.py +++ b/fsspec/spec.py @@ -1577,19 +1577,19 @@ def modified(self, path): raise NotImplementedError def tree( - self, - path: str = '/', - recursion_limit: int = 2, - max_display: int = 25, - display_size: bool = False, - prefix: str = "", - is_last: bool = True, - first: bool = True, - indent_size: int = 4 + self, + path: str = "/", + recursion_limit: int = 2, + max_display: int = 25, + display_size: bool = False, + prefix: str = "", + is_last: bool = True, + first: bool = True, + indent_size: int = 4, ) -> str: """ Return a tree-like structure of the filesystem starting from the given path as a string. - + Parameters ---------- path: Root path to start traversal from @@ -1597,10 +1597,10 @@ def tree( max_display: Maximum number of items to display per directory display_size: Whether to display file sizes prefix: Current line prefix for visual tree structure - is_last: Whether current item is last in its level + is_last: Whether current item is last in its level first: Whether this is the first call (displays root path) indent_size: Number of spaces by indent - + Returns ------- str: A string representing the tree structure. @@ -1613,6 +1613,7 @@ def tree( >>> tree = fs.tree(display_size=True, recursion_limit=3, indent_size=8, max_display=10) >>> print(tree) """ + def format_bytes(n: int) -> str: """Format bytes as text.""" for prefix, k in ( @@ -1625,38 +1626,54 @@ def format_bytes(n: int) -> str: if n >= 0.9 * k: return f"{n / k:.2f} {prefix}b" return f"{n}B" - + result = [] - + if first: result.append(path) - + if recursion_limit: indent = " " * indent_size contents = self.ls(path, detail=True) - contents.sort(key=lambda x: (not x.get('type') == 'directory', x.get('name', ''))) - + contents.sort( + key=lambda x: (x.get("type") != "directory", x.get("name", "")) + ) + if max_display is not None and len(contents) > max_display: displayed_contents = contents[:max_display] remaining_count = len(contents) - max_display else: displayed_contents = contents remaining_count = 0 - + for i, item in enumerate(displayed_contents): - is_last_item = (i == len(displayed_contents) - 1) and (remaining_count == 0) - - branch = "└" + ('─' * (indent_size - 2)) if is_last_item else "├" + ('─' * (indent_size - 2)) - branch += ' ' - new_prefix = prefix + (indent if is_last_item else "│" + " " * (indent_size - 1)) - - name = os.path.basename(item.get('name', '')) - - if display_size and item.get('type') == 'directory': - sub_contents = self.ls(item.get('name', ''), detail=True) - num_files = sum(1 for sub_item in sub_contents if sub_item.get('type') == 'file') - num_folders = sum(1 for sub_item in sub_contents if sub_item.get('type') == 'directory') - + is_last_item = (i == len(displayed_contents) - 1) and ( + remaining_count == 0 + ) + + branch = ( + "└" + ("─" * (indent_size - 2)) + if is_last_item + else "├" + ("─" * (indent_size - 2)) + ) + branch += " " + new_prefix = prefix + ( + indent if is_last_item else "│" + " " * (indent_size - 1) + ) + + name = os.path.basename(item.get("name", "")) + + if display_size and item.get("type") == "directory": + sub_contents = self.ls(item.get("name", ""), detail=True) + num_files = sum( + 1 for sub_item in sub_contents if sub_item.get("type") == "file" + ) + num_folders = sum( + 1 + for sub_item in sub_contents + if sub_item.get("type") == "directory" + ) + if num_files == 0 and num_folders == 0: size = " (empty folder)" elif num_files == 0: @@ -1665,32 +1682,34 @@ def format_bytes(n: int) -> str: size = f" ({num_files} file{'s' if num_files > 1 else ''})" else: size = f" ({num_files} file{'s' if num_files > 1 else ''}, {num_folders} subfolder{'s' if num_folders > 1 else ''})" - elif display_size and item.get('type') == 'file': + elif display_size and item.get("type") == "file": size = f" ({format_bytes(item.get('size', 0))})" else: size = "" - + result.append(f"{prefix}{branch}{name}{size}") - - if item.get('type') == 'directory' and recursion_limit > 0: + + if item.get("type") == "directory" and recursion_limit > 0: result.append( self.tree( - path=item.get('name', ''), + path=item.get("name", ""), recursion_limit=recursion_limit - 1, max_display=max_display, display_size=display_size, prefix=new_prefix, is_last=is_last_item, first=False, - indent_size=indent_size + indent_size=indent_size, ) ) - + if remaining_count > 0: more_message = f"{remaining_count} more item(s) not displayed." - result.append(f"{prefix}{'└' + ('─' * (indent_size - 2))} {more_message}") - - return "\n".join((_ for _ in result if _)) + result.append( + f"{prefix}{'└' + ('─' * (indent_size - 2))} {more_message}" + ) + + return "\n".join(_ for _ in result if _) # ------------------------------------------------------------------------ # Aliases