From 1d1b526849fafc2be4ab51358af3d66fd0fa8ee7 Mon Sep 17 00:00:00 2001
From: yanghua <yanghua1127@gmail.com>
Date: Wed, 30 Oct 2024 19:21:28 +0800
Subject: [PATCH] Walk via multiple-threads

---
 tosfs/compatible.py | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/tosfs/compatible.py b/tosfs/compatible.py
index faf393b..c1fd805 100644
--- a/tosfs/compatible.py
+++ b/tosfs/compatible.py
@@ -15,6 +15,7 @@
 """The compatible module about AbstractFileSystem in fsspec."""
 import os
 import re
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Any, Optional, Union
 
 from fsspec import AbstractFileSystem
@@ -194,14 +195,21 @@ def walk(  # noqa
                     yield path, dirs, files
                 return
 
-        for d in dirs:
-            yield from self.walk(
-                full_dirs[d],
-                maxdepth=maxdepth,
-                detail=detail,
-                topdown=topdown,
-                **kwargs,
-            )
+        with ThreadPoolExecutor() as executor:
+            futures = {
+                executor.submit(
+                    self.walk,
+                    full_dirs[d],
+                    maxdepth=maxdepth,
+                    detail=detail,
+                    topdown=topdown,
+                    **kwargs,
+                ): d
+                for d in dirs
+            }
+            for future in as_completed(futures):
+                for result in future.result():
+                    yield result
 
         if not topdown:
             # Yield after recursion if walking bottom up