From 9b64b781ee17d39c1f6e1ae4e997b2158e3cf2ed Mon Sep 17 00:00:00 2001
From: yanghua <yanghua1127@gmail.com>
Date: Wed, 25 Sep 2024 08:30:09 +0800
Subject: [PATCH] Add test cases for stability

---
 tosfs/retry.py            | 11 ++++++++---
 tosfs/tests/test_retry.py | 20 ++++++++++++++++++++
 2 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/tosfs/retry.py b/tosfs/retry.py
index b7291c6..7879ead 100644
--- a/tosfs/retry.py
+++ b/tosfs/retry.py
@@ -144,9 +144,14 @@ def _is_retryable_tos_server_exception(e: TosError) -> bool:
 
 
 def _is_retryable_tos_client_exception(e: TosError) -> bool:
-    return isinstance(e, TosClientError) and any(
-        isinstance(e.cause, excp) for excp in TOS_CLIENT_RETRYABLE_EXCEPTIONS
-    )
+    if isinstance(e, TosClientError):
+        cause = e.cause
+        while cause is not None:
+            for excp in TOS_CLIENT_RETRYABLE_EXCEPTIONS:
+                if isinstance(cause, excp):
+                    return True
+            cause = getattr(cause, "cause", None)
+    return False
 
 
 def _get_sleep_time(err: TosError, retry_count: int) -> float:
diff --git a/tosfs/tests/test_retry.py b/tosfs/tests/test_retry.py
index d69863c..6cb0a25 100644
--- a/tosfs/tests/test_retry.py
+++ b/tosfs/tests/test_retry.py
@@ -59,6 +59,26 @@
             ),
             True,
         ),
+        (
+            TosClientError(
+                "{'message': 'http request timeout', "
+                "'case': \"('Connection aborted.', "
+                "ConnectionResetError(104, 'Connection reset by peer'))\", "
+                "'request_url': "
+                "'http://proton-ci.tos-cn-beijing.volces.com/"
+                "nHnbR/yAlen'}",
+                TosClientError(
+                    "http request timeout",
+                    ConnectionError(
+                        ProtocolError(
+                            "Connection aborted.",
+                            ConnectionResetError(104, "Connection reset by peer"),
+                        )
+                    ),
+                ),
+            ),
+            True,
+        ),
     ],
 )
 def test_is_retry_exception(