Rework exception handling to properly capture the exception and backo…

…ff only when needed

scraper/src/mindtouch2zim/asset.py

@@ -7,15 +7,15 @@
 from kiwixstorage import KiwixStorage, NotFoundError
 from pif import get_public_ip
 from PIL import Image
-from requests.exceptions import HTTPError, RequestException
-from urllib3.exceptions import HTTPError as BaseHTTPError
+from requests.exceptions import RequestException
 from zimscraperlib.download import stream_file
 from zimscraperlib.image.optimization import optimize_webp
 from zimscraperlib.image.presets import WebpMedium
 from zimscraperlib.rewriting.url_rewriting import HttpUrl, ZimPath
 from zimscraperlib.zim import Creator
 
 from mindtouch2zim.constants import KNOWN_BAD_ASSETS_REGEX, logger, web_session
+from mindtouch2zim.errors import KnownBadAssetFailedError
 from mindtouch2zim.utils import backoff_hdlr
 
 SUPPORTED_IMAGE_MIME_TYPES = {
@@ -77,12 +77,6 @@ def process_asset(
             asset_path=asset_path, asset_details=asset_details, creator=creator
         )
 
-    @backoff.on_exception(
-        backoff.expo,
-        RequestException,
-        max_time=16,
-        on_backoff=backoff_hdlr,
-    )
     def _process_asset_internal(
         self,
         asset_path: ZimPath,
@@ -104,12 +98,9 @@ def _process_asset_internal(
                     content=asset_content.getvalue(),
                 )
                 break  # file found and added
-            except (HTTPError, BaseHTTPError) as exc:
-                if self.bad_assets_regex and self.bad_assets_regex.findall(
-                    asset_url.value
-                ):
-                    logger.debug(f"Ignoring asset for {asset_url.value}: {exc}")
-                    continue
+            except KnownBadAssetFailedError as exc:
+                logger.debug(f"Ignoring known bad asset for {asset_url.value}: {exc}")
+            except RequestException as exc:
                 with self.lock:
                     self.bad_assets_count += 1
                     if (
@@ -239,6 +230,12 @@ def _download_from_online(self, asset_url: HttpUrl) -> BytesIO:
         )
         return asset_content
 
+    @backoff.on_exception(
+        backoff.expo,
+        RequestException,
+        max_time=2,  # secs
+        on_backoff=backoff_hdlr,
+    )
     def get_asset_content(
         self, asset_path: ZimPath, asset_url: HttpUrl, *, always_fetch_online: bool
     ) -> BytesIO:
@@ -257,7 +254,14 @@ def get_asset_content(
                 else:
                     logger.debug(f"Not optimizing, unsupported mime type: {mime_type}")
 
-        return self._download_from_online(asset_url=asset_url)
+        try:
+            return self._download_from_online(asset_url=asset_url)
+        except RequestException as exc:
+            # check this early and raise custom exception to escape backoff (not need to
+            # loose time on assets which are expected to be bad)
+            if self.bad_assets_regex and self.bad_assets_regex.findall(asset_url.value):
+                raise KnownBadAssetFailedError() from exc
+            raise
 
     def _setup_s3(self):
         if not self.s3_url_with_credentials:

scraper/src/mindtouch2zim/errors.py

@@ -8,3 +8,9 @@ class NoIllustrationFoundError(Exception):
     """An exception raised when no suitable illustration has been found"""
 
     pass
+
+
+class KnownBadAssetFailedError(Exception):
+    """An exception raised when an asset known to be failing, failed as expected"""
+
+    pass