From 6394ec86fdc2be6e2d9e1fadb9b48204ce0620b1 Mon Sep 17 00:00:00 2001 From: Deer-Spangle Date: Fri, 15 Mar 2024 10:01:55 +0000 Subject: [PATCH] Retry network failures up to 5 times over up to 3 seconds when trying to fetch a page --- lib/faexport/scraper.rb | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/lib/faexport/scraper.rb b/lib/faexport/scraper.rb index 0f37c5f..353ec67 100644 --- a/lib/faexport/scraper.rb +++ b/lib/faexport/scraper.rb @@ -1302,6 +1302,8 @@ def fetch(path, extra_cookie = nil, as_guest: false) cookie_str = full_cookie(extra_cookie: extra_cookie, as_guest: as_guest) raw = @cache.add("url:#{url}:#{cookie_str}") do start = Time.now + retry_attempts = 0 + retry_sleep_time = 0 begin URI.parse(url).open({ "User-Agent" => USER_AGENT, "Cookie" => "#{cookie_str}" }) do |response| raise FAStatusError.new(url, response.status.join(" ")) if response.status[0] != "200" @@ -1328,8 +1330,29 @@ def fetch(path, extra_cookie = nil, as_guest: false) raise FASlowdownError.new(url) end end + # Retry some types of error + if e.io.status[0] == "502" || e.io.status[0] == "520" + retry_attempts += 1 + if retry_attempts < 5 + sleep(retry_sleep_time) + retry_sleep_time += 0.5 + retry + else + raise + end + end # Raise other HTTP errors as normal raise + rescue Errno::ECONNRESET => e + # Retry connection reset errors + retry_attempts += 1 + if retry_attempts < 5 + sleep(retry_sleep_time) + retry_sleep_time += 0.5 + retry + else + raise + end ensure request_time = Time.now - start $page_request_time.observe(request_time, labels: { page_type: page_type })