attempt to gracefully return 429 errors when uploading assets (#1118)

src/api/impl/ConnectedServer.cpp

@@ -131,6 +131,12 @@ _NODISCARD static bool GetJson([[maybe_unused]] _In_ const char* sApiName,
     if (pDocument.HasMember("Error"))
     {
         pResponse.ErrorMessage = pDocument["Error"].GetString();
+        if (httpResponse.StatusCode() == ra::services::Http::StatusCode::TooManyRequests)
+        {
+            pResponse.Result = ApiResult::Incomplete;
+            return false;
+        }
+
         if (!pResponse.ErrorMessage.empty())
         {
             pResponse.Result = ApiResult::Error;
@@ -382,15 +388,24 @@ static bool ValidateResponse(int nResult, const rc_api_response_t& api_response,
         if (api_response.error_message && *api_response.error_message)
         {
             pResponse.ErrorMessage = api_response.error_message;
-            pResponse.Result = ApiResult::Error;
-            RA_LOG_ERR("-- %s Error: %s", sApiName, pResponse.ErrorMessage);
+
+            if (nStatusCode == ra::services::Http::StatusCode::TooManyRequests)
+            {
+                pResponse.Result = ApiResult::Incomplete;
+            }
+            else
+            {
+                pResponse.Result = ApiResult::Error;
+                RA_LOG_ERR("-- %s Error: %s", sApiName, pResponse.ErrorMessage);
+            }
         }
         else
         {
             pResponse.Result = ApiResult::Failed;
             RA_LOG_ERR("-- %s Error: Success=false", sApiName, pResponse.ErrorMessage);
         }
 
+
         return false;
     }
 
@@ -1075,7 +1090,9 @@ UploadBadge::Response ConnectedServer::UploadBadge(const UploadBadge::Request& r
     {
         if (!document.HasMember("Response"))
         {
-            response.Result = ApiResult::Error;
+            if (response.Result == ApiResult::None)
+                response.Result = ApiResult::Error;
+
             if (response.ErrorMessage.empty())
                 response.ErrorMessage = ra::StringPrintf("%s not found in response", "Response");
         }

src/ui/viewmodels/AssetUploadViewModel.cpp

@@ -183,13 +183,22 @@ void AssetUploadViewModel::UploadBadge(const std::wstring& sBadge)
     ra::api::UploadBadge::Request request;
     request.ImageFilePath = sFilename;
 
-    std::vector<ra::data::models::AchievementModel*> vAffectedAchievements;
+    ra::api::UploadBadge::Response response;
     {
         // only allow one badge upload at a time to prevent a race condition on server that could
         // result in non-unique image IDs being returned.
         std::lock_guard<std::mutex> pLock(m_pMutex);
-        const auto response = request.Call();
+        response = request.Call();
+    }
 
+    if (response.Result == ra::api::ApiResult::Incomplete)
+    {
+        Rest();
+        UploadBadge(sBadge);
+        return;
+    }
+    std::vector<ra::data::models::AchievementModel*> vAffectedAchievements;
+    {
         // if the upload succeeded, update the badge property for each associated achievement and queue the
         // achievement update. if the upload failed, set the error message and don't queue the achievement update.
         for (auto& pItem : m_vUploadQueue)
@@ -278,6 +287,12 @@ void AssetUploadViewModel::UploadAchievement(ra::data::models::AchievementModel&
         pAchievement.UpdateLocalCheckpoint();
         pAchievement.UpdateServerCheckpoint();
     }
+    else if (response.Result == ra::api::ApiResult::Incomplete)
+    {
+        Rest();
+        UploadAchievement(pAchievement);
+        return;
+    }
 
     // update the queue
     std::lock_guard<std::mutex> pLock(m_pMutex);
@@ -322,6 +337,12 @@ void AssetUploadViewModel::UploadLeaderboard(ra::data::models::LeaderboardModel&
         pLeaderboard.UpdateLocalCheckpoint();
         pLeaderboard.UpdateServerCheckpoint();
     }
+    else if (response.Result == ra::api::ApiResult::Incomplete)
+    {
+        Rest();
+        UploadLeaderboard(pLeaderboard);
+        return;
+    }
 
     // update the queue
     std::lock_guard<std::mutex> pLock(m_pMutex);
@@ -355,6 +376,12 @@ void AssetUploadViewModel::UploadCodeNote(ra::data::models::CodeNotesModel& pNot
             pNotes.SetServerCodeNote(nAddress, L"");
             nState = UploadState::Success;
         }
+        else if (response.Result == ra::api::ApiResult::Incomplete)
+        {
+            Rest();
+            UploadCodeNote(pNotes, nAddress);
+            return;
+        }
 
         sErrorMessage = response.ErrorMessage;
     }
@@ -372,6 +399,12 @@ void AssetUploadViewModel::UploadCodeNote(ra::data::models::CodeNotesModel& pNot
             pNotes.SetServerCodeNote(nAddress, *pNote);
             nState = UploadState::Success;
         }
+        else if (response.Result == ra::api::ApiResult::Incomplete)
+        {
+            Rest();
+            UploadCodeNote(pNotes, nAddress);
+            return;
+        }
 
         sErrorMessage = response.ErrorMessage;
     }

src/ui/viewmodels/AssetUploadViewModel.hh

@@ -30,6 +30,13 @@ public:
 protected:
     void OnBegin() override;
 
+    virtual void Rest() const noexcept
+    {
+        // sleep 500-1500ms to allow server to stop throttling us.
+        // stagger value so we don't slam it in batches.
+        Sleep(rand() % 1000 + 500);
+    }
+
 private:
     enum class UploadState
     {

tests/ui/viewmodels/AssetUploadViewModel_Tests.cpp

@@ -220,6 +220,9 @@ TEST_CLASS(AssetUploadViewModel_Tests)
             Assert::IsTrue(bDialogSeen);
         }
 
+    protected:
+        void Rest() const noexcept override {}
+
     private:
         ra::data::context::GameAssets m_pAssets;
     };
@@ -707,6 +710,68 @@ TEST_CLASS(AssetUploadViewModel_Tests)
         vmUpload.AssertSuccess(2);
     }
 
+    TEST_METHOD(TestMultipleCoreAchievementsWithImages429)
+    {
+        AssetUploadViewModelHarness vmUpload;
+        auto& pAchievement1 = vmUpload.AddAchievement(AssetCategory::Core, 5, L"Title1", L"Desc1", L"local\\12345", "0xH1234=1");
+        auto& pAchievement2 = vmUpload.AddAchievement(AssetCategory::Core, 5, L"Title2", L"Desc2", L"local\\22222", "0xH1234=1");
+        Assert::AreEqual(AssetChanges::Unpublished, pAchievement1.GetChanges());
+        Assert::AreEqual(AssetChanges::Unpublished, pAchievement2.GetChanges());
+
+        vmUpload.QueueAsset(pAchievement1);
+        vmUpload.QueueAsset(pAchievement2);
+        Assert::AreEqual({ 2U }, vmUpload.TaskCount());
+
+        int nImagesUploaded = 0;
+        vmUpload.mockServer.HandleRequest<ra::api::UploadBadge>([&nImagesUploaded]
+                (const ra::api::UploadBadge::Request& pRequest, ra::api::UploadBadge::Response& pResponse)
+        {
+            ++nImagesUploaded;
+
+            if (pRequest.ImageFilePath == L"RACache\\Badges\\local\\12345")
+                pResponse.BadgeId = "76543";
+            else
+                pResponse.BadgeId = "55555";
+
+            if (nImagesUploaded % 2 == 0)
+                pResponse.Result = ra::api::ApiResult::Incomplete;
+            else
+                pResponse.Result = ra::api::ApiResult::Success;
+            return true;
+        });
+
+        int nAchievementsUploaded = 0;
+        vmUpload.mockServer.HandleRequest<ra::api::UpdateAchievement>([&nAchievementsUploaded]
+                (const ra::api::UpdateAchievement::Request& pRequest, ra::api::UpdateAchievement::Response& pResponse)
+        {
+            ++nAchievementsUploaded;
+
+            if (pRequest.Title == L"Title1")
+                Assert::AreEqual(std::string("76543"), pRequest.Badge);
+            else
+                Assert::AreEqual(std::string("55555"), pRequest.Badge);
+
+            pResponse.AchievementId = pRequest.AchievementId;
+
+            if (nAchievementsUploaded % 2 == 0)
+                pResponse.Result = ra::api::ApiResult::Incomplete;
+            else
+                pResponse.Result = ra::api::ApiResult::Success;
+            return true;
+        });
+
+        vmUpload.DoUpload();
+
+        Assert::AreEqual(3, nImagesUploaded);
+        Assert::AreEqual(3, nAchievementsUploaded);
+        Assert::AreEqual(AssetChanges::None, pAchievement1.GetChanges());
+        Assert::AreEqual(std::wstring(L"76543"), pAchievement1.GetBadge());
+        Assert::AreEqual(AssetChanges::None, pAchievement2.GetChanges());
+        Assert::AreEqual(std::wstring(L"55555"), pAchievement2.GetBadge());
+
+        vmUpload.AssertSuccess(2);
+    }
+
     TEST_METHOD(TestMultipleCoreAchievementsWithSameImage)
     {
         AssetUploadViewModelHarness vmUpload;
@@ -1220,6 +1285,51 @@ TEST_CLASS(AssetUploadViewModel_Tests)
 
         vmUpload.AssertSuccess(2);
     }
+
+    TEST_METHOD(TestMultipleCodeNotes429)
+    {
+        AssetUploadViewModelHarness vmUpload;
+        vmUpload.CodeNotes().SetCodeNote(0x1234, L"This is a note.");
+        vmUpload.CodeNotes().SetCodeNote(0x1235, L"This is another note.");
+        Assert::AreEqual(AssetChanges::Unpublished, vmUpload.CodeNotes().GetChanges());
+
+        vmUpload.QueueAsset(vmUpload.CodeNotes());
+        Assert::AreEqual({2U}, vmUpload.TaskCount());
+
+        int nApiCount = 0;
+        vmUpload.mockServer.HandleRequest<ra::api::UpdateCodeNote>(
+            [&nApiCount](const ra::api::UpdateCodeNote::Request& pRequest,
+                         ra::api::UpdateCodeNote::Response& pResponse) {
+                nApiCount++;
+                Assert::AreEqual(AssetUploadViewModelHarness::GameId, pRequest.GameId);
+                if (pRequest.Address == 0x1234U)
+                {
+                    Assert::AreEqual(0x1234U, pRequest.Address);
+                    Assert::AreEqual(std::wstring(L"This is a note."), pRequest.Note);
+                }
+                else
+                {
+                    Assert::AreEqual(0x1235U, pRequest.Address);
+                    Assert::AreEqual(std::wstring(L"This is another note."), pRequest.Note);
+                }
+
+                if (nApiCount % 2 == 0)
+                    pResponse.Result = ra::api::ApiResult::Incomplete;
+                else
+                    pResponse.Result = ra::api::ApiResult::Success;
+                return true;
+            });
+
+        vmUpload.DoUpload();
+
+        // 0 = 1234, success
+        // 1 = 1235, delayed
+        // 2 = 1235, success
+        Assert::AreEqual(3, nApiCount);
+        Assert::AreEqual(AssetChanges::None, vmUpload.CodeNotes().GetChanges());
+
+        vmUpload.AssertSuccess(2);
+    }
 };
 
 } // namespace tests