Use glaze for withdrawal fees crawler

sjanel · Nov 24, 2024 · a1b60fb · a1b60fb
1 parent d1a1d3c
commit a1b60fb
Show file tree

Hide file tree

Showing 14 changed files with 211 additions and 125 deletions.
diff --git a/src/api-objects/src/apikeysprovider.cpp b/src/api-objects/src/apikeysprovider.cpp
@@ -86,7 +86,7 @@ APIKeysProvider::APIKeysPerExchange APIKeysProvider::ParseAPIKeys(std::string_vi
 
   schema::APIKeysPerExchangeMap apiKeysPerExchangeMap;
 
-  ReadJsonOrThrow(secretsFile.readAll(), apiKeysPerExchangeMap);
+  ReadExactJsonOrThrow(secretsFile.readAll(), apiKeysPerExchangeMap);
 
   const auto& exchangesWithoutSecrets = exchangeSecretsInfo.exchangesWithoutSecrets();
 

diff --git a/src/api/common/src/commonapi.cpp b/src/api/common/src/commonapi.cpp
@@ -52,7 +52,7 @@ CommonAPI::CommonAPI(const CoincenterInfo& coincenterInfo, Duration fiatsUpdateF
     schema::FiatsCache fiatsCache;
     auto dataStr = GetFiatCacheFile(_coincenterInfo.dataDir()).readAll();
     if (!dataStr.empty()) {
-      ReadJsonOrThrow(dataStr, fiatsCache);
+      ReadExactJsonOrThrow(dataStr, fiatsCache);
       if (fiatsCache.timeepoch != 0) {
         CurrencyCodeSet fiats(std::move(fiatsCache.fiats));
         log::debug("Loaded {} fiats from cache file", fiats.size());
@@ -213,22 +213,25 @@ CurrencyCodeVector CommonAPI::FiatsFunc::retrieveFiatsSource2() {
 
 void CommonAPI::updateCacheFile() const {
   const auto fiatsCacheFile = GetFiatCacheFile(_coincenterInfo.dataDir());
-  auto data = fiatsCacheFile.readAllJson();
+  auto fiatsData = fiatsCacheFile.readAllJson();
   const auto fiatsPtrLastUpdatedTimePair = _fiatsCache.retrieve();
-  const auto timeEpochIt = data.find("timeepoch");
-  if (timeEpochIt != data.end()) {
+  const auto timeEpochIt = fiatsData.find("timeepoch");
+  bool updateFiatsCache = true;
+  if (timeEpochIt != fiatsData.end()) {
     const int64_t lastTimeFileUpdated = timeEpochIt->get<int64_t>();
     if (TimePoint(seconds(lastTimeFileUpdated)) >= fiatsPtrLastUpdatedTimePair.second) {
-      return;  // No update
+      updateFiatsCache = false;
     }
   }
-  data.clear();
-  if (fiatsPtrLastUpdatedTimePair.first != nullptr) {
-    for (CurrencyCode fiatCode : *fiatsPtrLastUpdatedTimePair.first) {
-      data["fiats"].emplace_back(fiatCode.str());
+  if (updateFiatsCache) {
+    fiatsData.clear();
+    if (fiatsPtrLastUpdatedTimePair.first != nullptr) {
+      for (CurrencyCode fiatCode : *fiatsPtrLastUpdatedTimePair.first) {
+        fiatsData["fiats"].emplace_back(fiatCode.str());
+      }
+      fiatsData["timeepoch"] = TimestampToSecondsSinceEpoch(fiatsPtrLastUpdatedTimePair.second);
+      fiatsCacheFile.writeJson(fiatsData);
     }
-    data["timeepoch"] = TimestampToSecondsSinceEpoch(fiatsPtrLastUpdatedTimePair.second);
-    fiatsCacheFile.writeJson(data);
   }
 
   _withdrawalFeesCrawler.updateCacheFile();

diff --git a/src/api/common/src/fiatconverter.cpp b/src/api/common/src/fiatconverter.cpp
@@ -57,13 +57,13 @@ FiatConverter::FiatConverter(const CoincenterInfo& coincenterInfo, Duration rate
       _dataDir(coincenterInfo.dataDir()) {
   const auto data = fiatsRatesCacheReader.readAll();
 
-  ReadJsonOrThrow(data, _pricesMap);
+  ReadExactJsonOrThrow(data, _pricesMap);
 
   log::debug("Loaded {} fiat currency rates from {}", _pricesMap.size(), kRatesCacheFile);
 }
 
 void FiatConverter::updateCacheFile() const {
-  auto dataStr = WriteJsonOrThrow(_pricesMap);
+  auto dataStr = WriteMiniJsonOrThrow(_pricesMap);
   GetRatesCacheFile(_dataDir).write(dataStr);
 }
 
@@ -89,13 +89,9 @@ std::optional<double> FiatConverter::queryCurrencyRateSource1(Market market) {
   schema::FreeCurrencyConverterResponse response;
 
   //{"query":{"count":1},"results":{"EUR_KRW":{"id":"EUR_KRW","val":1329.475323,"to":"KRW","fr":"EUR"}}}
-  auto ec = json::read<json::opts{.error_on_unknown_keys = false, .raw_string = true}>(response, dataStr);
+  auto ec = ReadPartialJson(dataStr, "fiat currency converter service's first source", response);
 
   if (ec) {
-    std::string_view prefixJsonContent = dataStr.substr(0, std::min<int>(dataStr.size(), 20));
-    log::error("Error while reading json content from fiat currency converter service's first source '{}{}': {}",
-               prefixJsonContent, prefixJsonContent.size() < dataStr.size() ? "..." : "",
-               json::format_error(ec, dataStr));
     return {};
   }
 
@@ -115,13 +111,9 @@ std::optional<double> FiatConverter::queryCurrencyRateSource2(Market market) {
 
   schema::FiatRatesSource2Response response;
 
-  auto ec = json::read<json::opts{.error_on_unknown_keys = false, .raw_string = true}>(response, dataStr);
+  auto ec = ReadPartialJson(dataStr, "fiat currency converter service's second source", response);
 
   if (ec) {
-    std::string_view prefixJsonContent = dataStr.substr(0, std::min<int>(dataStr.size(), 20));
-    log::error("Error while reading json content from fiat currency converter service's second source '{}{}': {}",
-               prefixJsonContent, prefixJsonContent.size() < dataStr.size() ? "..." : "",
-               json::format_error(ec, dataStr));
     return {};
   }
 
@@ -276,12 +268,9 @@ FiatConverter::ThirdPartySecret FiatConverter::LoadCurrencyConverterAPIKey(const
     return thirdPartySecret;
   }
 
-  auto ec = json::read<json::opts{.error_on_unknown_keys = false, .raw_string = true}>(thirdPartySecret, dataStr);
+  auto ec = ReadPartialJson(dataStr, "third party's secrets", thirdPartySecret);
 
   if (ec) {
-    std::string_view prefixJsonContent = dataStr.substr(0, std::min<int>(dataStr.size(), 20));
-    log::error("Error while reading json content from third party's secrets '{}{}': {}", prefixJsonContent,
-               prefixJsonContent.size() < dataStr.size() ? "..." : "", json::format_error(ec, dataStr));
     return thirdPartySecret;
   }
 

diff --git a/src/api/common/src/withdrawal-fees-schema.hpp b/src/api/common/src/withdrawal-fees-schema.hpp
@@ -0,0 +1,48 @@
+#pragma once
+
+#include <cstdint>
+#include <unordered_map>
+
+#include "cct_string.hpp"
+#include "cct_vector.hpp"
+#include "currencycode.hpp"
+#include "monetaryamount.hpp"
+
+namespace cct::schema {
+
+struct WithdrawInfoFileItemAsset {
+  MonetaryAmount min;  // only amount
+  MonetaryAmount fee;  // only amount
+};
+
+struct WithdrawInfoFileItem {
+  int64_t timeepoch;
+  std::unordered_map<CurrencyCode, WithdrawInfoFileItemAsset> assets;
+};
+
+using WithdrawInfoFile = std::unordered_map<string, WithdrawInfoFileItem>;
+
+struct WithdrawFeesCrawlerExchangeFeesCoinSource1 {
+  string symbol;
+
+  auto operator<=>(const WithdrawFeesCrawlerExchangeFeesCoinSource1&) const = default;
+};
+
+struct WithdrawFeesCrawlerExchangeFeesSource1 {
+  double amount;
+  double min;
+  WithdrawFeesCrawlerExchangeFeesCoinSource1 coin;
+
+  auto operator<=>(const WithdrawFeesCrawlerExchangeFeesSource1&) const = default;
+};
+
+struct WithdrawFeesCrawlerExchangeSource1 {
+  string name;
+  vector<WithdrawFeesCrawlerExchangeFeesSource1> fees;
+};
+
+struct WithdrawFeesCrawlerSource1 {
+  WithdrawFeesCrawlerExchangeSource1 exchange;
+};
+
+}  // namespace cct::schema
diff --git a/src/api/common/src/withdrawalfees-crawler.cpp b/src/api/common/src/withdrawalfees-crawler.cpp
@@ -12,7 +12,6 @@
 #include "cct_cctype.hpp"
 #include "cct_const.hpp"
 #include "cct_exception.hpp"
-#include "cct_json-container.hpp"
 #include "cct_log.hpp"
 #include "cct_string.hpp"
 #include "coincenterinfo.hpp"
@@ -22,8 +21,11 @@
 #include "httprequesttype.hpp"
 #include "monetaryamount.hpp"
 #include "permanentcurloptions.hpp"
+#include "read-json.hpp"
 #include "threadpool.hpp"
 #include "timedef.hpp"
+#include "withdrawal-fees-schema.hpp"
+#include "write-json.hpp"
 
 namespace cct {
 
@@ -40,37 +42,39 @@ WithdrawalFeesCrawler::WithdrawalFeesCrawler(const CoincenterInfo& coincenterInf
                                              CachedResultVault& cachedResultVault)
     : _coincenterInfo(coincenterInfo),
       _withdrawalFeesCache(CachedResultOptions(minDurationBetweenQueries, cachedResultVault), coincenterInfo) {
-  json::container data = GetWithdrawInfoFile(_coincenterInfo.dataDir()).readAllJson();
-  if (!data.empty()) {
-    const auto nowTime = Clock::now();
-    for (const auto& [exchangeName, exchangeData] : data.items()) {
-      TimePoint lastUpdatedTime(seconds(exchangeData["timeepoch"].get<int64_t>()));
-      if (nowTime - lastUpdatedTime < minDurationBetweenQueries) {
-        // we can reuse file data
-        WithdrawalInfoMaps withdrawalInfoMaps;
-
-        for (const auto& [curCodeStr, val] : exchangeData["assets"].items()) {
-          CurrencyCode cur(curCodeStr);
-          MonetaryAmount withdrawMin(val["min"].get<std::string_view>(), cur);
-          MonetaryAmount withdrawFee(val["fee"].get<std::string_view>(), cur);
-
-          log::trace("Updated {} withdrawal fee {} from cache", exchangeName, withdrawFee);
-          log::trace("Updated {} min withdraw {} from cache", exchangeName, withdrawMin);
-
-          withdrawalInfoMaps.first.insert(withdrawFee);
-          withdrawalInfoMaps.second.insert_or_assign(cur, withdrawMin);
-        }
+  auto data = GetWithdrawInfoFile(_coincenterInfo.dataDir()).readAll();
 
-        // Warning: we store a std::string_view in the cache, and 'exchangeName' will be destroyed at the end
-        // of this function. So we need to retrieve the 'constant' std::string_view of this exchange (in static memory)
-        // to store in the cache.
-        auto constantExchangeNameSVIt = std::ranges::find(kSupportedExchanges, exchangeName);
-        if (constantExchangeNameSVIt == std::end(kSupportedExchanges)) {
-          throw exception("unknown exchange name {}", exchangeName);
-        }
+  schema::WithdrawInfoFile withdrawInfoFileContent;
+
+  ReadExactJsonOrThrow(data, withdrawInfoFileContent);
+
+  const auto nowTime = Clock::now();
+  for (const auto& [exchangeName, exchangeData] : withdrawInfoFileContent) {
+    TimePoint lastUpdatedTime(seconds(exchangeData.timeepoch));
+    if (nowTime - lastUpdatedTime < minDurationBetweenQueries) {
+      // we can reuse file data
+      WithdrawalInfoMaps withdrawalInfoMaps;
+
+      for (const auto& [cur, val] : exchangeData.assets) {
+        MonetaryAmount withdrawMin(val.min, cur);
+        MonetaryAmount withdrawFee(val.fee, cur);
+
+        log::trace("Updated {} withdrawal fee {} from cache", exchangeName, withdrawFee);
+        log::trace("Updated {} min withdraw {} from cache", exchangeName, withdrawMin);
 
-        _withdrawalFeesCache.set(std::move(withdrawalInfoMaps), lastUpdatedTime, *constantExchangeNameSVIt);
+        withdrawalInfoMaps.first.insert(withdrawFee);
+        withdrawalInfoMaps.second.insert_or_assign(cur, withdrawMin);
       }
+
+      // Warning: we store a std::string_view in the cache, and 'exchangeName' will be destroyed at the end
+      // of this function. So we need to retrieve the 'constant' std::string_view of this exchange (in static memory)
+      // to store in the cache.
+      auto constantExchangeNameSVIt = std::ranges::find(kSupportedExchanges, exchangeName);
+      if (constantExchangeNameSVIt == std::end(kSupportedExchanges)) {
+        throw exception("unknown exchange name {}", exchangeName);
+      }
+
+      _withdrawalFeesCache.set(std::move(withdrawalInfoMaps), lastUpdatedTime, *constantExchangeNameSVIt);
     }
   }
 }
@@ -114,78 +118,65 @@ WithdrawalFeesCrawler::WithdrawalInfoMaps WithdrawalFeesCrawler::WithdrawalFeesF
 }
 
 void WithdrawalFeesCrawler::updateCacheFile() const {
-  json::container data;
+  schema::WithdrawInfoFile withdrawInfoFile;
   for (const std::string_view exchangeName : kSupportedExchanges) {
     const auto [withdrawalInfoMapsPtr, latestUpdate] = _withdrawalFeesCache.retrieve(exchangeName);
     if (withdrawalInfoMapsPtr != nullptr) {
       const WithdrawalInfoMaps& withdrawalInfoMaps = *withdrawalInfoMapsPtr;
 
-      json::container exchangeData;
-      exchangeData["timeepoch"] = TimestampToSecondsSinceEpoch(latestUpdate);
+      schema::WithdrawInfoFileItem& withdrawInfoFileItem =
+          withdrawInfoFile.emplace(std::make_pair(exchangeName, schema::WithdrawInfoFileItem{})).first->second;
+      withdrawInfoFileItem.timeepoch = TimestampToSecondsSinceEpoch(latestUpdate);
       for (const auto withdrawFee : withdrawalInfoMaps.first) {
-        string curCodeStr = withdrawFee.currencyCode().str();
-        exchangeData["assets"][curCodeStr]["min"] =
-            withdrawalInfoMaps.second.find(withdrawFee.currencyCode())->second.amountStr();
-        exchangeData["assets"][curCodeStr]["fee"] = withdrawFee.amountStr();
-      }
+        CurrencyCode cur = withdrawFee.currencyCode();
+
+        schema::WithdrawInfoFileItemAsset& asset = withdrawInfoFileItem.assets[cur];
 
-      data.emplace(exchangeName, std::move(exchangeData));
+        auto minIt = withdrawalInfoMaps.second.find(cur);
+        if (minIt != withdrawalInfoMaps.second.end()) {
+          asset.min = MonetaryAmount(minIt->second, CurrencyCode{});
+        }
+        asset.fee = MonetaryAmount(withdrawFee, CurrencyCode{});
+      }
     }
   }
-  GetWithdrawInfoFile(_coincenterInfo.dataDir()).writeJson(data);
+  auto dataStr = WriteMiniJsonOrThrow(withdrawInfoFile);
+
+  GetWithdrawInfoFile(_coincenterInfo.dataDir()).write(dataStr);
 }
 
 WithdrawalFeesCrawler::WithdrawalInfoMaps WithdrawalFeesCrawler::WithdrawalFeesFunc::get1(
     std::string_view exchangeName) {
   string path(exchangeName);
   path.append(".json");
-  std::string_view withdrawalFeesCsv = _curlHandle1.query(path, CurlOptions(HttpRequestType::kGet));
+  std::string_view dataStr = _curlHandle1.query(path, CurlOptions(HttpRequestType::kGet));
 
   WithdrawalInfoMaps ret;
 
-  if (!withdrawalFeesCsv.empty()) {
-    static constexpr bool kAllowExceptions = false;
-    const json::container jsonData = json::container::parse(withdrawalFeesCsv, nullptr, kAllowExceptions);
-    const auto exchangesIt = jsonData.find("exchange");
-    if (jsonData.is_discarded() || exchangesIt == jsonData.end()) {
-      log::error("no exchange data found in source 1 - either site information unavailable or code to be updated");
-      return ret;
-    }
-    const auto feesIt = exchangesIt->find("fees");
-    if (feesIt == exchangesIt->end() || !feesIt->is_array()) {
-      log::error("no fees data found in source 1 - either site information unavailable or code to be updated");
-      return ret;
-    }
+  schema::WithdrawFeesCrawlerSource1 withdrawalFeesCrawlerSource1;
+  ReadPartialJson(dataStr, "withdraw fees crawler service's first source", withdrawalFeesCrawlerSource1);
 
-    for (const json::container& feeJson : *feesIt) {
-      const auto amountIt = feeJson.find("amount");
-      if (amountIt == feeJson.end() || !amountIt->is_number_float()) {
-        continue;
-      }
+  if (withdrawalFeesCrawlerSource1.exchange.fees.empty()) {
+    log::error("no fees data found in source 1 - either site information unavailable or code to be updated");
+    return ret;
+  }
 
-      const auto coinIt = feeJson.find("coin");
-      if (coinIt == feeJson.end()) {
-        continue;
-      }
-      const auto symbolIt = coinIt->find("symbol");
-      if (symbolIt == coinIt->end() || !symbolIt->is_string()) {
-        continue;
-      }
+  for (const schema::WithdrawFeesCrawlerExchangeFeesSource1& fee : withdrawalFeesCrawlerSource1.exchange.fees) {
+    if (fee.coin.symbol.size() > CurrencyCode::kMaxLen) {
+      log::warn("Skipping {} withdrawal fees parsing from first source: symbol too long", fee.coin.symbol);
+      continue;
+    }
 
-      MonetaryAmount withdrawalFee(amountIt->get<double>(), symbolIt->get<std::string_view>());
-      log::trace("Updated {} withdrawal fee {} from first source", exchangeName, withdrawalFee);
-      ret.first.insert(withdrawalFee);
+    CurrencyCode cur{fee.coin.symbol};
 
-      const auto minWithdrawalIt = feeJson.find("min");
-      if (minWithdrawalIt == feeJson.end() || !minWithdrawalIt->is_number_float()) {
-        continue;
-      }
+    MonetaryAmount withdrawalFee(fee.amount, cur);
+    log::trace("Updated {} withdrawal fee {} from first source", exchangeName, withdrawalFee);
+    ret.first.insert(withdrawalFee);
 
-      MonetaryAmount minWithdrawal(minWithdrawalIt->get<double>(), symbolIt->get<std::string_view>());
+    MonetaryAmount minWithdrawal(fee.min, cur);
 
-      log::trace("Updated {} min withdrawal {} from first source", exchangeName, minWithdrawal);
-      ret.second.insert_or_assign(minWithdrawal.currencyCode(), minWithdrawal);
-    }
+    log::trace("Updated {} min withdrawal {} from first source", exchangeName, minWithdrawal);
+    ret.second.insert_or_assign(minWithdrawal.currencyCode(), minWithdrawal);
   }
 
   if (ret.first.empty() || ret.second.empty()) {

diff --git a/src/api/common/test/fiatconverter_test.cpp b/src/api/common/test/fiatconverter_test.cpp
@@ -86,7 +86,7 @@ std::string_view CurlHandle::query([[maybe_unused]] std::string_view endpoint, c
       }
     }
     if (res.val != 0) {
-      _queryData = WriteJsonOrThrow(response);
+      _queryData = WriteMiniJsonOrThrow(response);
     }
 
   } else {
@@ -96,7 +96,7 @@ std::string_view CurlHandle::query([[maybe_unused]] std::string_view endpoint, c
     response.rates["SUSHI"] = 36.78;
     response.rates["KRW"] = 1341.88;
     response.rates["NOK"] = 11.3375;
-    _queryData = WriteJsonOrThrow(response);
+    _queryData = WriteMiniJsonOrThrow(response);
   }
 
   return _queryData;

diff --git a/src/api/exchanges/src/binanceprivateapi.cpp b/src/api/exchanges/src/binanceprivateapi.cpp
@@ -238,7 +238,7 @@ T PrivateQuery(CurlHandle& curlHandle, const APIKey& apiKey, HttpRequestType req
   }
   if (throwIfError) {
     std::string_view errorMsg;
-    string jsonStr = WriteJsonOrThrow(ret);
+    string jsonStr = WriteMiniJsonOrThrow(ret);
     if constexpr (amc::is_detected<has_msg_t, T>::value) {
       if (ret.msg) {
         errorMsg = *ret.msg;