Use glaze for secrets and coincenter info

CMakeLists.txt

@@ -185,7 +185,7 @@ if(CCT_ENABLE_PROTO)
   else()
     # Check here for a new version: https://protobuf.dev/support/version-support/#cpp
     if (NOT PROTOBUF_VERSION)
-      set(PROTOBUF_VERSION v5.28.2)
+      set(PROTOBUF_VERSION v5.28.3)
     endif()
 
     message(STATUS "Configuring protobuf ${PROTOBUF_VERSION} from sources")

src/api-objects/src/apikeysprovider.cpp

@@ -8,13 +8,14 @@
 #include "accountowner.hpp"
 #include "apikey.hpp"
 #include "cct_exception.hpp"
-#include "cct_json-container.hpp"
 #include "cct_log.hpp"
 #include "cct_string.hpp"
 #include "exchangename.hpp"
 #include "exchangesecretsinfo.hpp"
 #include "file.hpp"
+#include "read-json.hpp"
 #include "runmodes.hpp"
+#include "secret-schema.hpp"
 
 namespace cct::api {
 namespace {
@@ -73,61 +74,48 @@ APIKeysProvider::APIKeysPerExchange APIKeysProvider::ParseAPIKeys(std::string_vi
                                                                   const ExchangeSecretsInfo& exchangeSecretsInfo,
                                                                   settings::RunMode runMode) {
   APIKeysProvider::APIKeysPerExchange apiKeysPerExchange;
+
   if (exchangeSecretsInfo.allExchangesWithoutSecrets()) {
     log::info("Not loading private keys, using only public exchanges");
-  } else {
-    std::string_view secretFileName = GetSecretFileName(runMode);
-    File secretsFile(dataDir, File::Type::kSecret, secretFileName,
-                     settings::AreTestKeysRequested(runMode) ? File::IfError::kThrow : File::IfError::kNoThrow);
-    json::container jsonData = secretsFile.readAllJson();
-    bool atLeastOneKeyFound = false;
-    for (auto& [publicExchangeName, keyObj] : jsonData.items()) {
-      const auto& exchangesWithoutSecrets = exchangeSecretsInfo.exchangesWithoutSecrets();
-      if (std::ranges::find(exchangesWithoutSecrets, ExchangeName(publicExchangeName)) !=
-          exchangesWithoutSecrets.end()) {
-        log::info("Not loading {} private keys as requested", publicExchangeName);
+    return apiKeysPerExchange;
+  }
+
+  std::string_view secretFileName = GetSecretFileName(runMode);
+  const auto throwOrNoThrow = settings::AreTestKeysRequested(runMode) ? File::IfError::kThrow : File::IfError::kNoThrow;
+  File secretsFile(dataDir, File::Type::kSecret, secretFileName, throwOrNoThrow);
+
+  schema::APIKeysPerExchangeMap apiKeysPerExchangeMap;
+
+  ReadJsonOrThrow(secretsFile.readAll(), apiKeysPerExchangeMap);
+
+  const auto& exchangesWithoutSecrets = exchangeSecretsInfo.exchangesWithoutSecrets();
+
+  bool atLeastOneKeyFound = false;
+  for (auto& [exchangeNameEnum, apiKeys] : apiKeysPerExchangeMap) {
+    auto publicExchangeName = kSupportedExchanges[static_cast<int>(exchangeNameEnum)];
+    if (std::ranges::any_of(exchangesWithoutSecrets, [exchangeNameEnum](const auto& exchangeName) {
+          return exchangeName.exchangeNameEnum() == exchangeNameEnum;
+        })) {
+      log::debug("Not loading {} private keys as requested", publicExchangeName);
+      continue;
+    }
+
+    for (auto& [keyName, apiKey] : apiKeys) {
+      if (apiKey.key.empty() || apiKey.priv.empty()) {
+        log::error("Wrong format for secret.json file. It should contain at least fields 'key' and 'private'");
         continue;
       }
-      ExchangeNameEnum exchangeNameEnum = static_cast<ExchangeNameEnum>(
-          std::find(std::begin(kSupportedExchanges), std::end(kSupportedExchanges), publicExchangeName) -
-          std::begin(kSupportedExchanges));
-
-      for (auto& [name, keySecretObj] : keyObj.items()) {
-        auto keyIt = keySecretObj.find("key");
-        auto privateIt = keySecretObj.find("private");
-        if (keyIt == keySecretObj.end() || privateIt == keySecretObj.end()) {
-          log::error("Wrong format for secret.json file. It should contain at least fields 'key' and 'private'");
-          continue;
-        }
-        string passphrase;
-        auto passphraseIt = keySecretObj.find("passphrase");
-        if (passphraseIt != keySecretObj.end()) {
-          passphrase = std::move(passphraseIt->get_ref<string&>());
-        }
-        std::string_view ownerEnName;
-        std::string_view ownerKoName;
-        auto accountOwnerPartIt = keySecretObj.find("accountOwner");
-        if (accountOwnerPartIt != keySecretObj.end()) {
-          auto ownerEnNameIt = accountOwnerPartIt->find("enName");
-          if (ownerEnNameIt != accountOwnerPartIt->end()) {
-            ownerEnName = ownerEnNameIt->get<std::string_view>();
-          }
-          auto ownerKoNameIt = accountOwnerPartIt->find("koName");
-          if (ownerKoNameIt != accountOwnerPartIt->end()) {
-            ownerKoName = ownerKoNameIt->get<std::string_view>();
-          }
-        }
-
-        apiKeysPerExchange[static_cast<int>(exchangeNameEnum)].emplace_back(
-            publicExchangeName, name, std::move(keyIt->get_ref<string&>()), std::move(privateIt->get_ref<string&>()),
-            std::move(passphrase), AccountOwner(ownerEnName, ownerKoName));
-        atLeastOneKeyFound = true;
-      }
-    }
-    if (!atLeastOneKeyFound) {
-      log::warn("No private api keys file '{}' found. Only public exchange queries will be supported", secretFileName);
+
+      apiKeysPerExchange[static_cast<int>(exchangeNameEnum)].emplace_back(
+          publicExchangeName, keyName, std::move(apiKey.key), std::move(apiKey.priv), std::move(apiKey.passphrase),
+          AccountOwner(apiKey.accountOwner.enName, apiKey.accountOwner.koName));
+
+      atLeastOneKeyFound = true;
     }
   }
+  if (!atLeastOneKeyFound) {
+    log::warn("No private api keys file '{}' found. Only public exchange queries will be supported", secretFileName);
+  }
 
   return apiKeysPerExchange;
 }

src/api/common/include/fiatconverter.hpp

@@ -4,9 +4,13 @@
 #include <optional>
 #include <string_view>
 #include <unordered_map>
+#include <utility>
 
 #include "cache-file-updator-interface.hpp"
+#include "cct_fixedcapacityvector.hpp"
+#include "cct_flatset.hpp"
 #include "cct_string.hpp"
+#include "cct_vector.hpp"
 #include "curlhandle.hpp"
 #include "currencycode.hpp"
 #include "file.hpp"
@@ -36,16 +40,16 @@ class CoincenterInfo;
 /// Conversion methods are thread safe.
 class FiatConverter : public CacheFileUpdatorInterface {
  public:
-  static File GetRatesCacheFile(std::string_view dataDir);
-
   /// Creates a FiatConverter able to perform live queries to free converter api.
   /// @param ratesUpdateFrequency the minimum time needed between two currency rates updates
   FiatConverter(const CoincenterInfo &coincenterInfo, Duration ratesUpdateFrequency);
 
   /// Creates a FiatConverter able to perform live queries to free converter api.
   /// @param ratesUpdateFrequency the minimum time needed between two currency rates updates
-  /// @param reader the reader from which to load the initial rates conversion cache
-  FiatConverter(const CoincenterInfo &coincenterInfo, Duration ratesUpdateFrequency, const Reader &reader);
+  /// @param fiatsRatesCacheReader the reader from which to load the initial rates conversion cache
+  /// @param thirdPartySecretReader the reader from which to load the third party secret
+  FiatConverter(const CoincenterInfo &coincenterInfo, Duration ratesUpdateFrequency,
+                const Reader &fiatsRatesCacheReader, const Reader &thirdPartySecretReader);
 
   std::optional<double> convert(double amount, CurrencyCode from, CurrencyCode to);
 
@@ -64,29 +68,58 @@ class FiatConverter : public CacheFileUpdatorInterface {
  private:
   struct PriceTimedValue {
     double rate;
-    TimePoint lastUpdatedTime;
+    int64_t timeepoch;
+
+    TimePoint lastUpdatedTime() const { return TimePoint(seconds(timeepoch)); }
+  };
+
+  struct ThirdPartySecret {
+    string freecurrencyconverter;
   };
 
+  static ThirdPartySecret LoadCurrencyConverterAPIKey(const Reader &thirdPartySecretReader);
+
   std::optional<double> queryCurrencyRate(Market market);
 
   std::optional<double> queryCurrencyRateSource1(Market market);
   std::optional<double> queryCurrencyRateSource2(Market market);
 
-  std::optional<double> retrieveRateFromCache(Market market) const;
+  enum class CacheReadMode : int8_t { kOnlyRecentRates, kUseAllRates };
+
+  std::optional<double> retrieveRateFromCache(Market market, CacheReadMode cacheReadMode);
 
   void store(Market market, double rate);
 
   void refreshLastUpdatedTime(Market market);
 
   using PricesMap = std::unordered_map<Market, PriceTimedValue>;
 
+  // For the algorithm computing rates
+  struct Node {
+    // hard limit to avoid unreasonable long paths and memory allocations
+    static constexpr std::size_t kMaxCurrencyPathSize = 6U;
+
+    using CurrencyPath = FixedCapacityVector<CurrencyCode, kMaxCurrencyPathSize>;
+
+    using trivially_relocatable = std::true_type;
+
+    CurrencyPath currencyPath;
+    double rate;
+    TimePoint oldestTs;
+  };
+
+  vector<Node> _nodes;
+  using VisitedCurrencyCodesSet = FlatSet<CurrencyCode>;
+
+  VisitedCurrencyCodesSet _visitedCurrencies;
+  vector<std::pair<Market, PriceTimedValue>> _tmpPriceRatesVector;
+
   CurlHandle _curlHandle1;
   CurlHandle _curlHandle2;
   PricesMap _pricesMap;
   Duration _ratesUpdateFrequency;
   std::mutex _pricesMutex;
-  string _apiKey;
+  ThirdPartySecret _thirdPartySecret;
   string _dataDir;
-  CurrencyCode _baseRateSource2;
 };
 }  // namespace cct

src/api/common/src/commonapi.cpp

@@ -1,25 +1,28 @@
 #include "commonapi.hpp"
 
 #include <cstdint>
+#include <glaze/glaze.hpp>  // IWYU pragma: export
 #include <mutex>
 #include <optional>
 #include <string_view>
 #include <utility>
 
 #include "cachedresult.hpp"
 #include "cct_const.hpp"
-#include "cct_json-container.hpp"
 #include "cct_log.hpp"
 #include "cct_string.hpp"
+#include "cct_vector.hpp"
 #include "coincenterinfo.hpp"
 #include "curloptions.hpp"
 #include "currencycode.hpp"
 #include "currencycodeset.hpp"
 #include "currencycodevector.hpp"
+#include "fiats-cache-schema.hpp"
 #include "file.hpp"
 #include "httprequesttype.hpp"
 #include "monetaryamountbycurrencyset.hpp"
 #include "permanentcurloptions.hpp"
+#include "read-json.hpp"
 #include "timedef.hpp"
 #include "withdrawalfees-crawler.hpp"
 
@@ -46,18 +49,15 @@ CommonAPI::CommonAPI(const CoincenterInfo& coincenterInfo, Duration fiatsUpdateF
                           coincenterInfo.getRunMode()),
       _withdrawalFeesCrawler(coincenterInfo, withdrawalFeesUpdateFrequency, _cachedResultVault) {
   if (atInit == AtInit::kLoadFromFileCache) {
-    json::container data = GetFiatCacheFile(_coincenterInfo.dataDir()).readAllJson();
-    if (!data.empty()) {
-      int64_t timeEpoch = data["timeepoch"].get<int64_t>();
-      auto& fiatsFile = data["fiats"];
-      CurrencyCodeSet fiats;
-      fiats.reserve(static_cast<CurrencyCodeSet::size_type>(fiatsFile.size()));
-      for (json::container& val : fiatsFile) {
-        log::trace("Reading fiat {} from cache file", val.get<std::string_view>());
-        fiats.emplace_hint(fiats.end(), std::move(val.get_ref<string&>()));
+    schema::FiatsCache fiatsCache;
+    auto dataStr = GetFiatCacheFile(_coincenterInfo.dataDir()).readAll();
+    if (!dataStr.empty()) {
+      ReadJsonOrThrow(dataStr, fiatsCache);
+      if (fiatsCache.timeepoch != 0) {
+        CurrencyCodeSet fiats(std::move(fiatsCache.fiats));
+        log::debug("Loaded {} fiats from cache file", fiats.size());
+        _fiatsCache.set(std::move(fiats), TimePoint(seconds(fiatsCache.timeepoch)));
       }
-      log::debug("Loaded {} fiats from cache file", fiats.size());
-      _fiatsCache.set(std::move(fiats), TimePoint(seconds(timeEpoch)));
     }
   }
 }
@@ -103,7 +103,7 @@ std::optional<MonetaryAmount> CommonAPI::tryQueryWithdrawalFee(std::string_view
 }
 
 namespace {
-constexpr std::string_view kFiatsUrlSource1 = "https://datahub.io/core/currency-codes/r/codes-all.json";
+constexpr std::string_view kFiatsUrlSource1 = "https://datahub.io/core/currency-codes/_r/-/data/codes-all.csv";
 constexpr std::string_view kFiatsUrlSource2 = "https://www.iban.com/currency-codes";
 }  // namespace
 
@@ -130,6 +130,14 @@ CurrencyCodeSet CommonAPI::FiatsFunc::operator()() {
   }
   return fiats;
 }
+struct CurrencyCSV {
+  vector<string> Entity;
+  vector<string> Currency;
+  vector<string> AlphabeticCode;
+  vector<string> NumericCode;
+  vector<string> MinorUnit;
+  vector<string> WithdrawalDate;
+};
 
 CurrencyCodeVector CommonAPI::FiatsFunc::retrieveFiatsSource1() {
   CurrencyCodeVector fiatsVec;
@@ -139,25 +147,27 @@ CurrencyCodeVector CommonAPI::FiatsFunc::retrieveFiatsSource1() {
     log::warn("Error parsing currency codes, no fiats found from first source");
     return fiatsVec;
   }
-  static constexpr bool kAllowExceptions = false;
-  json::container dataCSV = json::container::parse(data, nullptr, kAllowExceptions);
-  if (dataCSV.is_discarded()) {
-    log::warn("Error parsing json data of currency codes from source 1");
+
+  // data is UTF-8 encoded - but the relevant data that we will parse is ASCII normally
+
+  CurrencyCSV currencies;
+  auto ec = json::read<json::opts{.format = glz::CSV, .layout = glz::colwise}>(currencies, data);
+
+  if (ec || currencies.AlphabeticCode.size() != currencies.WithdrawalDate.size()) {
+    log::warn("Error parsing json data of currency codes from source 1: {}", glz::format_error(ec, data));
     return fiatsVec;
   }
-  for (const json::container& fiatData : dataCSV) {
-    static constexpr std::string_view kCodeKey = "AlphabeticCode";
-    static constexpr std::string_view kWithdrawalDateKey = "WithdrawalDate";
-
-    auto codeIt = fiatData.find(kCodeKey);
-    auto withdrawalDateIt = fiatData.find(kWithdrawalDateKey);
-    if (codeIt != fiatData.end() && !codeIt->is_null() && withdrawalDateIt != fiatData.end() &&
-        withdrawalDateIt->is_null()) {
-      fiatsVec.emplace_back(codeIt->get<std::string_view>());
-      log::debug("Stored {} fiat", codeIt->get<std::string_view>());
+
+  auto nbCurrencies = currencies.AlphabeticCode.size();
+  for (size_t currencyPos = 0; currencyPos < nbCurrencies; ++currencyPos) {
+    if (currencies.WithdrawalDate[currencyPos].empty()) {
+      fiatsVec.emplace_back(currencies.AlphabeticCode[currencyPos]);
+      log::debug("Stored {} fiat", fiatsVec.back());
     }
   }
 
+  log::info("Found {} fiats from first source", fiatsVec.size());
+
   return fiatsVec;
 }