From b2d4300d64b25e10aee6efa9acf19cf37f8e194d Mon Sep 17 00:00:00 2001 From: Stephane Janel Date: Tue, 19 Nov 2024 22:59:13 +0100 Subject: [PATCH] Use glaze for secrets and coincenter info --- CMakeLists.txt | 2 +- src/api-objects/src/apikeysprovider.cpp | 88 +++--- src/api/common/include/fiatconverter.hpp | 49 ++- src/api/common/src/commonapi.cpp | 64 ++-- src/api/common/src/fiatconverter.cpp | 294 +++++++++++------- .../src/fiats-converter-responses-schema.hpp | 27 ++ .../common/test/exchangeprivateapi_test.cpp | 4 +- .../common/test/exchangepublicapi_test.cpp | 2 +- src/api/common/test/fiatconverter_test.cpp | 66 ++-- .../test/bithumb_place_order_test.cpp | 3 +- .../exchanges/test/exchangecommonapi_test.hpp | 3 +- .../interface/test/exchangeretriever_test.cpp | 3 +- src/basic-objects/include/market.hpp | 2 +- src/basic-objects/test/market_test.cpp | 43 +++ src/engine/test/exchangedata_test.hpp | 3 +- src/objects/src/coincenterinfo.cpp | 32 +- src/schema/include/fiats-cache-schema.hpp | 17 + src/schema/include/read-json.hpp | 4 + src/schema/include/secret-schema.hpp | 38 +++ 19 files changed, 489 insertions(+), 255 deletions(-) create mode 100644 src/api/common/src/fiats-converter-responses-schema.hpp create mode 100644 src/schema/include/fiats-cache-schema.hpp create mode 100644 src/schema/include/secret-schema.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 31df70e8..105d2178 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -185,7 +185,7 @@ if(CCT_ENABLE_PROTO) else() # Check here for a new version: https://protobuf.dev/support/version-support/#cpp if (NOT PROTOBUF_VERSION) - set(PROTOBUF_VERSION v5.28.2) + set(PROTOBUF_VERSION v5.28.3) endif() message(STATUS "Configuring protobuf ${PROTOBUF_VERSION} from sources") diff --git a/src/api-objects/src/apikeysprovider.cpp b/src/api-objects/src/apikeysprovider.cpp index 5019b88e..520891e2 100644 --- a/src/api-objects/src/apikeysprovider.cpp +++ b/src/api-objects/src/apikeysprovider.cpp @@ -8,13 +8,14 @@ #include "accountowner.hpp" #include "apikey.hpp" #include "cct_exception.hpp" -#include "cct_json-container.hpp" #include "cct_log.hpp" #include "cct_string.hpp" #include "exchangename.hpp" #include "exchangesecretsinfo.hpp" #include "file.hpp" +#include "read-json.hpp" #include "runmodes.hpp" +#include "secret-schema.hpp" namespace cct::api { namespace { @@ -73,61 +74,48 @@ APIKeysProvider::APIKeysPerExchange APIKeysProvider::ParseAPIKeys(std::string_vi const ExchangeSecretsInfo& exchangeSecretsInfo, settings::RunMode runMode) { APIKeysProvider::APIKeysPerExchange apiKeysPerExchange; + if (exchangeSecretsInfo.allExchangesWithoutSecrets()) { log::info("Not loading private keys, using only public exchanges"); - } else { - std::string_view secretFileName = GetSecretFileName(runMode); - File secretsFile(dataDir, File::Type::kSecret, secretFileName, - settings::AreTestKeysRequested(runMode) ? File::IfError::kThrow : File::IfError::kNoThrow); - json::container jsonData = secretsFile.readAllJson(); - bool atLeastOneKeyFound = false; - for (auto& [publicExchangeName, keyObj] : jsonData.items()) { - const auto& exchangesWithoutSecrets = exchangeSecretsInfo.exchangesWithoutSecrets(); - if (std::ranges::find(exchangesWithoutSecrets, ExchangeName(publicExchangeName)) != - exchangesWithoutSecrets.end()) { - log::info("Not loading {} private keys as requested", publicExchangeName); + return apiKeysPerExchange; + } + + std::string_view secretFileName = GetSecretFileName(runMode); + const auto throwOrNoThrow = settings::AreTestKeysRequested(runMode) ? File::IfError::kThrow : File::IfError::kNoThrow; + File secretsFile(dataDir, File::Type::kSecret, secretFileName, throwOrNoThrow); + + schema::APIKeysPerExchangeMap apiKeysPerExchangeMap; + + ReadJsonOrThrow(secretsFile.readAll(), apiKeysPerExchangeMap); + + const auto& exchangesWithoutSecrets = exchangeSecretsInfo.exchangesWithoutSecrets(); + + bool atLeastOneKeyFound = false; + for (auto& [exchangeNameEnum, apiKeys] : apiKeysPerExchangeMap) { + auto publicExchangeName = kSupportedExchanges[static_cast(exchangeNameEnum)]; + if (std::ranges::any_of(exchangesWithoutSecrets, [exchangeNameEnum](const auto& exchangeName) { + return exchangeName.exchangeNameEnum() == exchangeNameEnum; + })) { + log::debug("Not loading {} private keys as requested", publicExchangeName); + continue; + } + + for (auto& [keyName, apiKey] : apiKeys) { + if (apiKey.key.empty() || apiKey.priv.empty()) { + log::error("Wrong format for secret.json file. It should contain at least fields 'key' and 'private'"); continue; } - ExchangeNameEnum exchangeNameEnum = static_cast( - std::find(std::begin(kSupportedExchanges), std::end(kSupportedExchanges), publicExchangeName) - - std::begin(kSupportedExchanges)); - - for (auto& [name, keySecretObj] : keyObj.items()) { - auto keyIt = keySecretObj.find("key"); - auto privateIt = keySecretObj.find("private"); - if (keyIt == keySecretObj.end() || privateIt == keySecretObj.end()) { - log::error("Wrong format for secret.json file. It should contain at least fields 'key' and 'private'"); - continue; - } - string passphrase; - auto passphraseIt = keySecretObj.find("passphrase"); - if (passphraseIt != keySecretObj.end()) { - passphrase = std::move(passphraseIt->get_ref()); - } - std::string_view ownerEnName; - std::string_view ownerKoName; - auto accountOwnerPartIt = keySecretObj.find("accountOwner"); - if (accountOwnerPartIt != keySecretObj.end()) { - auto ownerEnNameIt = accountOwnerPartIt->find("enName"); - if (ownerEnNameIt != accountOwnerPartIt->end()) { - ownerEnName = ownerEnNameIt->get(); - } - auto ownerKoNameIt = accountOwnerPartIt->find("koName"); - if (ownerKoNameIt != accountOwnerPartIt->end()) { - ownerKoName = ownerKoNameIt->get(); - } - } - - apiKeysPerExchange[static_cast(exchangeNameEnum)].emplace_back( - publicExchangeName, name, std::move(keyIt->get_ref()), std::move(privateIt->get_ref()), - std::move(passphrase), AccountOwner(ownerEnName, ownerKoName)); - atLeastOneKeyFound = true; - } - } - if (!atLeastOneKeyFound) { - log::warn("No private api keys file '{}' found. Only public exchange queries will be supported", secretFileName); + + apiKeysPerExchange[static_cast(exchangeNameEnum)].emplace_back( + publicExchangeName, keyName, std::move(apiKey.key), std::move(apiKey.priv), std::move(apiKey.passphrase), + AccountOwner(apiKey.accountOwner.enName, apiKey.accountOwner.koName)); + + atLeastOneKeyFound = true; } } + if (!atLeastOneKeyFound) { + log::warn("No private api keys file '{}' found. Only public exchange queries will be supported", secretFileName); + } return apiKeysPerExchange; } diff --git a/src/api/common/include/fiatconverter.hpp b/src/api/common/include/fiatconverter.hpp index 7b16d8c3..f82a31aa 100644 --- a/src/api/common/include/fiatconverter.hpp +++ b/src/api/common/include/fiatconverter.hpp @@ -4,9 +4,13 @@ #include #include #include +#include #include "cache-file-updator-interface.hpp" +#include "cct_fixedcapacityvector.hpp" +#include "cct_flatset.hpp" #include "cct_string.hpp" +#include "cct_vector.hpp" #include "curlhandle.hpp" #include "currencycode.hpp" #include "file.hpp" @@ -36,16 +40,16 @@ class CoincenterInfo; /// Conversion methods are thread safe. class FiatConverter : public CacheFileUpdatorInterface { public: - static File GetRatesCacheFile(std::string_view dataDir); - /// Creates a FiatConverter able to perform live queries to free converter api. /// @param ratesUpdateFrequency the minimum time needed between two currency rates updates FiatConverter(const CoincenterInfo &coincenterInfo, Duration ratesUpdateFrequency); /// Creates a FiatConverter able to perform live queries to free converter api. /// @param ratesUpdateFrequency the minimum time needed between two currency rates updates - /// @param reader the reader from which to load the initial rates conversion cache - FiatConverter(const CoincenterInfo &coincenterInfo, Duration ratesUpdateFrequency, const Reader &reader); + /// @param fiatsRatesCacheReader the reader from which to load the initial rates conversion cache + /// @param thirdPartySecretReader the reader from which to load the third party secret + FiatConverter(const CoincenterInfo &coincenterInfo, Duration ratesUpdateFrequency, + const Reader &fiatsRatesCacheReader, const Reader &thirdPartySecretReader); std::optional convert(double amount, CurrencyCode from, CurrencyCode to); @@ -64,15 +68,25 @@ class FiatConverter : public CacheFileUpdatorInterface { private: struct PriceTimedValue { double rate; - TimePoint lastUpdatedTime; + int64_t timeepoch; + + TimePoint lastUpdatedTime() const { return TimePoint(seconds(timeepoch)); } + }; + + struct ThirdPartySecret { + string freecurrencyconverter; }; + static ThirdPartySecret LoadCurrencyConverterAPIKey(const Reader &thirdPartySecretReader); + std::optional queryCurrencyRate(Market market); std::optional queryCurrencyRateSource1(Market market); std::optional queryCurrencyRateSource2(Market market); - std::optional retrieveRateFromCache(Market market) const; + enum class CacheReadMode : int8_t { kOnlyRecentRates, kUseAllRates }; + + std::optional retrieveRateFromCache(Market market, CacheReadMode cacheReadMode); void store(Market market, double rate); @@ -80,13 +94,32 @@ class FiatConverter : public CacheFileUpdatorInterface { using PricesMap = std::unordered_map; + // For the algorithm computing rates + struct Node { + // hard limit to avoid unreasonable long paths and memory allocations + static constexpr std::size_t kMaxCurrencyPathSize = 6U; + + using CurrencyPath = FixedCapacityVector; + + using trivially_relocatable = std::true_type; + + CurrencyPath currencyPath; + double rate; + TimePoint oldestTs; + }; + + vector _nodes; + using VisitedCurrencyCodesSet = FlatSet; + + VisitedCurrencyCodesSet _visitedCurrencies; + vector> _tmpPriceRatesVector; + CurlHandle _curlHandle1; CurlHandle _curlHandle2; PricesMap _pricesMap; Duration _ratesUpdateFrequency; std::mutex _pricesMutex; - string _apiKey; + ThirdPartySecret _thirdPartySecret; string _dataDir; - CurrencyCode _baseRateSource2; }; } // namespace cct diff --git a/src/api/common/src/commonapi.cpp b/src/api/common/src/commonapi.cpp index 9ffe5f67..adcdbc51 100644 --- a/src/api/common/src/commonapi.cpp +++ b/src/api/common/src/commonapi.cpp @@ -1,6 +1,7 @@ #include "commonapi.hpp" #include +#include // IWYU pragma: export #include #include #include @@ -8,18 +9,20 @@ #include "cachedresult.hpp" #include "cct_const.hpp" -#include "cct_json-container.hpp" #include "cct_log.hpp" #include "cct_string.hpp" +#include "cct_vector.hpp" #include "coincenterinfo.hpp" #include "curloptions.hpp" #include "currencycode.hpp" #include "currencycodeset.hpp" #include "currencycodevector.hpp" +#include "fiats-cache-schema.hpp" #include "file.hpp" #include "httprequesttype.hpp" #include "monetaryamountbycurrencyset.hpp" #include "permanentcurloptions.hpp" +#include "read-json.hpp" #include "timedef.hpp" #include "withdrawalfees-crawler.hpp" @@ -46,18 +49,15 @@ CommonAPI::CommonAPI(const CoincenterInfo& coincenterInfo, Duration fiatsUpdateF coincenterInfo.getRunMode()), _withdrawalFeesCrawler(coincenterInfo, withdrawalFeesUpdateFrequency, _cachedResultVault) { if (atInit == AtInit::kLoadFromFileCache) { - json::container data = GetFiatCacheFile(_coincenterInfo.dataDir()).readAllJson(); - if (!data.empty()) { - int64_t timeEpoch = data["timeepoch"].get(); - auto& fiatsFile = data["fiats"]; - CurrencyCodeSet fiats; - fiats.reserve(static_cast(fiatsFile.size())); - for (json::container& val : fiatsFile) { - log::trace("Reading fiat {} from cache file", val.get()); - fiats.emplace_hint(fiats.end(), std::move(val.get_ref())); + schema::FiatsCache fiatsCache; + auto dataStr = GetFiatCacheFile(_coincenterInfo.dataDir()).readAll(); + if (!dataStr.empty()) { + ReadJsonOrThrow(dataStr, fiatsCache); + if (fiatsCache.timeepoch != 0) { + CurrencyCodeSet fiats(std::move(fiatsCache.fiats)); + log::debug("Loaded {} fiats from cache file", fiats.size()); + _fiatsCache.set(std::move(fiats), TimePoint(seconds(fiatsCache.timeepoch))); } - log::debug("Loaded {} fiats from cache file", fiats.size()); - _fiatsCache.set(std::move(fiats), TimePoint(seconds(timeEpoch))); } } } @@ -103,7 +103,7 @@ std::optional CommonAPI::tryQueryWithdrawalFee(std::string_view } namespace { -constexpr std::string_view kFiatsUrlSource1 = "https://datahub.io/core/currency-codes/r/codes-all.json"; +constexpr std::string_view kFiatsUrlSource1 = "https://datahub.io/core/currency-codes/_r/-/data/codes-all.csv"; constexpr std::string_view kFiatsUrlSource2 = "https://www.iban.com/currency-codes"; } // namespace @@ -130,6 +130,14 @@ CurrencyCodeSet CommonAPI::FiatsFunc::operator()() { } return fiats; } +struct CurrencyCSV { + vector Entity; + vector Currency; + vector AlphabeticCode; + vector NumericCode; + vector MinorUnit; + vector WithdrawalDate; +}; CurrencyCodeVector CommonAPI::FiatsFunc::retrieveFiatsSource1() { CurrencyCodeVector fiatsVec; @@ -139,25 +147,27 @@ CurrencyCodeVector CommonAPI::FiatsFunc::retrieveFiatsSource1() { log::warn("Error parsing currency codes, no fiats found from first source"); return fiatsVec; } - static constexpr bool kAllowExceptions = false; - json::container dataCSV = json::container::parse(data, nullptr, kAllowExceptions); - if (dataCSV.is_discarded()) { - log::warn("Error parsing json data of currency codes from source 1"); + + // data is UTF-8 encoded - but the relevant data that we will parse is ASCII normally + + CurrencyCSV currencies; + auto ec = json::read(currencies, data); + + if (ec || currencies.AlphabeticCode.size() != currencies.WithdrawalDate.size()) { + log::warn("Error parsing json data of currency codes from source 1: {}", glz::format_error(ec, data)); return fiatsVec; } - for (const json::container& fiatData : dataCSV) { - static constexpr std::string_view kCodeKey = "AlphabeticCode"; - static constexpr std::string_view kWithdrawalDateKey = "WithdrawalDate"; - - auto codeIt = fiatData.find(kCodeKey); - auto withdrawalDateIt = fiatData.find(kWithdrawalDateKey); - if (codeIt != fiatData.end() && !codeIt->is_null() && withdrawalDateIt != fiatData.end() && - withdrawalDateIt->is_null()) { - fiatsVec.emplace_back(codeIt->get()); - log::debug("Stored {} fiat", codeIt->get()); + + auto nbCurrencies = currencies.AlphabeticCode.size(); + for (size_t currencyPos = 0; currencyPos < nbCurrencies; ++currencyPos) { + if (currencies.WithdrawalDate[currencyPos].empty()) { + fiatsVec.emplace_back(currencies.AlphabeticCode[currencyPos]); + log::debug("Stored {} fiat", fiatsVec.back()); } } + log::info("Found {} fiats from first source", fiatsVec.size()); + return fiatsVec; } diff --git a/src/api/common/src/fiatconverter.cpp b/src/api/common/src/fiatconverter.cpp index cbb92d1f..e4f27bc8 100644 --- a/src/api/common/src/fiatconverter.cpp +++ b/src/api/common/src/fiatconverter.cpp @@ -6,93 +6,74 @@ #include #include -#include "cct_json-container.hpp" #include "cct_log.hpp" #include "cct_string.hpp" #include "coincenterinfo.hpp" #include "curloptions.hpp" #include "currencycode.hpp" +#include "durationstring.hpp" +#include "fiats-converter-responses-schema.hpp" #include "file.hpp" #include "httprequesttype.hpp" #include "market.hpp" #include "permanentcurloptions.hpp" +#include "read-json.hpp" #include "reader.hpp" #include "timedef.hpp" +#include "timestring.hpp" +#include "write-json.hpp" namespace cct { namespace { constexpr std::string_view kRatesCacheFile = "ratescache.json"; +constexpr std::string_view kThirdPartySecretFileName = "thirdparty_secret.json"; constexpr std::string_view kFiatConverterSource1BaseUrl = "https://free.currconv.com"; constexpr std::string_view kFiatConverterSource2BaseUrl = "https://api.vatcomply.com/rates"; -string LoadCurrencyConverterAPIKey(std::string_view dataDir) { - static constexpr std::string_view kDefaultCommunityKey = "b25453de7984135a084b"; - // example http://free.currconv.com/api/v7/currencies?apiKey=b25453de7984135a084b - static constexpr std::string_view kThirdPartySecretFileName = "thirdparty_secret.json"; - - const File thirdPartySecret(dataDir, File::Type::kSecret, kThirdPartySecretFileName, File::IfError::kNoThrow); - json::container data = thirdPartySecret.readAllJson(); - auto freeConverterIt = data.find("freecurrencyconverter"); - if (freeConverterIt == data.end() || freeConverterIt->get() == kDefaultCommunityKey) { - log::warn("Unable to find custom Free Currency Converter key in {}", kThirdPartySecretFileName); - log::warn("If you want to use extensively coincenter, please create your own key by going to"); - log::warn("https://free.currencyconverterapi.com/free-api-key and place it in"); - log::warn("'{}/secret/{}' like this:", dataDir, kThirdPartySecretFileName); - log::warn(R"( {"freecurrencyconverter": ""})"); - log::warn("Using default key provided as a demo to the community"); - return string(kDefaultCommunityKey); - } - return std::move(freeConverterIt->get_ref()); +File GetRatesCacheFile(std::string_view dataDir) { + return {dataDir, File::Type::kCache, kRatesCacheFile, File::IfError::kNoThrow}; +} + +File GetThirdPartySecretFile(std::string_view dataDir) { + return {dataDir, File::Type::kSecret, kThirdPartySecretFileName, File::IfError::kNoThrow}; } } // namespace FiatConverter::FiatConverter(const CoincenterInfo& coincenterInfo, Duration ratesUpdateFrequency) - : FiatConverter(coincenterInfo, ratesUpdateFrequency, GetRatesCacheFile(coincenterInfo.dataDir())) {} + : FiatConverter(coincenterInfo, ratesUpdateFrequency, GetRatesCacheFile(coincenterInfo.dataDir()), + GetThirdPartySecretFile(coincenterInfo.dataDir())) {} FiatConverter::FiatConverter(const CoincenterInfo& coincenterInfo, Duration ratesUpdateFrequency, - const Reader& fiatsCacheReader) + const Reader& fiatsRatesCacheReader, const Reader& thirdPartySecretReader) : _curlHandle1(kFiatConverterSource1BaseUrl, coincenterInfo.metricGatewayPtr(), PermanentCurlOptions(), coincenterInfo.getRunMode()), _curlHandle2(kFiatConverterSource2BaseUrl, coincenterInfo.metricGatewayPtr(), PermanentCurlOptions(), coincenterInfo.getRunMode()), _ratesUpdateFrequency(ratesUpdateFrequency), - _apiKey(LoadCurrencyConverterAPIKey(coincenterInfo.dataDir())), + _thirdPartySecret(LoadCurrencyConverterAPIKey(thirdPartySecretReader)), _dataDir(coincenterInfo.dataDir()) { - const json::container data = fiatsCacheReader.readAllJson(); + const auto data = fiatsRatesCacheReader.readAll(); - _pricesMap.reserve(data.size()); - for (const auto& [marketStr, rateAndTimeData] : data.items()) { - const double rate = rateAndTimeData["rate"]; - const int64_t timeStamp = rateAndTimeData["timeepoch"]; + ReadJsonOrThrow(data, _pricesMap); - log::trace("Stored rate {} for market {} from {}", rate, marketStr, kRatesCacheFile); - _pricesMap.insert_or_assign(Market(marketStr, '-'), PriceTimedValue{rate, TimePoint(seconds(timeStamp))}); - } log::debug("Loaded {} fiat currency rates from {}", _pricesMap.size(), kRatesCacheFile); } -File FiatConverter::GetRatesCacheFile(std::string_view dataDir) { - return {dataDir, File::Type::kCache, kRatesCacheFile, File::IfError::kNoThrow}; -} - void FiatConverter::updateCacheFile() const { - json::container data; - for (const auto& [market, priceTimeValue] : _pricesMap) { - const string marketPairStr = market.assetsPairStrUpper('-'); - - data[marketPairStr]["rate"] = priceTimeValue.rate; - data[marketPairStr]["timeepoch"] = TimestampToSecondsSinceEpoch(priceTimeValue.lastUpdatedTime); - } - GetRatesCacheFile(_dataDir).writeJson(data); + auto dataStr = WriteJsonOrThrow(_pricesMap); + GetRatesCacheFile(_dataDir).write(dataStr); } std::optional FiatConverter::queryCurrencyRate(Market market) { - auto ret = queryCurrencyRateSource1(market); - if (ret) { - return ret; + std::optional ret; + if (!_thirdPartySecret.freecurrencyconverter.empty()) { + ret = queryCurrencyRateSource1(market); + if (ret) { + return ret; + } } ret = queryCurrencyRateSource2(market); return ret; @@ -101,59 +82,62 @@ std::optional FiatConverter::queryCurrencyRate(Market market) { std::optional FiatConverter::queryCurrencyRateSource1(Market market) { const auto qStr = market.assetsPairStrUpper('_'); - const CurlOptions opts(HttpRequestType::kGet, {{"q", qStr}, {"apiKey", _apiKey}}); + const CurlOptions opts(HttpRequestType::kGet, {{"q", qStr}, {"apiKey", _thirdPartySecret.freecurrencyconverter}}); const auto dataStr = _curlHandle1.query("/api/v7/convert", opts); - static constexpr bool kAllowExceptions = false; - const auto data = json::container::parse(dataStr, nullptr, kAllowExceptions); + schema::FreeCurrencyConverterResponse response; //{"query":{"count":1},"results":{"EUR_KRW":{"id":"EUR_KRW","val":1329.475323,"to":"KRW","fr":"EUR"}}} - const auto resultsIt = data.find("results"); - if (data.is_discarded() || resultsIt == data.end() || !resultsIt->contains(qStr)) { + auto ec = json::read(response, dataStr); + + if (ec) { + std::string_view prefixJsonContent = dataStr.substr(0, std::min(dataStr.size(), 20)); + log::error("Error while reading json content from fiat currency converter service's first source '{}{}': {}", + prefixJsonContent, prefixJsonContent.size() < dataStr.size() ? "..." : "", + json::format_error(ec, dataStr)); + return {}; + } + + const auto ratesIt = response.results.find(qStr); + if (ratesIt == response.results.end()) { log::warn("No JSON data received from fiat currency converter service's first source for pair '{}'", market); refreshLastUpdatedTime(market); return std::nullopt; } - const auto& rates = (*resultsIt)[qStr]; - const double rate = rates["val"]; + const double rate = ratesIt->second.val; store(market, rate); return rate; } std::optional FiatConverter::queryCurrencyRateSource2(Market market) { const auto dataStr = _curlHandle2.query("", CurlOptions(HttpRequestType::kGet)); - static constexpr bool kAllowExceptions = false; - const json::container jsonData = json::container::parse(dataStr, nullptr, kAllowExceptions); - if (jsonData.is_discarded()) { - log::error("Invalid response received from fiat currency converter service's second source"); - return {}; - } - const auto baseIt = jsonData.find("base"); - const auto ratesIt = jsonData.find("rates"); - if (baseIt == jsonData.end() || ratesIt == jsonData.end()) { - log::warn("No JSON data received from fiat currency converter service's second source", market); - return {}; - } - const TimePoint nowTime = Clock::now(); + schema::FiatRatesSource2Response response; - _baseRateSource2 = baseIt->get(); - for (const auto& [currencyCodeStr, rate] : ratesIt->items()) { - const double rateDouble = rate.get(); - const CurrencyCode currencyCode(currencyCodeStr); + auto ec = json::read(response, dataStr); - _pricesMap.insert_or_assign(Market(_baseRateSource2, currencyCode), PriceTimedValue(rateDouble, nowTime)); + if (ec) { + std::string_view prefixJsonContent = dataStr.substr(0, std::min(dataStr.size(), 20)); + log::error("Error while reading json content from fiat currency converter service's second source '{}{}': {}", + prefixJsonContent, prefixJsonContent.size() < dataStr.size() ? "..." : "", + json::format_error(ec, dataStr)); + return {}; } - return retrieveRateFromCache(market); + + for (const auto& [currencyCode, rateDouble] : response.rates) { + store(Market(response.base, currencyCode), rateDouble); + } + + return retrieveRateFromCache(market, CacheReadMode::kUseAllRates); } void FiatConverter::store(Market market, double rate) { log::debug("Stored rate {} for {}", rate, market); const TimePoint nowTime = Clock::now(); + const auto ts = TimestampToSecondsSinceEpoch(nowTime); - _pricesMap.insert_or_assign(market.reverse(), PriceTimedValue(static_cast(1) / rate, nowTime)); - _pricesMap.insert_or_assign(std::move(market), PriceTimedValue(rate, nowTime)); + _pricesMap.insert_or_assign(std::move(market), PriceTimedValue(rate, ts)); } void FiatConverter::refreshLastUpdatedTime(Market market) { @@ -161,9 +145,9 @@ void FiatConverter::refreshLastUpdatedTime(Market market) { if (it != _pricesMap.end()) { // Update cache time anyway to avoid querying too much the service const TimePoint nowTime = Clock::now(); + const auto ts = TimestampToSecondsSinceEpoch(nowTime); - it->second.lastUpdatedTime = nowTime; - _pricesMap[market.reverse()].lastUpdatedTime = nowTime; + it->second.timeepoch = ts; } } @@ -171,59 +155,141 @@ std::optional FiatConverter::convert(double amount, CurrencyCode from, C if (from == to) { return amount; } - const Market market(from, to); - double rate; + const Market market(from, to); std::lock_guard guard(_pricesMutex); - const auto optRate = retrieveRateFromCache(market); + // First query in the cache with not up to date rates + auto optRate = retrieveRateFromCache(market, CacheReadMode::kOnlyRecentRates); if (optRate) { - rate = *optRate; - } else { - if (_ratesUpdateFrequency == Duration::max()) { - log::error("Unable to query fiat currency rates and no rate found in cache for {}", market); - return {}; - } - std::optional queriedRate = queryCurrencyRate(market); - if (queriedRate) { - rate = *queriedRate; - } else { - const auto it = _pricesMap.find(market); - if (it == _pricesMap.end()) { - log::error("Unable to query fiat currency rates and no rate found in cache for {}", market); - return {}; - } - log::warn("Fiat currency rate service unavailable, use not up to date currency rate in cache"); - rate = it->second.rate; - } + return amount * *optRate; + } + + if (_ratesUpdateFrequency == Duration::max()) { + log::error("Fiat converter live queries disabled and no rate found in cache for {}", market); + return {}; + } + + // Updates the rates + optRate = queryCurrencyRate(market); + if (optRate) { + return amount * *optRate; } - return amount * rate; + // Query the rates from the update cache + optRate = retrieveRateFromCache(market, CacheReadMode::kUseAllRates); + if (optRate) { + return amount * *optRate; + } + + log::error("Unable to retrieve rate for {}", market); + return {}; } -std::optional FiatConverter::retrieveRateFromCache(Market market) const { - const auto rateIfYoung = [this, nowTime = Clock::now()](Market mk) -> std::optional { - const auto it = _pricesMap.find(mk); - if (it != _pricesMap.end() && nowTime - it->second.lastUpdatedTime < _ratesUpdateFrequency) { - return it->second.rate; +std::optional FiatConverter::retrieveRateFromCache(Market market, CacheReadMode cacheReadMode) { + // single rate check first + auto nowTime = Clock::now(); + + auto isPriceUpToDate = [this, nowTime](const auto& pair) { + return nowTime - pair.second.lastUpdatedTime() < _ratesUpdateFrequency; + }; + + auto directConversionIt = _pricesMap.find(market); + if (directConversionIt != _pricesMap.end() && + (cacheReadMode == CacheReadMode::kUseAllRates || isPriceUpToDate(*directConversionIt))) { + return directConversionIt->second.rate; + } + + if (cacheReadMode == CacheReadMode::kOnlyRecentRates) { + _tmpPriceRatesVector.clear(); + std::ranges::copy_if(_pricesMap, std::back_inserter(_tmpPriceRatesVector), isPriceUpToDate); + } else { + _tmpPriceRatesVector.resize(_pricesMap.size()); + std::ranges::copy(_pricesMap, _tmpPriceRatesVector.begin()); + } + + struct NodeCompare { + bool operator()(const Node& lhs, const Node& rhs) const { + // We will use a heap with the smallest currency path first to favor the rate paths with the least number of + // conversions + return rhs.currencyPath.size() < lhs.currencyPath.size(); } - return {}; }; - const auto directRate = rateIfYoung(market); - if (directRate) { - return directRate; - } - if (_baseRateSource2.isDefined()) { - // Try with dual rates from base source. - const auto rateBase1 = rateIfYoung(Market(_baseRateSource2, market.base())); - if (rateBase1) { - const auto rateBase2 = rateIfYoung(Market(_baseRateSource2, market.quote())); - if (rateBase2) { - return *rateBase2 / *rateBase1; + + NodeCompare comp; + + _nodes.resize(1, Node{Node::CurrencyPath(1U, market.base()), 1.0, nowTime}); + _visitedCurrencies.clear(); + + while (!_nodes.empty()) { + std::ranges::pop_heap(_nodes, comp); + Node node = std::move(_nodes.back()); + _nodes.pop_back(); + + auto cur = node.currencyPath.back(); + + // stop criteria + if (cur == market.quote()) { + _pricesMap.insert_or_assign(market, PriceTimedValue(node.rate, TimestampToSecondsSinceEpoch(node.oldestTs))); + return node.rate; + } + + if (node.currencyPath.size() == node.currencyPath.max_size()) { + log::warn("[fiat conversion] currency path too long for {}, stopping exploration", market); + continue; + } + + // Cache the visited currency to avoid exploration of same paths + if (_visitedCurrencies.contains(cur)) { + continue; + } + _visitedCurrencies.insert(cur); + + // generation of new nodes + for (const auto& [mk, priceTimedValue] : _tmpPriceRatesVector) { + if (cur == mk.base() && std::ranges::find(node.currencyPath, mk.quote()) == node.currencyPath.end()) { + auto curPath = node.currencyPath; + curPath.emplace_back(mk.quote()); + _nodes.emplace_back(std::move(curPath), node.rate * priceTimedValue.rate, + std::min(node.oldestTs, priceTimedValue.lastUpdatedTime())); + std::ranges::push_heap(_nodes, comp); + } else if (cur == mk.quote() && std::ranges::find(node.currencyPath, mk.base()) == node.currencyPath.end()) { + auto curPath = node.currencyPath; + curPath.emplace_back(mk.base()); + _nodes.emplace_back(std::move(curPath), node.rate / priceTimedValue.rate, + std::min(node.oldestTs, priceTimedValue.lastUpdatedTime())); + std::ranges::push_heap(_nodes, comp); } } } + return {}; } + +FiatConverter::ThirdPartySecret FiatConverter::LoadCurrencyConverterAPIKey(const Reader& thirdPartySecretReader) { + auto dataStr = thirdPartySecretReader.readAll(); + ThirdPartySecret thirdPartySecret; + + if (dataStr.empty()) { + log::debug("No third party secret file found in {}", kThirdPartySecretFileName); + return thirdPartySecret; + } + + auto ec = json::read(thirdPartySecret, dataStr); + + if (ec) { + std::string_view prefixJsonContent = dataStr.substr(0, std::min(dataStr.size(), 20)); + log::error("Error while reading json content from third party's secrets '{}{}': {}", prefixJsonContent, + prefixJsonContent.size() < dataStr.size() ? "..." : "", json::format_error(ec, dataStr)); + return thirdPartySecret; + } + + if (thirdPartySecret.freecurrencyconverter.empty()) { + log::debug("Unable to find custom Free Currency Converter key in {}", kThirdPartySecretFileName); + } + + return thirdPartySecret; +} + } // namespace cct diff --git a/src/api/common/src/fiats-converter-responses-schema.hpp b/src/api/common/src/fiats-converter-responses-schema.hpp new file mode 100644 index 00000000..c23516fc --- /dev/null +++ b/src/api/common/src/fiats-converter-responses-schema.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include + +#include "cct_string.hpp" +#include "currencycode.hpp" + +namespace cct::schema { + +struct FreeCurrencyConverterResponse { + struct Quote { + string id; + double val; + string to; + string fr; + }; + + std::unordered_map results; +}; + +struct FiatRatesSource2Response { + string date; + CurrencyCode base; + std::unordered_map rates; +}; + +} // namespace cct::schema \ No newline at end of file diff --git a/src/api/common/test/exchangeprivateapi_test.cpp b/src/api/common/test/exchangeprivateapi_test.cpp index b45f16df..ed301215 100644 --- a/src/api/common/test/exchangeprivateapi_test.cpp +++ b/src/api/common/test/exchangeprivateapi_test.cpp @@ -89,7 +89,9 @@ class ExchangePrivateTest : public ::testing::Test { LoadConfiguration loadConfiguration{kDefaultDataDir, LoadConfiguration::ExchangeConfigFileType::kTest}; CoincenterInfo coincenterInfo{settings::RunMode::kTestKeys, loadConfiguration}; CommonAPI commonAPI{coincenterInfo, Duration::max()}; - FiatConverter fiatConverter{coincenterInfo, Duration::max(), Reader()}; // max to avoid real Fiat converter queries + + // max to avoid real Fiat converter queries + FiatConverter fiatConverter{coincenterInfo, Duration::max(), Reader(), Reader()}; MockExchangePublic exchangePublic{ExchangeNameEnum::binance, fiatConverter, commonAPI, coincenterInfo}; APIKey key{"test", "testUser", "", "", ""}; diff --git a/src/api/common/test/exchangepublicapi_test.cpp b/src/api/common/test/exchangepublicapi_test.cpp index 0c2168ac..e4ba4366 100644 --- a/src/api/common/test/exchangepublicapi_test.cpp +++ b/src/api/common/test/exchangepublicapi_test.cpp @@ -56,7 +56,7 @@ class ExchangePublicTest : public ::testing::Test { CoincenterInfo coincenterInfo{runMode, loadConfiguration, schema::GeneralConfig(), LoggingInfo(), MonitoringInfo(), Reader(), StableCoinReader()}; CommonAPI commonAPI{coincenterInfo, Duration::max()}; - FiatConverter fiatConverter{coincenterInfo, Duration::max(), FiatConverterReader()}; + FiatConverter fiatConverter{coincenterInfo, Duration::max(), FiatConverterReader(), Reader()}; MockExchangePublic exchangePublic{ExchangeNameEnum::binance, fiatConverter, commonAPI, coincenterInfo}; MarketSet markets{{"BTC", "EUR"}, {"XLM", "EUR"}, {"ETH", "EUR"}, {"ETH", "BTC"}, {"BTC", "KRW"}, diff --git a/src/api/common/test/fiatconverter_test.cpp b/src/api/common/test/fiatconverter_test.cpp index ef1512fe..350becce 100644 --- a/src/api/common/test/fiatconverter_test.cpp +++ b/src/api/common/test/fiatconverter_test.cpp @@ -6,14 +6,15 @@ #include #include +#include "../src/fiats-converter-responses-schema.hpp" #include "besturlpicker.hpp" -#include "cct_json-container.hpp" #include "coincenterinfo.hpp" #include "curlhandle.hpp" #include "curloptions.hpp" #include "permanentcurloptions.hpp" #include "runmodes.hpp" #include "timedef.hpp" +#include "write-json.hpp" namespace cct { @@ -28,6 +29,17 @@ constexpr double kUSD = 1.21; constexpr double kGBP = 0.88; constexpr std::string_view kSomeFakeURL = "some/fake/url"; + +class DummyThirdPartyReader : public Reader { + [[nodiscard]] string readAll() const override { + return R"( +{ + "freecurrencyconverter": "blabla", + "exchangeratesapi": "blabla" +} +)"; + } +}; } // namespace CurlHandle::CurlHandle([[maybe_unused]] BestURLPicker bestURLPicker, @@ -38,65 +50,65 @@ CurlHandle::CurlHandle([[maybe_unused]] BestURLPicker bestURLPicker, // NOLINTNEXTLINE(readability-convert-member-functions-to-static) std::string_view CurlHandle::query([[maybe_unused]] std::string_view endpoint, const CurlOptions &opts) { - json::container jsonData; - // Rates std::string_view marketStr = opts.postData().get("q"); if (!marketStr.empty()) { - double rate = 0; + // First source std::string_view fromCurrency = marketStr.substr(0, 3); std::string_view targetCurrency = marketStr.substr(4); + schema::FreeCurrencyConverterResponse response; + + auto &res = response.results[string(marketStr)]; + + res.to = string(targetCurrency); + res.fr = string(fromCurrency); if (fromCurrency == "EUR") { if (targetCurrency == "KRW") { - rate = kKRW; + res.val = kKRW; } else if (targetCurrency == "USD") { - rate = kUSD; + res.val = kUSD; } else if (targetCurrency == "GBP") { - rate = kGBP; + res.val = kGBP; } } else if (fromCurrency == "KRW") { if (targetCurrency == "EUR") { - rate = 1 / kKRW; + res.val = 1 / kKRW; } else if (targetCurrency == "USD") { - rate = kUSD / kKRW; + res.val = kUSD / kKRW; } else if (targetCurrency == "GBP") { - rate = kGBP / kKRW; + res.val = kGBP / kKRW; } } else if (fromCurrency == "GBP") { if (targetCurrency == "USD") { - rate = kUSD / kGBP; + res.val = kUSD / kGBP; } } - if (rate != 0) { - jsonData["results"][marketStr]["val"] = rate; + if (res.val != 0) { + _queryData = WriteJsonOrThrow(response); } + } else { // second source - jsonData = R"( -{ - "base": "EUR", - "rates": { - "SUSHI": 36.78, - "KRW": 1341.88, - "NOK": 11.3375 - } -} -)"_json; + schema::FiatRatesSource2Response response; + response.base = "EUR"; + response.rates["SUSHI"] = 36.78; + response.rates["KRW"] = 1341.88; + response.rates["NOK"] = 11.3375; + _queryData = WriteJsonOrThrow(response); } - _queryData = jsonData.dump(); return _queryData; } -CurlHandle::~CurlHandle() {} // NOLINT +CurlHandle::~CurlHandle() = default; // NOLINT class FiatConverterTest : public ::testing::Test { protected: settings::RunMode runMode = settings::RunMode::kTestKeys; CoincenterInfo coincenterInfo{runMode}; - FiatConverter converter{coincenterInfo, milliseconds(1), Reader()}; + FiatConverter converter{coincenterInfo, milliseconds(1), Reader(), DummyThirdPartyReader()}; }; TEST_F(FiatConverterTest, DirectConversion) { @@ -117,7 +129,7 @@ TEST_F(FiatConverterTest, DoubleConversion) { AreDoubleEqual(converter.convert(amount, "KRW", "USD").value_or(0), (amount / kKRW) * kUSD); AreDoubleEqual(converter.convert(amount, "GBP", "USD").value_or(0), (amount / kGBP) * kUSD); - EXPECT_EQ(converter.convert(amount, "SUSHI", "KRW"), 729679173.46383917); + EXPECT_EQ(converter.convert(amount, "SUSHI", "KRW"), 729679173.46383893); } TEST_F(FiatConverterTest, NoConversionPossible) { diff --git a/src/api/exchanges/test/bithumb_place_order_test.cpp b/src/api/exchanges/test/bithumb_place_order_test.cpp index 2f4b431b..9bc7bf9c 100644 --- a/src/api/exchanges/test/bithumb_place_order_test.cpp +++ b/src/api/exchanges/test/bithumb_place_order_test.cpp @@ -39,7 +39,8 @@ class BithumbPrivateAPIPlaceOrderTest : public ::testing::Test { settings::RunMode runMode = settings::RunMode::kQueryResponseOverriden; LoadConfiguration loadConfig{kDefaultDataDir, LoadConfiguration::ExchangeConfigFileType::kTest}; CoincenterInfo coincenterInfo{runMode, loadConfig}; - FiatConverter fiatConverter{coincenterInfo, Duration::max(), Reader()}; // max to avoid real Fiat converter queries + // max to avoid real Fiat converter queries + FiatConverter fiatConverter{coincenterInfo, Duration::max(), Reader(), Reader()}; CommonAPI commonAPI{coincenterInfo, Duration::max()}; BithumbPublic exchangePublic{coincenterInfo, fiatConverter, commonAPI}; APIKeysProvider apiKeysProvider{coincenterInfo.dataDir(), coincenterInfo.getRunMode()}; diff --git a/src/api/exchanges/test/exchangecommonapi_test.hpp b/src/api/exchanges/test/exchangecommonapi_test.hpp index 08da1fcc..f6a91ac3 100644 --- a/src/api/exchanges/test/exchangecommonapi_test.hpp +++ b/src/api/exchanges/test/exchangecommonapi_test.hpp @@ -55,7 +55,8 @@ class TestAPI { LoadConfiguration loadConfig{kDefaultDataDir, LoadConfiguration::ExchangeConfigFileType::kTest}; CoincenterInfo coincenterInfo{runMode, loadConfig}; APIKeysProvider apiKeysProvider{coincenterInfo.dataDir(), coincenterInfo.getRunMode()}; - FiatConverter fiatConverter{coincenterInfo, Duration::max(), Reader()}; // max to avoid real Fiat converter queries + // max to avoid real Fiat converter queries + FiatConverter fiatConverter{coincenterInfo, Duration::max(), Reader(), Reader()}; CommonAPI commonAPI{coincenterInfo, Duration::max()}; PublicExchangeT exchangePublic{coincenterInfo, fiatConverter, commonAPI}; std::optional exchangePrivateOpt; diff --git a/src/api/interface/test/exchangeretriever_test.cpp b/src/api/interface/test/exchangeretriever_test.cpp index e0815df9..4181becc 100644 --- a/src/api/interface/test/exchangeretriever_test.cpp +++ b/src/api/interface/test/exchangeretriever_test.cpp @@ -34,7 +34,8 @@ class ExchangeRetrieverTest : public ::testing::Test { CoincenterInfo coincenterInfo{settings::RunMode::kTestKeys, loadConfiguration}; api::CommonAPI commonAPI{coincenterInfo, Duration::max()}; - FiatConverter fiatConverter{coincenterInfo, Duration::max(), Reader()}; // max to avoid real Fiat converter queries + // max to avoid real Fiat converter queries + FiatConverter fiatConverter{coincenterInfo, Duration::max(), Reader(), Reader()}; api::MockExchangePublic exchangePublic1{ExchangeNameEnum::bithumb, fiatConverter, commonAPI, coincenterInfo}; api::MockExchangePublic exchangePublic2{ExchangeNameEnum::kraken, fiatConverter, commonAPI, coincenterInfo}; diff --git a/src/basic-objects/include/market.hpp b/src/basic-objects/include/market.hpp index a9ff8ea3..b61c3221 100644 --- a/src/basic-objects/include/market.hpp +++ b/src/basic-objects/include/market.hpp @@ -163,7 +163,7 @@ struct from { static void op(auto &&value, is_context auto &&, It &&it, End &&end) noexcept { // used as a value. As a key, the first quote will not be present. auto endIt = std::find(*it == '"' ? ++it : it, end, '"'); - value = std::string_view(it, endIt); + value = ::cct::Market(std::string_view(it, endIt)); it = ++endIt; } }; diff --git a/src/basic-objects/test/market_test.cpp b/src/basic-objects/test/market_test.cpp index 36322450..2233091f 100644 --- a/src/basic-objects/test/market_test.cpp +++ b/src/basic-objects/test/market_test.cpp @@ -3,6 +3,7 @@ #include #include "cct_exception.hpp" +#include "cct_json-serialization.hpp" #include "currencycode.hpp" namespace cct { @@ -65,4 +66,46 @@ TEST(MarketTest, StrLen) { market = Market("1INCH", "EUR", Market::Type::kFiatConversionMarket); EXPECT_EQ(market.strLen(), 10); } + +struct Foo { + bool operator==(const Foo &) const noexcept = default; + + Market market; +}; + +TEST(MarketTest, JsonSerializationValue) { + Foo foo{Market{"DOGE", "BTC"}}; + + string buffer; + auto res = json::write(foo, buffer); // NOLINT(readability-implicit-bool-conversion) + + EXPECT_FALSE(res); + + EXPECT_EQ(buffer, R"({"market":"DOGE-BTC"})"); +} + +using MarketMap = std::map; + +TEST(MarketTest, JsonSerializationKey) { + MarketMap map{{Market{"DOGE", "BTC"}, true}, {Market{"BTC", "ETH"}, false}}; + + string buffer; + auto res = json::write(map, buffer); // NOLINT(readability-implicit-bool-conversion) + + EXPECT_FALSE(res); + + EXPECT_EQ(buffer, R"({"BTC-ETH":false,"DOGE-BTC":true})"); +} + +TEST(MarketTest, JsonDeserialization) { + Foo foo; + + // NOLINTNEXTLINE(readability-implicit-bool-conversion) + auto ec = json::read(foo, R"({"market":"DOGE-ETH"})"); + + ASSERT_FALSE(ec); + + EXPECT_EQ(foo, Foo{Market("DOGE", "ETH")}); +} + } // namespace cct diff --git a/src/engine/test/exchangedata_test.hpp b/src/engine/test/exchangedata_test.hpp index df6e889e..b4389bed 100644 --- a/src/engine/test/exchangedata_test.hpp +++ b/src/engine/test/exchangedata_test.hpp @@ -35,7 +35,8 @@ class ExchangesBaseTest : public ::testing::Test { CoincenterInfo coincenterInfo{runMode, loadConfiguration}; api::CommonAPI commonAPI{coincenterInfo, Duration::max(), Duration::max(), api::CommonAPI::AtInit::kLoadFromFileCache}; - FiatConverter fiatConverter{coincenterInfo, Duration::max(), Reader()}; // max to avoid real Fiat converter queries + // max to avoid real Fiat converter queries + FiatConverter fiatConverter{coincenterInfo, Duration::max(), Reader(), Reader()}; api::MockExchangePublic exchangePublic1{ExchangeNameEnum::binance, fiatConverter, commonAPI, coincenterInfo}; api::MockExchangePublic exchangePublic2{ExchangeNameEnum::bithumb, fiatConverter, commonAPI, coincenterInfo}; api::MockExchangePublic exchangePublic3{ExchangeNameEnum::huobi, fiatConverter, commonAPI, coincenterInfo}; diff --git a/src/objects/src/coincenterinfo.cpp b/src/objects/src/coincenterinfo.cpp index e7b3783b..85d7fc3d 100644 --- a/src/objects/src/coincenterinfo.cpp +++ b/src/objects/src/coincenterinfo.cpp @@ -6,7 +6,6 @@ #include #include "cct_exception.hpp" -#include "cct_json-container.hpp" #include "cct_log.hpp" #include "cct_string.hpp" #include "currencycode.hpp" @@ -14,6 +13,7 @@ #include "loadconfiguration.hpp" #include "logginginfo.hpp" #include "monitoringinfo.hpp" +#include "read-json.hpp" #include "reader.hpp" #include "runmodes.hpp" #include "toupperlower-string.hpp" @@ -28,26 +28,17 @@ namespace cct { namespace { -CoincenterInfo::CurrencyEquivalentAcronymMap ComputeCurrencyEquivalentAcronymMap( - const Reader& currencyAcronymsTranslatorReader) { - json::container jsonData = currencyAcronymsTranslatorReader.readAllJson(); + +CoincenterInfo::CurrencyEquivalentAcronymMap ComputeCurrencyEquivalentAcronymMap(const Reader& reader) { CoincenterInfo::CurrencyEquivalentAcronymMap map; - map.reserve(jsonData.size()); - for (const auto& [key, value] : jsonData.items()) { - log::trace("Currency {} <=> {}", key, value.get()); - map.insert_or_assign(CurrencyCode(key), value.get()); - } + ReadJsonOrThrow(reader.readAll(), map); return map; } -CoincenterInfo::StableCoinsMap ComputeStableCoinsMap(const Reader& stableCoinsReader) { - json::container jsonData = stableCoinsReader.readAllJson(); - CoincenterInfo::StableCoinsMap ret; - for (const auto& [key, value] : jsonData.items()) { - log::trace("Stable Crypto {} <=> {}", key, value.get()); - ret.emplace(key, value.get()); - } - return ret; +CoincenterInfo::StableCoinsMap ComputeStableCoinsMap(const Reader& reader) { + CoincenterInfo::StableCoinsMap map; + ReadJsonOrThrow(reader.readAll(), map); + return map; } #ifdef CCT_ENABLE_PROMETHEUS @@ -73,12 +64,11 @@ CoincenterInfo::CoincenterInfo(settings::RunMode runMode, const LoadConfiguratio ? new MetricGatewayType(monitoringInfo) : nullptr), _monitoringInfo(std::move(monitoringInfo)) { - json::container jsonData = currencyPrefixesReader.readAllJson(); - for (auto& [prefix, acronym_prefix] : jsonData.items()) { - log::trace("Currency prefix {} <=> {}", prefix, acronym_prefix.get()); + ReadJsonOrThrow(currencyPrefixesReader.readAll(), _currencyPrefixAcronymMap); + for (auto& [prefix, acronym_prefix] : _currencyPrefixAcronymMap) { + log::trace("Currency prefix {} <=> {}", prefix, acronym_prefix); _minPrefixLen = std::min(_minPrefixLen, static_cast(prefix.length())); _maxPrefixLen = std::max(_maxPrefixLen, static_cast(prefix.length())); - _currencyPrefixAcronymMap.insert_or_assign(ToUpper(prefix), std::move(acronym_prefix.get_ref())); } } diff --git a/src/schema/include/fiats-cache-schema.hpp b/src/schema/include/fiats-cache-schema.hpp new file mode 100644 index 00000000..7bf62689 --- /dev/null +++ b/src/schema/include/fiats-cache-schema.hpp @@ -0,0 +1,17 @@ +#pragma once + +#include + +#include "cct_vector.hpp" +#include "currencycode.hpp" + +namespace cct::schema { + +struct FiatsCache { + int64_t timeepoch{}; + vector fiats; + + using trivially_relocatable = is_trivially_relocatable>::type; +}; + +} // namespace cct::schema \ No newline at end of file diff --git a/src/schema/include/read-json.hpp b/src/schema/include/read-json.hpp index 89dd40b7..f6ee81cc 100644 --- a/src/schema/include/read-json.hpp +++ b/src/schema/include/read-json.hpp @@ -15,6 +15,10 @@ constexpr auto kJsonOptions = template void ReadJsonOrThrow(std::string_view strContent, auto &outObject) { + if (strContent.empty()) { + return; + } + auto ec = json::read(outObject, strContent); if (ec) { diff --git a/src/schema/include/secret-schema.hpp b/src/schema/include/secret-schema.hpp new file mode 100644 index 00000000..a4665039 --- /dev/null +++ b/src/schema/include/secret-schema.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include + +#include "cct_const.hpp" +#include "cct_json-serialization.hpp" +#include "cct_string.hpp" + +namespace cct::schema { + +struct AccountOwner { + string enName; + string koName; + + using trivially_relocatable = is_trivially_relocatable::type; +}; + +struct APIKey { + string key; + string priv; // private is a reserved keyword - we override the json field name below + string passphrase; + AccountOwner accountOwner; + + using trivially_relocatable = is_trivially_relocatable::type; +}; + +using APIKeys = std::unordered_map; + +using APIKeysPerExchangeMap = std::unordered_map; + +} // namespace cct::schema + +template <> +struct glz::meta<::cct::schema::APIKey> { + using V = ::cct::schema::APIKey; + static constexpr auto value = + object("key", &V::key, "private", &V::priv, "passphrase", &V::passphrase, "accountOwner", &V::accountOwner); +}; \ No newline at end of file