From 9052cf10f9e2e47b3d27cd8e92621fdc5f91ea77 Mon Sep 17 00:00:00 2001 From: Stephane Janel Date: Wed, 14 Feb 2024 23:35:47 +0100 Subject: [PATCH] Add second source for fiat conversion, not requiring any API key --- src/api/common/include/exchangebase.hpp | 14 +- src/api/common/include/fiatconverter.hpp | 26 +++- src/api/common/src/fiatconverter.cpp | 158 ++++++++++++++++----- src/api/common/test/fiatconverter_test.cpp | 103 ++++++++------ 4 files changed, 211 insertions(+), 90 deletions(-) diff --git a/src/api/common/include/exchangebase.hpp b/src/api/common/include/exchangebase.hpp index ce72770e..43066c15 100644 --- a/src/api/common/include/exchangebase.hpp +++ b/src/api/common/include/exchangebase.hpp @@ -21,13 +21,13 @@ class UniqueQueryHandle { } UniqueQueryHandle(const UniqueQueryHandle &) = delete; - UniqueQueryHandle(UniqueQueryHandle &&o) noexcept - : _pCachedResultVault(std::exchange(o._pCachedResultVault, nullptr)) {} + UniqueQueryHandle(UniqueQueryHandle &&rhs) noexcept + : _pCachedResultVault(std::exchange(rhs._pCachedResultVault, nullptr)) {} UniqueQueryHandle &operator=(const UniqueQueryHandle &) = delete; - UniqueQueryHandle &operator=(UniqueQueryHandle &&o) noexcept { - if (this != std::addressof(o)) { - _pCachedResultVault = std::exchange(o._pCachedResultVault, nullptr); + UniqueQueryHandle &operator=(UniqueQueryHandle &&rhs) noexcept { + if (this != std::addressof(rhs)) { + _pCachedResultVault = std::exchange(rhs._pCachedResultVault, nullptr); } return *this; } @@ -44,10 +44,10 @@ class UniqueQueryHandle { class ExchangeBase { public: - virtual void updateCacheFile() const {} - virtual ~ExchangeBase() = default; + virtual void updateCacheFile() const {} + protected: ExchangeBase() = default; }; diff --git a/src/api/common/include/fiatconverter.hpp b/src/api/common/include/fiatconverter.hpp index c66cbbe4..ab9caa39 100644 --- a/src/api/common/include/fiatconverter.hpp +++ b/src/api/common/include/fiatconverter.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include "cct_string.hpp" @@ -26,6 +27,8 @@ class CoincenterInfo; /// such that 'coincenter' uses it instead of the hardcoded one. The reason is that API services are hourly limited and /// reaching the limit would make it basically unusable for the community. /// +/// Fallback mechanism exists if api key does not exist or is expired. +/// /// Conversion methods are thread safe. class FiatConverter { public: @@ -33,10 +36,14 @@ class FiatConverter { /// @param ratesUpdateFrequency the minimum time needed between two currency rates updates FiatConverter(const CoincenterInfo &coincenterInfo, Duration ratesUpdateFrequency); - double convert(double amount, CurrencyCode from, CurrencyCode to); + std::optional convert(double amount, CurrencyCode from, CurrencyCode to); - MonetaryAmount convert(MonetaryAmount amount, CurrencyCode to) { - return MonetaryAmount(convert(amount.toDouble(), amount.currencyCode(), to), to); + std::optional convert(MonetaryAmount amount, CurrencyCode to) { + auto optDouble = convert(amount.toDouble(), amount.currencyCode(), to); + if (optDouble) { + return MonetaryAmount(*optDouble, to); + } + return {}; } /// Store rates in a file to make data persistent. @@ -51,13 +58,24 @@ class FiatConverter { std::optional queryCurrencyRate(Market mk); + std::optional queryCurrencyRateSource1(Market mk); + std::optional queryCurrencyRateSource2(Market mk); + + std::optional retrieveRateFromCache(Market mk) const; + + void store(Market mk, double rate); + + void refreshLastUpdatedTime(Market mk); + using PricesMap = std::unordered_map; - CurlHandle _curlHandle; + CurlHandle _curlHandle1; + CurlHandle _curlHandle2; PricesMap _pricesMap; Duration _ratesUpdateFrequency; std::mutex _pricesMutex; string _apiKey; string _dataDir; + CurrencyCode _baseRateSource2; }; } // namespace cct diff --git a/src/api/common/src/fiatconverter.cpp b/src/api/common/src/fiatconverter.cpp index d609198d..6467a51d 100644 --- a/src/api/common/src/fiatconverter.cpp +++ b/src/api/common/src/fiatconverter.cpp @@ -6,7 +6,6 @@ #include #include -#include "cct_exception.hpp" #include "cct_json.hpp" #include "cct_string.hpp" #include "coincenterinfo.hpp" @@ -21,11 +20,17 @@ namespace cct { namespace { +constexpr std::string_view kRatesCacheFile = "ratescache.json"; + +constexpr std::string_view kFiatConverterSource1BaseUrl = "https://free.currconv.com"; +constexpr std::string_view kFiatConverterSource2BaseUrl = "https://api.vatcomply.com/rates"; + string LoadCurrencyConverterAPIKey(std::string_view dataDir) { static constexpr std::string_view kDefaultCommunityKey = "b25453de7984135a084b"; // example http://free.currconv.com/api/v7/currencies?apiKey=b25453de7984135a084b static constexpr std::string_view kThirdPartySecretFileName = "thirdparty_secret.json"; - File thirdPartySecret(dataDir, File::Type::kSecret, kThirdPartySecretFileName, File::IfError::kNoThrow); + + const File thirdPartySecret(dataDir, File::Type::kSecret, kThirdPartySecretFileName, File::IfError::kNoThrow); json data = thirdPartySecret.readAllJson(); auto freeConverterIt = data.find("freecurrencyconverter"); if (freeConverterIt == data.end() || freeConverterIt->get() == kDefaultCommunityKey) { @@ -40,29 +45,30 @@ string LoadCurrencyConverterAPIKey(std::string_view dataDir) { return std::move(freeConverterIt->get_ref()); } -constexpr std::string_view kRatesCacheFile = "ratescache.json"; - File GetRatesCacheFile(std::string_view dataDir) { return {dataDir, File::Type::kCache, kRatesCacheFile, File::IfError::kNoThrow}; } -constexpr std::string_view kFiatConverterBaseUrl = "https://free.currconv.com"; } // namespace FiatConverter::FiatConverter(const CoincenterInfo& coincenterInfo, Duration ratesUpdateFrequency) - : _curlHandle(kFiatConverterBaseUrl, coincenterInfo.metricGatewayPtr(), PermanentCurlOptions(), - coincenterInfo.getRunMode()), + : _curlHandle1(kFiatConverterSource1BaseUrl, coincenterInfo.metricGatewayPtr(), PermanentCurlOptions(), + coincenterInfo.getRunMode()), + _curlHandle2(kFiatConverterSource2BaseUrl, coincenterInfo.metricGatewayPtr(), PermanentCurlOptions(), + coincenterInfo.getRunMode()), _ratesUpdateFrequency(ratesUpdateFrequency), _apiKey(LoadCurrencyConverterAPIKey(coincenterInfo.dataDir())), _dataDir(coincenterInfo.dataDir()) { - File ratesCacheFile = GetRatesCacheFile(_dataDir); - json data = ratesCacheFile.readAllJson(); + const File ratesCacheFile = GetRatesCacheFile(_dataDir); + const json data = ratesCacheFile.readAllJson(); + _pricesMap.reserve(data.size()); for (const auto& [marketStr, rateAndTimeData] : data.items()) { - double rate = rateAndTimeData["rate"]; - int64_t timeepoch = rateAndTimeData["timeepoch"]; + const double rate = rateAndTimeData["rate"]; + const int64_t timeStamp = rateAndTimeData["timeepoch"]; + log::trace("Stored rate {} for market {} from {}", rate, marketStr, kRatesCacheFile); - _pricesMap.insert_or_assign(Market(marketStr, '-'), PriceTimedValue{rate, TimePoint(TimeInS(timeepoch))}); + _pricesMap.insert_or_assign(Market(marketStr, '-'), PriceTimedValue{rate, TimePoint(TimeInS(timeStamp))}); } log::debug("Loaded {} fiat currency rates from {}", _pricesMap.size(), kRatesCacheFile); } @@ -70,7 +76,8 @@ FiatConverter::FiatConverter(const CoincenterInfo& coincenterInfo, Duration rate void FiatConverter::updateCacheFile() const { json data; for (const auto& [market, priceTimeValue] : _pricesMap) { - string marketPairStr = market.assetsPairStrUpper('-'); + const string marketPairStr = market.assetsPairStrUpper('-'); + data[marketPairStr]["rate"] = priceTimeValue.rate; data[marketPairStr]["timeepoch"] = TimestampToS(priceTimeValue.lastUpdatedTime); } @@ -78,53 +85,104 @@ void FiatConverter::updateCacheFile() const { } std::optional FiatConverter::queryCurrencyRate(Market mk) { - string qStr(mk.assetsPairStrUpper('_')); - CurlOptions opts(HttpRequestType::kGet, {{"q", qStr}, {"apiKey", _apiKey}}); - auto dataStr = _curlHandle.query("/api/v7/convert", opts); + auto ret = queryCurrencyRateSource1(mk); + if (ret) { + return ret; + } + ret = queryCurrencyRateSource2(mk); + return ret; +} + +std::optional FiatConverter::queryCurrencyRateSource1(Market mk) { + const auto qStr = mk.assetsPairStrUpper('_'); + + const CurlOptions opts(HttpRequestType::kGet, {{"q", qStr}, {"apiKey", _apiKey}}); + + const auto dataStr = _curlHandle1.query("/api/v7/convert", opts); + static constexpr bool kAllowExceptions = false; - auto data = json::parse(dataStr, nullptr, kAllowExceptions); + const auto data = json::parse(dataStr, nullptr, kAllowExceptions); + //{"query":{"count":1},"results":{"EUR_KRW":{"id":"EUR_KRW","val":1329.475323,"to":"KRW","fr":"EUR"}}} - auto resultsIt = data.find("results"); + const auto resultsIt = data.find("results"); if (data == json::value_t::discarded || resultsIt == data.end() || !resultsIt->contains(qStr)) { - log::error("No JSON data received from fiat currency converter service for pair '{}'", mk); - auto it = _pricesMap.find(mk); - if (it != _pricesMap.end()) { - // Update cache time anyway to avoid querying too much the service - TimePoint nowTime = Clock::now(); - it->second.lastUpdatedTime = nowTime; - _pricesMap[mk.reverse()].lastUpdatedTime = nowTime; - } + log::warn("No JSON data received from fiat currency converter service's first source for pair '{}'", mk); + refreshLastUpdatedTime(mk); return std::nullopt; } const auto& rates = (*resultsIt)[qStr]; - double rate = rates["val"]; - log::debug("Stored rate {} for market {}", rate, qStr); - TimePoint nowTime = Clock::now(); - _pricesMap.insert_or_assign(mk.reverse(), PriceTimedValue{static_cast(1) / rate, nowTime}); - _pricesMap.insert_or_assign(std::move(mk), PriceTimedValue{rate, nowTime}); + const double rate = rates["val"]; + store(mk, rate); return rate; } -double FiatConverter::convert(double amount, CurrencyCode from, CurrencyCode to) { +std::optional FiatConverter::queryCurrencyRateSource2(Market mk) { + const auto dataStr = _curlHandle2.query("", CurlOptions(HttpRequestType::kGet)); + const json jsonData = json::parse(dataStr); + const auto baseIt = jsonData.find("base"); + const auto ratesIt = jsonData.find("rates"); + if (baseIt == jsonData.end() || ratesIt == jsonData.end()) { + log::warn("No JSON data received from fiat currency converter service's second source", mk); + return {}; + } + + const TimePoint nowTime = Clock::now(); + + _baseRateSource2 = baseIt->get(); + for (const auto& [currencyCodeStr, rate] : ratesIt->items()) { + const double rateDouble = rate.get(); + const CurrencyCode currencyCode(currencyCodeStr); + + _pricesMap.insert_or_assign(Market(_baseRateSource2, currencyCode), PriceTimedValue(rateDouble, nowTime)); + } + return retrieveRateFromCache(mk); +} + +void FiatConverter::store(Market mk, double rate) { + log::debug("Stored rate {} for {}", rate, mk); + const TimePoint nowTime = Clock::now(); + + _pricesMap.insert_or_assign(mk.reverse(), PriceTimedValue(static_cast(1) / rate, nowTime)); + _pricesMap.insert_or_assign(std::move(mk), PriceTimedValue(rate, nowTime)); +} + +void FiatConverter::refreshLastUpdatedTime(Market mk) { + const auto it = _pricesMap.find(mk); + if (it != _pricesMap.end()) { + // Update cache time anyway to avoid querying too much the service + const TimePoint nowTime = Clock::now(); + + it->second.lastUpdatedTime = nowTime; + _pricesMap[mk.reverse()].lastUpdatedTime = nowTime; + } +} + +std::optional FiatConverter::convert(double amount, CurrencyCode from, CurrencyCode to) { if (from == to) { return amount; } - Market mk(from, to); + const Market mk(from, to); + double rate; + std::lock_guard guard(_pricesMutex); - auto it = _pricesMap.find(mk); - if (it != _pricesMap.end() && Clock::now() - it->second.lastUpdatedTime < _ratesUpdateFrequency) { - rate = it->second.rate; + + const auto optRate = retrieveRateFromCache(mk); + if (optRate) { + rate = *optRate; } else { if (_ratesUpdateFrequency == Duration::max()) { - throw exception("Unable to query fiat currency rates and no rate found in cache"); + log::error("Unable to query fiat currency rates and no rate found in cache for {}", mk); + return {}; } std::optional queriedRate = queryCurrencyRate(mk); if (queriedRate) { rate = *queriedRate; } else { + const auto it = _pricesMap.find(mk); if (it == _pricesMap.end()) { - throw exception("Unable to query fiat currency rates and no rate found in cache"); + log::error("Unable to query fiat currency rates and no rate found in cache for {}", mk); + return {}; } log::warn("Fiat currency rate service unavailable, use not up to date currency rate in cache"); rate = it->second.rate; @@ -134,4 +192,28 @@ double FiatConverter::convert(double amount, CurrencyCode from, CurrencyCode to) return amount * rate; } +std::optional FiatConverter::retrieveRateFromCache(Market mk) const { + const auto rateIfYoung = [this, nowTime = Clock::now()](Market mk) -> std::optional { + const auto it = _pricesMap.find(mk); + if (it != _pricesMap.end() && nowTime - it->second.lastUpdatedTime < _ratesUpdateFrequency) { + return it->second.rate; + } + return {}; + }; + const auto directRate = rateIfYoung(mk); + if (directRate) { + return directRate; + } + if (_baseRateSource2.isDefined()) { + // Try with dual rates from base source. + const auto rateBase1 = rateIfYoung(Market(_baseRateSource2, mk.base())); + if (rateBase1) { + const auto rateBase2 = rateIfYoung(Market(_baseRateSource2, mk.quote())); + if (rateBase2) { + return *rateBase2 / *rateBase1; + } + } + } + return {}; +} } // namespace cct diff --git a/src/api/common/test/fiatconverter_test.cpp b/src/api/common/test/fiatconverter_test.cpp index adb9a1e5..f94f6baf 100644 --- a/src/api/common/test/fiatconverter_test.cpp +++ b/src/api/common/test/fiatconverter_test.cpp @@ -2,10 +2,11 @@ #include +#include +#include #include #include "besturlpicker.hpp" -#include "cct_exception.hpp" #include "cct_json.hpp" #include "coincenterinfo.hpp" #include "curlhandle.hpp" @@ -19,11 +20,7 @@ namespace cct { namespace { void AreDoubleEqual(double lhs, double rhs) { static constexpr double kEpsilon = 0.00000001; - if (lhs < rhs) { - EXPECT_LT(rhs - lhs, kEpsilon); - } else { - EXPECT_LT(lhs - rhs, kEpsilon); - } + EXPECT_LT(std::abs(rhs - lhs), kEpsilon); } constexpr double kKRW = 1341.88; @@ -45,33 +42,48 @@ std::string_view CurlHandle::query([[maybe_unused]] std::string_view endpoint, c // Rates std::string_view marketStr = opts.postData().get("q"); - std::string_view fromCurrency = marketStr.substr(0, 3); - std::string_view targetCurrency = marketStr.substr(4); - double rate = 0; - if (fromCurrency == "EUR") { - if (targetCurrency == "KRW") { - rate = kKRW; - } else if (targetCurrency == "USD") { - rate = kUSD; - } else if (targetCurrency == "GBP") { - rate = kGBP; - } - } else if (fromCurrency == "KRW") { - if (targetCurrency == "EUR") { - rate = 1 / kKRW; - } else if (targetCurrency == "USD") { - rate = kUSD / kKRW; - } else if (targetCurrency == "GBP") { - rate = kGBP / kKRW; + if (!marketStr.empty()) { + double rate = 0; + + std::string_view fromCurrency = marketStr.substr(0, 3); + std::string_view targetCurrency = marketStr.substr(4); + + if (fromCurrency == "EUR") { + if (targetCurrency == "KRW") { + rate = kKRW; + } else if (targetCurrency == "USD") { + rate = kUSD; + } else if (targetCurrency == "GBP") { + rate = kGBP; + } + } else if (fromCurrency == "KRW") { + if (targetCurrency == "EUR") { + rate = 1 / kKRW; + } else if (targetCurrency == "USD") { + rate = kUSD / kKRW; + } else if (targetCurrency == "GBP") { + rate = kGBP / kKRW; + } + } else if (fromCurrency == "GBP") { + if (targetCurrency == "USD") { + rate = kUSD / kGBP; + } } - } else if (fromCurrency == "GBP") { - if (targetCurrency == "USD") { - rate = kUSD / kGBP; + if (rate != 0) { + jsonData["results"][marketStr]["val"] = rate; } + } else { + // second source + jsonData = R"( +{ + "base": "EUR", + "rates": { + "SUSHI": 36.78, + "KRW": 1341.88, + "NOK": 11.3375 } - - if (rate != 0) { - jsonData["results"][marketStr]["val"] = rate; +} +)"_json; } _queryData = jsonData.dump(); @@ -88,20 +100,29 @@ class FiatConverterTest : public ::testing::Test { }; TEST_F(FiatConverterTest, DirectConversion) { - const double amount = 10; - AreDoubleEqual(converter.convert(amount, "KRW", "KRW"), amount); - AreDoubleEqual(converter.convert(amount, "EUR", "KRW"), amount * kKRW); - AreDoubleEqual(converter.convert(amount, "EUR", "USD"), amount * kUSD); - AreDoubleEqual(converter.convert(amount, "EUR", "GBP"), amount * kGBP); - EXPECT_THROW(converter.convert(amount, "EUR", "SUSHI"), exception); + constexpr double amount = 10; + + AreDoubleEqual(converter.convert(amount, "KRW", "KRW").value(), amount); + AreDoubleEqual(converter.convert(amount, "EUR", "KRW").value(), amount * kKRW); + AreDoubleEqual(converter.convert(amount, "EUR", "USD").value(), amount * kUSD); + AreDoubleEqual(converter.convert(amount, "EUR", "GBP").value(), amount * kGBP); + + EXPECT_EQ(converter.convert(amount, "EUR", "SUSHI"), 367.8); } TEST_F(FiatConverterTest, DoubleConversion) { - const double amount = 20'000'000; - AreDoubleEqual(converter.convert(amount, "KRW", "EUR"), amount / kKRW); - AreDoubleEqual(converter.convert(amount, "KRW", "USD"), (amount / kKRW) * kUSD); - AreDoubleEqual(converter.convert(amount, "GBP", "USD"), (amount / kGBP) * kUSD); - EXPECT_THROW(converter.convert(amount, "SUSHI", "EUR"), exception); + constexpr double amount = 20'000'000; + + AreDoubleEqual(converter.convert(amount, "KRW", "EUR").value(), amount / kKRW); + AreDoubleEqual(converter.convert(amount, "KRW", "USD").value(), (amount / kKRW) * kUSD); + AreDoubleEqual(converter.convert(amount, "GBP", "USD").value(), (amount / kGBP) * kUSD); + + EXPECT_EQ(converter.convert(amount, "SUSHI", "KRW"), 729679173.46383917); +} + +TEST_F(FiatConverterTest, NoConversionPossible) { + constexpr double amount = 10; + EXPECT_EQ(converter.convert(amount, "SUSHI", "USD"), std::nullopt); } } // namespace cct