diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Governance.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Governance.hs index 33c6332f1..c1dfda141 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Governance.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Governance.hs @@ -71,7 +71,7 @@ drepDistr = newCommittee :: IOManager -> [(Text, Text)] -> Assertion newCommittee = - withFullConfigAndLogs conwayConfigDir testLabel $ \interpreter server dbSync -> do + withFullConfig conwayConfigDir testLabel $ \interpreter server dbSync -> do startDBSync dbSync -- Add stake diff --git a/cardano-db-sync/src/Cardano/DbSync.hs b/cardano-db-sync/src/Cardano/DbSync.hs index 9df654d4c..0db24f4a4 100644 --- a/cardano-db-sync/src/Cardano/DbSync.hs +++ b/cardano-db-sync/src/Cardano/DbSync.hs @@ -204,7 +204,7 @@ runSyncNode metricsSetters trce iomgr dbConnString ranMigrations runMigrationFnc unless useLedger $ liftIO $ do logInfo trce "Migrating to a no ledger schema" Db.noLedgerMigrations backend trce - insertValidateGenesisDist syncEnv (dncNetworkName syncNodeConfigFromFile) genCfg (useShelleyInit syncNodeConfigFromFile) + insertValidateGenesisDist syncEnv Nothing (dncNetworkName syncNodeConfigFromFile) genCfg (useShelleyInit syncNodeConfigFromFile) -- communication channel between datalayer thread and chainsync-client thread threadChannels <- liftIO newThreadChannels diff --git a/cardano-db-sync/src/Cardano/DbSync/Api/Ledger.hs b/cardano-db-sync/src/Cardano/DbSync/Api/Ledger.hs index 399541c49..7f9269d78 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Api/Ledger.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Api/Ledger.hs @@ -51,16 +51,18 @@ import qualified Ouroboros.Consensus.Shelley.Ledger.Ledger as Consensus bootStrapMaybe :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> + SlotDetails -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -bootStrapMaybe syncEnv = do +bootStrapMaybe syncEnv slotDetails = do bts <- liftIO $ readTVarIO (envBootstrap syncEnv) - when bts $ migrateBootstrapUTxO syncEnv + when bts $ migrateBootstrapUTxO syncEnv slotDetails migrateBootstrapUTxO :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> + SlotDetails -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -migrateBootstrapUTxO syncEnv = do +migrateBootstrapUTxO syncEnv slotDetails = do case envLedgerEnv syncEnv of HasLedger lenv -> do liftIO $ logInfo trce "Starting UTxO bootstrap migration" @@ -70,7 +72,7 @@ migrateBootstrapUTxO syncEnv = do liftIO $ logWarning trce $ "Found and deleted " <> textShow count <> " tx_out." - storeUTxOFromLedger syncEnv cls + storeUTxOFromLedger syncEnv slotDetails cls lift $ DB.insertExtraMigration DB.BootstrapFinished liftIO $ logInfo trce "UTxO bootstrap migration done" liftIO $ atomically $ writeTVar (envBootstrap syncEnv) False @@ -79,10 +81,15 @@ migrateBootstrapUTxO syncEnv = do where trce = getTrace syncEnv -storeUTxOFromLedger :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> ExtLedgerState CardanoBlock -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -storeUTxOFromLedger env st = case ledgerState st of - LedgerStateBabbage bts -> storeUTxO env (getUTxO bts) - LedgerStateConway stc -> storeUTxO env (getUTxO stc) +storeUTxOFromLedger :: + (MonadBaseControl IO m, MonadIO m) => + SyncEnv -> + SlotDetails -> + ExtLedgerState CardanoBlock -> + ExceptT SyncNodeError (ReaderT SqlBackend m) () +storeUTxOFromLedger env slotDetails st = case ledgerState st of + LedgerStateBabbage bts -> storeUTxO env slotDetails (getUTxO bts) + LedgerStateConway stc -> storeUTxO env slotDetails (getUTxO stc) _otherwise -> liftIO $ logError trce "storeUTxOFromLedger is only supported after Babbage" where trce = getTrace env @@ -104,9 +111,10 @@ storeUTxO :: , NativeScript era ~ Timelock era ) => SyncEnv -> + SlotDetails -> Map (TxIn StandardCrypto) (BabbageTxOut era) -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -storeUTxO env mp = do +storeUTxO env slotDetails mp = do liftIO $ logInfo trce $ mconcat @@ -115,7 +123,7 @@ storeUTxO env mp = do , " tx_out as pages of " , textShow pageSize ] - mapM_ (storePage env pagePerc) . zip [0 ..] . chunksOf pageSize . Map.toList $ mp + mapM_ (storePage env slotDetails pagePerc) . zip [0 ..] . chunksOf pageSize . Map.toList $ mp where trce = getTrace env npages = size `div` pageSize @@ -134,12 +142,13 @@ storePage :: , MonadBaseControl IO m ) => SyncEnv -> + SlotDetails -> Float -> (Int, [(TxIn StandardCrypto, BabbageTxOut era)]) -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -storePage syncEnv percQuantum (n, ls) = do +storePage syncEnv slotDetails percQuantum (n, ls) = do when (n `mod` 10 == 0) $ liftIO $ logInfo trce $ "Bootstrap in progress " <> prc <> "%" - txOuts <- mapM (prepareTxOut syncEnv) ls + txOuts <- mapM (prepareTxOut syncEnv slotDetails) ls txOutIds <- lift . DB.insertManyTxOut False $ etoTxOut . fst <$> txOuts let maTxOuts = concatMap (mkmaTxOuts txOutTableType) $ zip txOutIds (snd <$> txOuts) @@ -161,12 +170,13 @@ prepareTxOut :: , NativeScript era ~ Timelock era ) => SyncEnv -> + SlotDetails -> (TxIn StandardCrypto, BabbageTxOut era) -> ExceptT SyncNodeError (ReaderT SqlBackend m) (ExtendedTxOut, [MissingMaTxOut]) -prepareTxOut syncEnv (TxIn txIntxId (TxIx index), txOut) = do +prepareTxOut syncEnv slotDetails (TxIn txIntxId (TxIx index), txOut) = do let txHashByteString = Generic.safeHashToByteString $ unTxId txIntxId let genTxOut = fromTxOut index txOut - txId <- liftLookupFail "prepareTxOut" $ queryTxIdWithCache cache txIntxId + txId <- liftLookupFail "prepareTxOut" $ queryTxIdWithCache slotDetails cache txIntxId insertTxOut trce cache iopts (txId, txHashByteString) genTxOut where trce = getTrace syncEnv diff --git a/cardano-db-sync/src/Cardano/DbSync/Cache.hs b/cardano-db-sync/src/Cardano/DbSync/Cache.hs index 67818311e..8f18fd8af 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Cache.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Cache.hs @@ -22,6 +22,7 @@ module Cardano.DbSync.Cache ( queryStakeAddrWithCache, queryTxIdWithCache, rollbackCache, + deleteSyncingCaches, tryUpdateCacheTx, -- * CacheStatistics @@ -39,6 +40,7 @@ import Cardano.DbSync.Era.Shelley.Query import Cardano.DbSync.Era.Util import Cardano.DbSync.Error import Cardano.DbSync.Types +import Cardano.DbSync.Util (getSyncStatusInHalfHour) import qualified Cardano.Ledger.Address as Ledger import Cardano.Ledger.BaseTypes (Network) import Cardano.Ledger.Mary.Value @@ -80,6 +82,17 @@ rollbackCache (ActiveCache cache) blockId = do atomically $ modifyTVar (cTxIds cache) FIFO.cleanupCache void $ rollbackMapEpochInCache cache blockId +-- TODO: need to work out which caches to clean up. +deleteSyncingCaches :: MonadIO m => CacheStatus -> ReaderT SqlBackend m () +deleteSyncingCaches NoCache = pure () +deleteSyncingCaches (ActiveCache cache) = do + liftIO $ do + atomically $ writeTVar (cPrevBlock cache) Nothing + atomically $ modifyTVar (cDatum cache) LRU.cleanup + atomically $ modifyTVar (cTxIds cache) FIFO.cleanupCache + atomically $ writeTVar (cStake cache) (StakeCache Map.empty (LRU.empty 0)) + atomically $ writeTVar (cMultiAssets cache) (LRU.empty 0) + getCacheStatistics :: CacheStatus -> IO CacheStatistics getCacheStatistics cs = case cs of @@ -193,42 +206,54 @@ deleteStakeCache scred scache = queryPoolKeyWithCache :: MonadIO m => + Maybe SlotDetails -> CacheStatus -> CacheAction -> PoolKeyHash -> ReaderT SqlBackend m (Either DB.LookupFail DB.PoolHashId) -queryPoolKeyWithCache cache cacheUA hsh = - case cache of - NoCache -> do - mPhId <- DB.queryPoolHashId (Generic.unKeyHashRaw hsh) - case mPhId of - Nothing -> pure $ Left (DB.DbLookupMessage "PoolKeyHash") - Just phId -> pure $ Right phId - ActiveCache ci -> do - mp <- liftIO $ readTVarIO (cPools ci) - case Map.lookup hsh mp of - Just phId -> do - liftIO $ hitPools (cStats ci) - -- hit so we can't cache even with 'CacheNew' - when (cacheUA == EvictAndUpdateCache) $ - liftIO $ - atomically $ - modifyTVar (cPools ci) $ - Map.delete hsh - pure $ Right phId - Nothing -> do - liftIO $ missPools (cStats ci) +queryPoolKeyWithCache mSlotDetails cache cacheUA hsh = + case mSlotDetails of + Nothing -> query + Just slotDetails -> + case getSyncStatusInHalfHour slotDetails of + SyncFollowing -> do mPhId <- DB.queryPoolHashId (Generic.unKeyHashRaw hsh) case mPhId of Nothing -> pure $ Left (DB.DbLookupMessage "PoolKeyHash") - Just phId -> do - -- missed so we can't evict even with 'EvictAndReturn' - when (shouldCache cacheUA) $ - liftIO $ - atomically $ - modifyTVar (cPools ci) $ - Map.insert hsh phId - pure $ Right phId + Just phId -> pure $ Right phId + SyncLagging -> query + where + query = case cache of + NoCache -> do + mPhId <- DB.queryPoolHashId (Generic.unKeyHashRaw hsh) + case mPhId of + Nothing -> pure $ Left (DB.DbLookupMessage "PoolKeyHash") + Just phId -> pure $ Right phId + ActiveCache ci -> do + mp <- liftIO $ readTVarIO (cPools ci) + case Map.lookup hsh mp of + Just phId -> do + liftIO $ hitPools (cStats ci) + -- hit so we can't cache even with 'CacheNew' + when (cacheUA == EvictAndUpdateCache) $ + liftIO $ + atomically $ + modifyTVar (cPools ci) $ + Map.delete hsh + pure $ Right phId + Nothing -> do + liftIO $ missPools (cStats ci) + mPhId <- DB.queryPoolHashId (Generic.unKeyHashRaw hsh) + case mPhId of + Nothing -> pure $ Left (DB.DbLookupMessage "PoolKeyHash") + Just phId -> do + -- missed so we can't evict even with 'EvictAndReturn' + when (shouldCache cacheUA) $ + liftIO $ + atomically $ + modifyTVar (cPools ci) $ + Map.insert hsh phId + pure $ Right phId insertPoolKeyWithCache :: (MonadBaseControl IO m, MonadIO m) => @@ -274,13 +299,14 @@ queryPoolKeyOrInsert :: (MonadBaseControl IO m, MonadIO m) => Text -> Trace IO Text -> + Maybe SlotDetails -> CacheStatus -> CacheAction -> Bool -> PoolKeyHash -> ReaderT SqlBackend m DB.PoolHashId -queryPoolKeyOrInsert txt trce cache cacheUA logsWarning hsh = do - pk <- queryPoolKeyWithCache cache cacheUA hsh +queryPoolKeyOrInsert txt trce slotDetails cache cacheUA logsWarning hsh = do + pk <- queryPoolKeyWithCache slotDetails cache cacheUA hsh case pk of Right poolHashId -> pure poolHashId Left err -> do @@ -329,24 +355,28 @@ queryMAWithCache cache policyId asset = queryPrevBlockWithCache :: MonadIO m => + SlotDetails -> Text -> CacheStatus -> ByteString -> ExceptT SyncNodeError (ReaderT SqlBackend m) DB.BlockId -queryPrevBlockWithCache msg cache hsh = - case cache of - NoCache -> liftLookupFail msg $ DB.queryBlockId hsh - ActiveCache ci -> do - mCachedPrev <- liftIO $ readTVarIO (cPrevBlock ci) - case mCachedPrev of - -- if the cached block matches the requested hash, we return its db id. - Just (cachedBlockId, cachedHash) -> - if cachedHash == hsh - then do - liftIO $ hitPBlock (cStats ci) - pure cachedBlockId - else queryFromDb ci - Nothing -> queryFromDb ci +queryPrevBlockWithCache slotDetails msg cache hsh = + case getSyncStatusInHalfHour slotDetails of + SyncFollowing -> liftLookupFail msg $ DB.queryBlockId hsh + SyncLagging -> + case cache of + NoCache -> liftLookupFail msg $ DB.queryBlockId hsh + ActiveCache ci -> do + mCachedPrev <- liftIO $ readTVarIO (cPrevBlock ci) + case mCachedPrev of + -- if the cached block matches the requested hash, we return its db id. + Just (cachedBlockId, cachedHash) -> + if cachedHash == hsh + then do + liftIO $ hitPBlock (cStats ci) + pure cachedBlockId + else queryFromDb ci + Nothing -> queryFromDb ci where queryFromDb :: MonadIO m => @@ -358,34 +388,38 @@ queryPrevBlockWithCache msg cache hsh = queryTxIdWithCache :: MonadIO m => + SlotDetails -> CacheStatus -> Ledger.TxId StandardCrypto -> ReaderT SqlBackend m (Either DB.LookupFail DB.TxId) -queryTxIdWithCache cache txIdLedger = do - case cache of - -- Direct database query if no cache. - NoCache -> DB.queryTxId txHash - ActiveCache cacheInternal -> do - -- Read current cache state. - cacheTx <- liftIO $ readTVarIO (cTxIds cacheInternal) - - case FIFO.lookup txIdLedger cacheTx of - -- Cache hit, return the transaction ID. - Just txId -> do - liftIO $ hitTxIds (cStats cacheInternal) - pure $ Right txId - -- Cache miss. - Nothing -> do - eTxId <- DB.queryTxId txHash - liftIO $ missTxIds (cStats cacheInternal) - case eTxId of - Right txId -> do - -- Update cache. - liftIO $ atomically $ modifyTVar (cTxIds cacheInternal) $ FIFO.insert txIdLedger txId - -- Return ID after updating cache. +queryTxIdWithCache slotDetails cache txIdLedger = do + case getSyncStatusInHalfHour slotDetails of + SyncFollowing -> DB.queryTxId txHash + SyncLagging -> + case cache of + -- Direct database query if no cache. + NoCache -> DB.queryTxId txHash + ActiveCache cacheInternal -> do + -- Read current cache state. + cacheTx <- liftIO $ readTVarIO (cTxIds cacheInternal) + + case FIFO.lookup txIdLedger cacheTx of + -- Cache hit, return the transaction ID. + Just txId -> do + liftIO $ hitTxIds (cStats cacheInternal) pure $ Right txId - -- Return lookup failure. - Left _ -> pure $ Left $ DB.DbLookupTxHash txHash + -- Cache miss. + Nothing -> do + eTxId <- DB.queryTxId txHash + liftIO $ missTxIds (cStats cacheInternal) + case eTxId of + Right txId -> do + -- Update cache. + liftIO $ atomically $ modifyTVar (cTxIds cacheInternal) $ FIFO.insert txIdLedger txId + -- Return ID after updating cache. + pure $ Right txId + -- Return lookup failure. + Left _ -> pure $ Left $ DB.DbLookupTxHash txHash where txHash = Generic.unTxHash txIdLedger @@ -403,18 +437,22 @@ tryUpdateCacheTx cache ledgerTxId txId = do insertBlockAndCache :: (MonadIO m, MonadBaseControl IO m) => + SlotDetails -> CacheStatus -> DB.Block -> ReaderT SqlBackend m DB.BlockId -insertBlockAndCache cache block = +insertBlockAndCache slotDetails cache block = case cache of NoCache -> DB.insertBlock block - ActiveCache ci -> do - bid <- DB.insertBlock block - liftIO $ do - missPrevBlock (cStats ci) - atomically $ writeTVar (cPrevBlock ci) $ Just (bid, DB.blockHash block) - pure bid + ActiveCache ci -> + case getSyncStatusInHalfHour slotDetails of + SyncFollowing -> DB.insertBlock block + SyncLagging -> do + bid <- DB.insertBlock block + liftIO $ do + missPrevBlock (cStats ci) + atomically $ writeTVar (cPrevBlock ci) $ Just (bid, DB.blockHash block) + pure bid queryDatum :: MonadIO m => diff --git a/cardano-db-sync/src/Cardano/DbSync/Default.hs b/cardano-db-sync/src/Cardano/DbSync/Default.hs index 010ee9fcc..30a2ded15 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Default.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Default.hs @@ -83,7 +83,7 @@ applyAndInsertBlockMaybe syncEnv tracer cblk = do , ". Time to restore consistency." ] rollbackFromBlockNo syncEnv (blockNo cblk) - void $ migrateStakeDistr syncEnv (apOldLedger applyRes) + void $ migrateStakeDistr syncEnv Nothing (apOldLedger applyRes) insertBlock syncEnv cblk applyRes True tookSnapshot liftIO $ setConsistentLevel syncEnv Consistent Right blockId | Just (adaPots, slotNo, epochNo) <- getAdaPots applyRes -> do @@ -129,7 +129,7 @@ insertBlock syncEnv cblk applyRes firstAfterRollback tookSnapshot = do let !details = apSlotDetails applyResult let !withinTwoMin = isWithinTwoMin details let !withinHalfHour = isWithinHalfHour details - insertNewEpochLedgerEvents syncEnv (sdEpochNo details) (apEvents applyResult) + insertNewEpochLedgerEvents syncEnv details (sdEpochNo details) (apEvents applyResult) let isNewEpochEvent = hasNewEpochEvent (apEvents applyResult) let isStartEventOrRollback = hasEpochStartEvent (apEvents applyResult) || firstAfterRollback let isMember poolId = Set.member poolId (apPoolsRegistered applyResult) @@ -149,7 +149,7 @@ insertBlock syncEnv cblk applyRes firstAfterRollback tookSnapshot = do case cblk of BlockByron blk -> newExceptT $ - insertByronBlock syncEnv isStartEventOrRollback blk details + insertByronBlock syncEnv details isStartEventOrRollback blk details BlockShelley blk -> newExceptT $ insertBlockUniversal' $ @@ -180,7 +180,7 @@ insertBlock syncEnv cblk applyRes firstAfterRollback tookSnapshot = do when (unBlockNo blkNo `mod` getPruneInterval syncEnv == 0) $ do lift $ DB.deleteConsumedTxOut tracer txOutTableType (getSafeBlockNoDiff syncEnv) - commitOrIndexes withinTwoMin withinHalfHour + commitOrIndexes details withinTwoMin withinHalfHour where tracer = getTrace syncEnv txOutTableType = getTxOutTableType syncEnv @@ -203,8 +203,8 @@ insertBlock syncEnv cblk applyRes firstAfterRollback tookSnapshot = do Strict.Nothing | hasLedgerState syncEnv -> Just $ Ledger.Prices minBound minBound Strict.Nothing -> Nothing - commitOrIndexes :: Bool -> Bool -> ExceptT SyncNodeError (ReaderT SqlBackend (LoggingT IO)) () - commitOrIndexes withinTwoMin withinHalfHour = do + commitOrIndexes :: SlotDetails -> Bool -> Bool -> ExceptT SyncNodeError (ReaderT SqlBackend (LoggingT IO)) () + commitOrIndexes slotDetails withinTwoMin withinHalfHour = do commited <- if withinTwoMin || tookSnapshot then do @@ -212,7 +212,7 @@ insertBlock syncEnv cblk applyRes firstAfterRollback tookSnapshot = do pure True else pure False when withinHalfHour $ do - bootStrapMaybe syncEnv + bootStrapMaybe syncEnv slotDetails ranIndexes <- liftIO $ getRanIndexes syncEnv lift $ addConstraintsIfNotExist syncEnv tracer unless ranIndexes $ do diff --git a/cardano-db-sync/src/Cardano/DbSync/Era.hs b/cardano-db-sync/src/Cardano/DbSync/Era.hs index 32c203f20..8bec48447 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era.hs @@ -11,16 +11,18 @@ import Cardano.DbSync.Config import qualified Cardano.DbSync.Era.Byron.Genesis as Byron import qualified Cardano.DbSync.Era.Shelley.Genesis as Shelley import Cardano.DbSync.Error +import Cardano.DbSync.Types (SlotDetails) import Cardano.Prelude insertValidateGenesisDist :: SyncEnv -> + Maybe SlotDetails -> NetworkName -> GenesisConfig -> Bool -> ExceptT SyncNodeError IO () -insertValidateGenesisDist syncEnv nname genCfg shelleyInitiation = +insertValidateGenesisDist syncEnv mSlotDetails nname genCfg shelleyInitiation = case genCfg of GenesisCardano _ bCfg sCfg _aCfg _ -> do Byron.insertValidateGenesisDist syncEnv nname bCfg - Shelley.insertValidateGenesisDist syncEnv (unNetworkName nname) (scConfig sCfg) shelleyInitiation + Shelley.insertValidateGenesisDist syncEnv mSlotDetails (unNetworkName nname) (scConfig sCfg) shelleyInitiation diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Byron/Insert.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Byron/Insert.hs index 90e03c85f..1822cece1 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Byron/Insert.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Byron/Insert.hs @@ -54,15 +54,16 @@ data ValueFee = ValueFee insertByronBlock :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> + SlotDetails -> Bool -> ByronBlock -> SlotDetails -> ReaderT SqlBackend m (Either SyncNodeError ()) -insertByronBlock syncEnv firstBlockOfEpoch blk details = do +insertByronBlock syncEnv slotDetails firstBlockOfEpoch blk details = do res <- runExceptT $ case byronBlockRaw blk of - Byron.ABOBBlock ablk -> insertABlock syncEnv firstBlockOfEpoch ablk details - Byron.ABOBBoundary abblk -> insertABOBBoundary syncEnv abblk details + Byron.ABOBBlock ablk -> insertABlock syncEnv slotDetails firstBlockOfEpoch ablk details + Byron.ABOBBoundary abblk -> insertABOBBoundary syncEnv slotDetails abblk details -- Serializing things during syncing can drastically slow down full sync -- times (ie 10x or more). when @@ -73,14 +74,15 @@ insertByronBlock syncEnv firstBlockOfEpoch blk details = do insertABOBBoundary :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> + SlotDetails -> Byron.ABoundaryBlock ByteString -> SlotDetails -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertABOBBoundary syncEnv blk details = do +insertABOBBoundary syncEnv slotDetails blk details = do let tracer = getTrace syncEnv cache = envCache syncEnv -- Will not get called in the OBFT part of the Byron era. - pbid <- queryPrevBlockWithCache "insertABOBBoundary" cache (Byron.ebbPrevHash blk) + pbid <- queryPrevBlockWithCache slotDetails "insertABOBBoundary" cache (Byron.ebbPrevHash blk) let epochNo = unEpochNo $ sdEpochNo details slid <- lift . DB.insertSlotLeader $ @@ -90,7 +92,7 @@ insertABOBBoundary syncEnv blk details = do , DB.slotLeaderDescription = "Epoch boundary slot leader" } blkId <- - lift . insertBlockAndCache cache $ + lift . insertBlockAndCache slotDetails cache $ DB.Block { DB.blockHash = Byron.unHeaderHash $ Byron.boundaryHashAnnotated blk , DB.blockEpochNo = Just epochNo @@ -139,16 +141,17 @@ insertABOBBoundary syncEnv blk details = do insertABlock :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> + SlotDetails -> Bool -> Byron.ABlock ByteString -> SlotDetails -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertABlock syncEnv firstBlockOfEpoch blk details = do - pbid <- queryPrevBlockWithCache "insertABlock" cache (Byron.blockPreviousHash blk) +insertABlock syncEnv slotDetails firstBlockOfEpoch blk details = do + pbid <- queryPrevBlockWithCache slotDetails "insertABlock" cache (Byron.blockPreviousHash blk) slid <- lift . DB.insertSlotLeader $ Byron.mkSlotLeader blk let txs = Byron.blockPayload blk blkId <- - lift . insertBlockAndCache cache $ + lift . insertBlockAndCache slotDetails cache $ DB.Block { DB.blockHash = Byron.blockHash blk , DB.blockEpochNo = Just $ unEpochNo (sdEpochNo details) diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Genesis.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Genesis.hs index 0dcde23af..8e0a4f29e 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Genesis.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Genesis.hs @@ -25,6 +25,7 @@ import Cardano.DbSync.Era.Universal.Insert.Other (insertStakeAddressRefIfMissing import Cardano.DbSync.Era.Universal.Insert.Pool (insertPoolRegister) import Cardano.DbSync.Era.Util (liftLookupFail) import Cardano.DbSync.Error +import Cardano.DbSync.Types (SlotDetails) import Cardano.DbSync.Util import Cardano.Ledger.Address (serialiseAddr) import qualified Cardano.Ledger.Coin as Ledger @@ -60,11 +61,12 @@ import Paths_cardano_db_sync (version) -- 'shelleyInitiation' is True for testnets that fork at 0 to Shelley. insertValidateGenesisDist :: SyncEnv -> + Maybe SlotDetails -> Text -> ShelleyGenesis StandardCrypto -> Bool -> ExceptT SyncNodeError IO () -insertValidateGenesisDist syncEnv networkName cfg shelleyInitiation = do +insertValidateGenesisDist syncEnv mSlotDetails networkName cfg shelleyInitiation = do let prunes = getPrunes syncEnv -- Setting this to True will log all 'Persistent' operations which is great -- for debugging, but otherwise *way* too chatty. @@ -158,7 +160,7 @@ insertValidateGenesisDist syncEnv networkName cfg shelleyInitiation = do "Initial genesis distribution populated. Hash " <> renderByteArray (configGenesisHash cfg) when hasStakes $ - insertStaking tracer useNoCache bid cfg + insertStaking tracer mSlotDetails useNoCache bid cfg -- | Validate that the initial Genesis distribution in the DB matches the Genesis data. validateGenesisDistribution :: @@ -312,11 +314,12 @@ insertTxOuts syncEnv trce blkId (TxIn txInId _, txOut) = do insertStaking :: (MonadBaseControl IO m, MonadIO m) => Trace IO Text -> + Maybe SlotDetails -> CacheStatus -> DB.BlockId -> ShelleyGenesis StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertStaking tracer cache blkId genesis = do +insertStaking tracer mSlotDetails cache blkId genesis = do -- All Genesis staking comes from an artifical transaction -- with a hash generated by hashing the address. txId <- @@ -344,7 +347,7 @@ insertStaking tracer cache blkId genesis = do forM_ stakes $ \(n, (keyStaking, keyPool)) -> do -- TODO: add initial deposits for genesis stake keys. insertStakeRegistration tracer cache (EpochNo 0) Nothing txId (2 * n) (Generic.annotateStakingCred network (KeyHashObj keyStaking)) - insertDelegation tracer cache network (EpochNo 0) 0 txId (2 * n + 1) Nothing (KeyHashObj keyStaking) keyPool + insertDelegation tracer mSlotDetails cache network (EpochNo 0) 0 txId (2 * n + 1) Nothing (KeyHashObj keyStaking) keyPool -- ----------------------------------------------------------------------------- diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Adjust.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Adjust.hs index 942e6fc82..0d15f1d10 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Adjust.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Adjust.hs @@ -15,7 +15,7 @@ import Cardano.DbSync.Cache ( ) import Cardano.DbSync.Cache.Types (CacheAction (..), CacheStatus) import qualified Cardano.DbSync.Era.Shelley.Generic.Rewards as Generic -import Cardano.DbSync.Types (StakeCred) +import Cardano.DbSync.Types (SlotDetails, StakeCred) import Cardano.Ledger.BaseTypes (Network) import Cardano.Prelude hiding (from, groupBy, on) import Cardano.Slotting.Slot (EpochNo (..)) @@ -50,12 +50,13 @@ adjustEpochRewards :: (MonadBaseControl IO m, MonadIO m) => Trace IO Text -> Network -> + SlotDetails -> CacheStatus -> EpochNo -> Generic.Rewards -> Set StakeCred -> ReaderT SqlBackend m () -adjustEpochRewards trce nw cache epochNo rwds creds = do +adjustEpochRewards trce nw slotDetails cache epochNo rwds creds = do let eraIgnored = Map.toList $ Generic.unRewards rwds liftIO . logInfo trce $ mconcat @@ -66,7 +67,7 @@ adjustEpochRewards trce nw cache epochNo rwds creds = do ] forM_ eraIgnored $ \(cred, rewards) -> forM_ (Set.toList rewards) $ \rwd -> - deleteReward trce nw cache epochNo (cred, rwd) + deleteReward trce nw cache slotDetails epochNo (cred, rwd) crds <- rights <$> forM (Set.toList creds) (queryStakeAddrWithCache trce cache DoNotUpdateCache nw) deleteOrphanedRewards epochNo crds @@ -75,12 +76,13 @@ deleteReward :: Trace IO Text -> Network -> CacheStatus -> + SlotDetails -> EpochNo -> (StakeCred, Generic.Reward) -> ReaderT SqlBackend m () -deleteReward trce nw cache epochNo (cred, rwd) = do +deleteReward trce nw cache slotDetails epochNo (cred, rwd) = do mAddrId <- queryStakeAddrWithCache trce cache DoNotUpdateCache nw cred - eiPoolId <- queryPoolKeyWithCache cache DoNotUpdateCache (Generic.rewardPool rwd) + eiPoolId <- queryPoolKeyWithCache (Just slotDetails) cache DoNotUpdateCache (Generic.rewardPool rwd) case (mAddrId, eiPoolId) of (Right addrId, Right poolId) -> do delete $ do diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Block.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Block.hs index 2eed5603c..0613db7de 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Block.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Block.hs @@ -16,6 +16,7 @@ import qualified Cardano.Db as DB import Cardano.DbSync.Api import Cardano.DbSync.Api.Types (InsertOptions (..), SyncEnv (..), SyncOptions (..)) import Cardano.DbSync.Cache ( + deleteSyncingCaches, insertBlockAndCache, queryPoolKeyWithCache, queryPrevBlockWithCache, @@ -64,16 +65,18 @@ insertBlockUniversal :: ApplyResult -> ReaderT SqlBackend m (Either SyncNodeError ()) insertBlockUniversal syncEnv shouldLog withinTwoMins withinHalfHour blk details isMember applyResult = do + -- if we're syncing within 30 mins of the tip, we should delete the syncing caches. + when (getSyncStatusInHalfHour details == SyncFollowing) $ deleteSyncingCaches cache runExceptT $ do pbid <- case Generic.blkPreviousHash blk of Nothing -> liftLookupFail (renderErrorMessage (Generic.blkEra blk)) DB.queryGenesis -- this is for networks that fork from Byron on epoch 0. - Just pHash -> queryPrevBlockWithCache (renderErrorMessage (Generic.blkEra blk)) cache pHash - mPhid <- lift $ queryPoolKeyWithCache cache UpdateCache $ coerceKeyRole $ Generic.blkSlotLeader blk + Just pHash -> queryPrevBlockWithCache details (renderErrorMessage (Generic.blkEra blk)) cache pHash + mPhid <- lift $ queryPoolKeyWithCache (Just details) cache UpdateCache $ coerceKeyRole $ Generic.blkSlotLeader blk let epochNo = sdEpochNo details slid <- lift . DB.insertSlotLeader $ Generic.mkSlotLeader (ioShelley iopts) (Generic.unKeyHashRaw $ Generic.blkSlotLeader blk) (eitherToMaybe mPhid) blkId <- - lift . insertBlockAndCache cache $ + lift . insertBlockAndCache details cache $ DB.Block { DB.blockHash = Generic.blkHash blk , DB.blockEpochNo = Just $ unEpochNo epochNo @@ -94,7 +97,7 @@ insertBlockUniversal syncEnv shouldLog withinTwoMins withinHalfHour blk details } let zippedTx = zip [0 ..] (Generic.blkTxs blk) - let txInserter = insertTx syncEnv isMember blkId (sdEpochNo details) (Generic.blkSlotNo blk) applyResult + let txInserter = insertTx syncEnv details isMember blkId (sdEpochNo details) (Generic.blkSlotNo blk) applyResult blockGroupedData <- foldM (\gp (idx, tx) -> txInserter idx tx gp) mempty zippedTx minIds <- insertBlockGroupedData syncEnv blockGroupedData @@ -147,9 +150,9 @@ insertBlockUniversal syncEnv shouldLog withinTwoMins withinHalfHour blk details ] whenStrictJust (apNewEpoch applyResult) $ \newEpoch -> do - insertOnNewEpoch syncEnv blkId (Generic.blkSlotNo blk) epochNo newEpoch + insertOnNewEpoch syncEnv details blkId (Generic.blkSlotNo blk) epochNo newEpoch - insertStakeSlice syncEnv $ apStakeSlice applyResult + insertStakeSlice syncEnv (Just details) $ apStakeSlice applyResult when (ioGov iopts && (withinHalfHour || unBlockNo (Generic.blkBlockNo blk) `mod` 10000 == 0)) . lift diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Epoch.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Epoch.hs index cc1f86205..ee60e710d 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Epoch.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Epoch.hs @@ -64,12 +64,13 @@ import Database.Persist.Sql (SqlBackend) insertOnNewEpoch :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> + SlotDetails -> DB.BlockId -> SlotNo -> EpochNo -> Generic.NewEpoch -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertOnNewEpoch syncEnv blkId slotNo epochNo newEpoch = do +insertOnNewEpoch syncEnv slotDetails blkId slotNo epochNo newEpoch = do whenStrictJust (Generic.euProtoParams epochUpdate) $ \params -> lift $ insertEpochParam tracer blkId epochNo params (Generic.euNonce epochUpdate) whenStrictJust (Generic.neAdaPots newEpoch) $ \pots -> @@ -77,18 +78,18 @@ insertOnNewEpoch syncEnv blkId slotNo epochNo newEpoch = do spoVoting <- whenStrictJustDefault Map.empty (Generic.neDRepState newEpoch) $ \dreps -> whenDefault Map.empty (ioGov iopts) $ do let (drepSnapshot, ratifyState) = finishDRepPulser dreps lift $ insertDrepDistr epochNo drepSnapshot - updateRatified cache epochNo (toList $ rsEnacted ratifyState) - updateExpired cache epochNo (toList $ rsExpired ratifyState) + updateRatified slotDetails cache epochNo (toList $ rsEnacted ratifyState) + updateExpired slotDetails cache epochNo (toList $ rsExpired ratifyState) pure (Ledger.psPoolDistr drepSnapshot) whenStrictJust (Generic.neEnacted newEpoch) $ \enactedSt -> do when (ioGov iopts) $ do - insertUpdateEnacted tracer cache blkId epochNo enactedSt + insertUpdateEnacted tracer slotDetails cache blkId epochNo enactedSt whenStrictJust (Generic.nePoolDistr newEpoch) $ \(poolDistrDeleg, poolDistrNBlocks) -> when (ioPoolStats iopts) $ do let nothingMap = Map.fromList $ (,Nothing) <$> (Map.keys poolDistrNBlocks <> Map.keys spoVoting) let mapWithAllKeys = Map.union (Map.map Just poolDistrDeleg) nothingMap let poolStats = Map.mapWithKey (mkPoolStats poolDistrNBlocks spoVoting) mapWithAllKeys - lift $ insertPoolStats syncEnv epochNo poolStats + lift $ insertPoolStats syncEnv slotDetails epochNo poolStats where epochUpdate :: Generic.EpochUpdate epochUpdate = Generic.neEpochUpdate newEpoch @@ -196,11 +197,12 @@ hasEpochStartEvent = any isNewEpoch insertStakeSlice :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> + Maybe SlotDetails -> Generic.StakeSliceRes -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertStakeSlice _ Generic.NoSlices = pure () -insertStakeSlice syncEnv (Generic.Slice slice finalSlice) = do - insertEpochStake syncEnv network (Generic.sliceEpochNo slice) (Map.toList $ Generic.sliceDistr slice) +insertStakeSlice _ _ Generic.NoSlices = pure () +insertStakeSlice syncEnv slotDetails (Generic.Slice slice finalSlice) = do + insertEpochStake syncEnv slotDetails network (Generic.sliceEpochNo slice) (Map.toList $ Generic.sliceDistr slice) when finalSlice $ do lift $ DB.updateSetComplete $ unEpochNo $ Generic.sliceEpochNo slice size <- lift $ DB.queryEpochStakeCount (unEpochNo $ Generic.sliceEpochNo slice) @@ -217,11 +219,12 @@ insertStakeSlice syncEnv (Generic.Slice slice finalSlice) = do insertEpochStake :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> + Maybe SlotDetails -> Network -> EpochNo -> [(StakeCred, (Shelley.Coin, PoolKeyHash))] -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertEpochStake syncEnv nw epochNo stakeChunk = do +insertEpochStake syncEnv slotDetails nw epochNo stakeChunk = do let cache = envCache syncEnv DB.ManualDbConstraints {..} <- liftIO $ readTVarIO $ envDbConstraints syncEnv dbStakes <- mapM (mkStake cache) stakeChunk @@ -236,7 +239,7 @@ insertEpochStake syncEnv nw epochNo stakeChunk = do ExceptT SyncNodeError (ReaderT SqlBackend m) DB.EpochStake mkStake cache (saddr, (coin, pool)) = do saId <- lift $ queryOrInsertStakeAddress trce cache UpdateCacheStrong nw saddr - poolId <- lift $ queryPoolKeyOrInsert "insertEpochStake" trce cache UpdateCache (ioShelley iopts) pool + poolId <- lift $ queryPoolKeyOrInsert "insertEpochStake" trce slotDetails cache UpdateCache (ioShelley iopts) pool pure $ DB.EpochStake { DB.epochStakeAddrId = saId @@ -251,13 +254,14 @@ insertEpochStake syncEnv nw epochNo stakeChunk = do insertRewards :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> + SlotDetails -> Network -> EpochNo -> EpochNo -> CacheStatus -> [(StakeCred, Set Generic.Reward)] -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertRewards syncEnv nw earnedEpoch spendableEpoch cache rewardsChunk = do +insertRewards syncEnv slotDetails nw earnedEpoch spendableEpoch cache rewardsChunk = do DB.ManualDbConstraints {..} <- liftIO $ readTVarIO $ envDbConstraints syncEnv dbRewards <- concatMapM mkRewards rewardsChunk let chunckDbRewards = splittRecordsEvery 100000 dbRewards @@ -294,7 +298,7 @@ insertRewards syncEnv nw earnedEpoch spendableEpoch cache rewardsChunk = do PoolKeyHash -> ExceptT SyncNodeError (ReaderT SqlBackend m) DB.PoolHashId queryPool poolHash = - lift (queryPoolKeyOrInsert "insertRewards" trce cache UpdateCache (ioShelley iopts) poolHash) + lift (queryPoolKeyOrInsert "insertRewards" trce (Just slotDetails) cache UpdateCache (ioShelley iopts) poolHash) trce = getTrace syncEnv iopts = getInsertOptions syncEnv @@ -374,11 +378,12 @@ splittRecordsEvery val = go insertPoolDepositRefunds :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> + SlotDetails -> EpochNo -> Generic.Rewards -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertPoolDepositRefunds syncEnv epochNo refunds = do - insertRewards syncEnv nw epochNo epochNo (envCache syncEnv) (Map.toList rwds) +insertPoolDepositRefunds syncEnv slotDetails epochNo refunds = do + insertRewards syncEnv slotDetails nw epochNo epochNo (envCache syncEnv) (Map.toList rwds) liftIO . logInfo tracer $ "Inserted " <> show (Generic.rewardsCount refunds) <> " deposit refund rewards" where tracer = getTrace syncEnv @@ -397,16 +402,17 @@ insertPoolStats :: forall m. (MonadBaseControl IO m, MonadIO m) => SyncEnv -> + SlotDetails -> EpochNo -> Map PoolKeyHash Generic.PoolStats -> ReaderT SqlBackend m () -insertPoolStats syncEnv epochNo mp = do +insertPoolStats syncEnv slotDetails epochNo mp = do poolStats <- mapM preparePoolStat $ Map.toList mp DB.insertManyPoolStat poolStats where preparePoolStat :: (PoolKeyHash, Generic.PoolStats) -> ReaderT SqlBackend m DB.PoolStat preparePoolStat (pkh, ps) = do - poolId <- queryPoolKeyOrInsert "insertPoolStats" trce cache UpdateCache True pkh + poolId <- queryPoolKeyOrInsert "insertPoolStats" trce (Just slotDetails) cache UpdateCache True pkh pure DB.PoolStat { DB.poolStatPoolHashId = poolId diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Certificate.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Certificate.hs index 46aac293a..2e5ef9691 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Certificate.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Certificate.hs @@ -60,6 +60,7 @@ import Ouroboros.Consensus.Cardano.Block (StandardCrypto) insertCertificate :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> + SlotDetails -> IsPoolMember -> Maybe Generic.Deposits -> DB.BlockId -> @@ -69,12 +70,12 @@ insertCertificate :: Map Word64 DB.RedeemerId -> Generic.TxCertificate -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertCertificate syncEnv isMember mDeposits blkId txId epochNo slotNo redeemers (Generic.TxCertificate ridx idx cert) = +insertCertificate syncEnv slotDetails isMember mDeposits blkId txId epochNo slotNo redeemers (Generic.TxCertificate ridx idx cert) = case cert of Left (ShelleyTxCertDelegCert deleg) -> - when (ioShelley iopts) $ insertDelegCert tracer cache mDeposits network txId idx mRedeemerId epochNo slotNo deleg + when (ioShelley iopts) $ insertDelegCert tracer slotDetails cache mDeposits network txId idx mRedeemerId epochNo slotNo deleg Left (ShelleyTxCertPool pool) -> - when (ioShelley iopts) $ insertPoolCert tracer cache isMember mDeposits network epochNo blkId txId idx pool + when (ioShelley iopts) $ insertPoolCert tracer slotDetails cache isMember mDeposits network epochNo blkId txId idx pool Left (ShelleyTxCertMir mir) -> when (ioShelley iopts) $ insertMirCert tracer cache network txId idx mir Left (ShelleyTxCertGenesisDeleg _gen) -> @@ -82,9 +83,9 @@ insertCertificate syncEnv isMember mDeposits blkId txId epochNo slotNo redeemers liftIO $ logWarning tracer "insertCertificate: Unhandled DCertGenesis certificate" Right (ConwayTxCertDeleg deleg) -> - insertConwayDelegCert syncEnv mDeposits txId idx mRedeemerId epochNo slotNo deleg + insertConwayDelegCert syncEnv slotDetails mDeposits txId idx mRedeemerId epochNo slotNo deleg Right (ConwayTxCertPool pool) -> - when (ioShelley iopts) $ insertPoolCert tracer cache isMember mDeposits network epochNo blkId txId idx pool + when (ioShelley iopts) $ insertPoolCert tracer slotDetails cache isMember mDeposits network epochNo blkId txId idx pool Right (ConwayTxCertGov c) -> when (ioGov iopts) $ case c of ConwayRegDRep cred coin anchor -> @@ -107,6 +108,7 @@ insertCertificate syncEnv isMember mDeposits blkId txId epochNo slotNo redeemers insertDelegCert :: (MonadBaseControl IO m, MonadIO m) => Trace IO Text -> + SlotDetails -> CacheStatus -> Maybe Generic.Deposits -> Ledger.Network -> @@ -117,15 +119,16 @@ insertDelegCert :: SlotNo -> ShelleyDelegCert StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertDelegCert tracer cache mDeposits network txId idx mRedeemerId epochNo slotNo dCert = +insertDelegCert tracer slotDetails cache mDeposits network txId idx mRedeemerId epochNo slotNo dCert = case dCert of ShelleyRegCert cred -> insertStakeRegistration tracer cache epochNo mDeposits txId idx $ Generic.annotateStakingCred network cred ShelleyUnRegCert cred -> insertStakeDeregistration tracer cache network epochNo txId idx mRedeemerId cred - ShelleyDelegCert cred poolkh -> insertDelegation tracer cache network epochNo slotNo txId idx mRedeemerId cred poolkh + ShelleyDelegCert cred poolkh -> insertDelegation tracer (Just slotDetails) cache network epochNo slotNo txId idx mRedeemerId cred poolkh insertConwayDelegCert :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> + SlotDetails -> Maybe Generic.Deposits -> DB.TxId -> Word16 -> @@ -134,7 +137,7 @@ insertConwayDelegCert :: SlotNo -> ConwayDelegCert StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertConwayDelegCert syncEnv mDeposits txId idx mRedeemerId epochNo slotNo dCert = +insertConwayDelegCert syncEnv slotDetails mDeposits txId idx mRedeemerId epochNo slotNo dCert = case dCert of ConwayRegCert cred _dep -> when (ioShelley iopts) $ @@ -153,13 +156,13 @@ insertConwayDelegCert syncEnv mDeposits txId idx mRedeemerId epochNo slotNo dCer insertDeleg cred = \case DelegStake poolkh -> when (ioShelley iopts) $ - insertDelegation trce cache network epochNo slotNo txId idx mRedeemerId cred poolkh + insertDelegation trce (Just slotDetails) cache network epochNo slotNo txId idx mRedeemerId cred poolkh DelegVote drep -> when (ioGov iopts) $ insertDelegationVote trce cache network txId idx cred drep DelegStakeVote poolkh drep -> do when (ioShelley iopts) $ - insertDelegation trce cache network epochNo slotNo txId idx mRedeemerId cred poolkh + insertDelegation trce (Just slotDetails) cache network epochNo slotNo txId idx mRedeemerId cred poolkh when (ioGov iopts) $ insertDelegationVote trce cache network txId idx cred drep @@ -402,6 +405,7 @@ mkAdaPots blockId slotNo epochNo pots = insertDelegation :: (MonadBaseControl IO m, MonadIO m) => Trace IO Text -> + Maybe SlotDetails -> CacheStatus -> Ledger.Network -> EpochNo -> @@ -412,9 +416,9 @@ insertDelegation :: StakeCred -> Ledger.KeyHash 'Ledger.StakePool StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertDelegation trce cache network (EpochNo epoch) slotNo txId idx mRedeemerId cred poolkh = do +insertDelegation trce mSlotDetails cache network (EpochNo epoch) slotNo txId idx mRedeemerId cred poolkh = do addrId <- lift $ queryOrInsertStakeAddress trce cache UpdateCacheStrong network cred - poolHashId <- lift $ queryPoolKeyOrInsert "insertDelegation" trce cache UpdateCache True poolkh + poolHashId <- lift $ queryPoolKeyOrInsert "insertDelegation" trce mSlotDetails cache UpdateCache True poolkh void . lift . DB.insertDelegation $ DB.Delegation { DB.delegationAddrId = addrId diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/GovAction.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/GovAction.hs index 6de4a5362..1f89bc129 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/GovAction.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/GovAction.hs @@ -40,6 +40,7 @@ import Cardano.DbSync.Era.Universal.Insert.Other (toDouble) import Cardano.DbSync.Era.Util (liftLookupFail) import Cardano.DbSync.Error import Cardano.DbSync.Ledger.State +import Cardano.DbSync.Types (SlotDetails) import Cardano.DbSync.Util import Cardano.DbSync.Util.Bech32 (serialiseDrepToBech32) import Cardano.Ledger.BaseTypes @@ -70,6 +71,7 @@ insertGovActionProposal :: forall m. (MonadIO m, MonadBaseControl IO m) => Trace IO Text -> + SlotDetails -> CacheStatus -> DB.BlockId -> DB.TxId -> @@ -77,7 +79,7 @@ insertGovActionProposal :: Maybe (ConwayGovState StandardConway) -> (Word64, (GovActionId StandardCrypto, ProposalProcedure StandardConway)) -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertGovActionProposal trce cache blkId txId govExpiresAt mcgs (index, (govId, pp)) = do +insertGovActionProposal trce slotDetails cache blkId txId govExpiresAt mcgs (index, (govId, pp)) = do addrId <- lift $ queryOrInsertRewardAccount trce cache UpdateCache $ pProcReturnAddr pp votingAnchorId <- lift $ insertVotingAnchor blkId DB.GovActionAnchor $ pProcAnchor pp @@ -88,7 +90,7 @@ insertGovActionProposal trce cache blkId txId govExpiresAt mcgs (index, (govId, _ -> pure Nothing prevGovActionDBId <- case mprevGovAction of Nothing -> pure Nothing - Just prevGovActionId -> Just <$> resolveGovActionProposal cache prevGovActionId + Just prevGovActionId -> Just <$> resolveGovActionProposal slotDetails cache prevGovActionId govActionProposalId <- lift $ DB.insertGovActionProposal $ @@ -171,12 +173,13 @@ insertCommittee mgapId committee = do -------------------------------------------------------------------------------------- resolveGovActionProposal :: MonadIO m => + SlotDetails -> CacheStatus -> GovActionId StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) DB.GovActionProposalId -resolveGovActionProposal cache gaId = do +resolveGovActionProposal slotDetails cache gaId = do let txId = gaidTxId gaId - gaTxId <- liftLookupFail "resolveGovActionProposal.queryTxId" $ queryTxIdWithCache cache txId + gaTxId <- liftLookupFail "resolveGovActionProposal.queryTxId" $ queryTxIdWithCache slotDetails cache txId let (GovActionIx index) = gaidGovActionIx gaId liftLookupFail "resolveGovActionProposal.queryGovActionProposalId" $ DB.queryGovActionProposalId gaTxId (fromIntegral index) -- TODO: Use Word32? @@ -265,25 +268,27 @@ insertConstitution blockId mgapId constitution = do insertVotingProcedures :: (MonadIO m, MonadBaseControl IO m) => Trace IO Text -> + SlotDetails -> CacheStatus -> DB.BlockId -> DB.TxId -> (Voter StandardCrypto, [(GovActionId StandardCrypto, VotingProcedure StandardConway)]) -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertVotingProcedures trce cache blkId txId (voter, actions) = - mapM_ (insertVotingProcedure trce cache blkId txId voter) (zip [0 ..] actions) +insertVotingProcedures trce slotDetails cache blkId txId (voter, actions) = + mapM_ (insertVotingProcedure trce slotDetails cache blkId txId voter) (zip [0 ..] actions) insertVotingProcedure :: (MonadIO m, MonadBaseControl IO m) => Trace IO Text -> + SlotDetails -> CacheStatus -> DB.BlockId -> DB.TxId -> Voter StandardCrypto -> (Word16, (GovActionId StandardCrypto, VotingProcedure StandardConway)) -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertVotingProcedure trce cache blkId txId voter (index, (gaId, vp)) = do - govActionId <- resolveGovActionProposal cache gaId +insertVotingProcedure trce slotDetails cache blkId txId voter (index, (gaId, vp)) = do + govActionId <- resolveGovActionProposal slotDetails cache gaId votingAnchorId <- whenMaybe (strictMaybeToMaybe $ vProcAnchor vp) $ lift . insertVotingAnchor blkId DB.VoteAnchor (mCommitteeVoterId, mDRepVoter, mStakePoolVoter) <- case voter of CommitteeVoter cred -> do @@ -293,7 +298,7 @@ insertVotingProcedure trce cache blkId txId voter (index, (gaId, vp)) = do drep <- lift $ insertCredDrepHash cred pure (Nothing, Just drep, Nothing) StakePoolVoter poolkh -> do - poolHashId <- lift $ queryPoolKeyOrInsert "insertVotingProcedure" trce cache UpdateCache False poolkh + poolHashId <- lift $ queryPoolKeyOrInsert "insertVotingProcedure" trce (Just slotDetails) cache UpdateCache False poolkh pure (Nothing, Nothing, Just poolHashId) void . lift @@ -386,49 +391,53 @@ insertCostModel _blkId cms = updateRatified :: forall m. MonadIO m => + SlotDetails -> CacheStatus -> EpochNo -> [GovActionState StandardConway] -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -updateRatified cache epochNo ratifiedActions = do +updateRatified slotDetails cache epochNo ratifiedActions = do forM_ ratifiedActions $ \action -> do - gaId <- resolveGovActionProposal cache $ gasId action + gaId <- resolveGovActionProposal slotDetails cache $ gasId action lift $ DB.updateGovActionRatified gaId (unEpochNo epochNo) updateExpired :: forall m. MonadIO m => + SlotDetails -> CacheStatus -> EpochNo -> [GovActionId StandardCrypto] -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -updateExpired cache epochNo ratifiedActions = do +updateExpired slotDetails cache epochNo ratifiedActions = do forM_ ratifiedActions $ \action -> do - gaId <- resolveGovActionProposal cache action + gaId <- resolveGovActionProposal slotDetails cache action lift $ DB.updateGovActionExpired gaId (unEpochNo epochNo) updateDropped :: forall m. MonadIO m => + SlotDetails -> CacheStatus -> EpochNo -> [GovActionId StandardCrypto] -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -updateDropped cache epochNo ratifiedActions = do +updateDropped slotDetails cache epochNo ratifiedActions = do forM_ ratifiedActions $ \action -> do - gaId <- resolveGovActionProposal cache action + gaId <- resolveGovActionProposal slotDetails cache action lift $ DB.updateGovActionDropped gaId (unEpochNo epochNo) insertUpdateEnacted :: forall m. (MonadBaseControl IO m, MonadIO m) => Trace IO Text -> + SlotDetails -> CacheStatus -> DB.BlockId -> EpochNo -> ConwayGovState StandardConway -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertUpdateEnacted trce cache blkId epochNo enactedState = do +insertUpdateEnacted trce slotDetails cache blkId epochNo enactedState = do (mcommitteeId, mnoConfidenceGaId) <- handleCommittee constitutionId <- handleConstitution void $ @@ -447,7 +456,7 @@ insertUpdateEnacted trce cache blkId epochNo enactedState = do mCommitteeGaId <- case strictMaybeToMaybe (grCommittee govIds) of Nothing -> pure Nothing Just prevId -> - fmap Just <$> resolveGovActionProposal cache $ unGovPurposeId prevId + fmap Just <$> resolveGovActionProposal slotDetails cache $ unGovPurposeId prevId case (mCommitteeGaId, strictMaybeToMaybe (cgsCommittee enactedState)) of (Nothing, Nothing) -> pure (Nothing, Nothing) @@ -485,7 +494,7 @@ insertUpdateEnacted trce cache blkId epochNo enactedState = do mConstitutionGaId <- case strictMaybeToMaybe (grConstitution govIds) of Nothing -> pure Nothing Just prevId -> - fmap Just <$> resolveGovActionProposal cache $ unGovPurposeId prevId + fmap Just <$> resolveGovActionProposal slotDetails cache $ unGovPurposeId prevId constitutionIds <- lift $ DB.queryProposalConstitution mConstitutionGaId case constitutionIds of diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Grouped.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Grouped.hs index dc6b61234..d5be4c4f6 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Grouped.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Grouped.hs @@ -26,6 +26,7 @@ import qualified Cardano.DbSync.Era.Shelley.Generic as Generic import Cardano.DbSync.Era.Shelley.Query import Cardano.DbSync.Era.Util import Cardano.DbSync.Error +import Cardano.DbSync.Types (SlotDetails) import Cardano.Prelude import Control.Monad.Trans.Control (MonadBaseControl) import qualified Data.List as List @@ -185,17 +186,18 @@ insertReverseIndex blockId minIdsWrapper = resolveTxInputs :: MonadIO m => SyncEnv -> + SlotDetails -> Bool -> Bool -> [ExtendedTxOut] -> Generic.TxIn -> ExceptT SyncNodeError (ReaderT SqlBackend m) (Generic.TxIn, DB.TxId, Either Generic.TxIn DB.TxOutIdW, Maybe DbLovelace) -resolveTxInputs syncEnv hasConsumed needsValue groupedOutputs txIn = +resolveTxInputs syncEnv slotDetails hasConsumed needsValue groupedOutputs txIn = liftLookupFail ("resolveTxInputs " <> textShow txIn <> " ") $ do qres <- case (hasConsumed, needsValue) of (_, True) -> fmap convertFoundAll <$> resolveInputTxOutIdValue syncEnv txIn - (False, _) -> fmap convertnotFoundCache <$> queryTxIdWithCache (envCache syncEnv) (Generic.txInTxId txIn) + (False, _) -> fmap convertnotFoundCache <$> queryTxIdWithCache slotDetails (envCache syncEnv) (Generic.txInTxId txIn) (True, False) -> fmap convertFoundTxOutId <$> resolveInputTxOutId syncEnv txIn case qres of Right ret -> pure $ Right ret diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/LedgerEvent.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/LedgerEvent.hs index c4938e8f6..ec3ea1a16 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/LedgerEvent.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/LedgerEvent.hs @@ -41,10 +41,11 @@ import Database.Persist.SqlBackend.Internal.StatementCache insertNewEpochLedgerEvents :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> + SlotDetails -> EpochNo -> [LedgerEvent] -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertNewEpochLedgerEvents syncEnv currentEpochNo@(EpochNo curEpoch) = +insertNewEpochLedgerEvents syncEnv slotDetails currentEpochNo@(EpochNo curEpoch) = mapM_ handler where tracer = getTrace syncEnv @@ -82,14 +83,14 @@ insertNewEpochLedgerEvents syncEnv currentEpochNo@(EpochNo curEpoch) = liftIO . logInfo tracer $ "Starting at epoch " <> textShow (unEpochNo en) LedgerDeltaRewards _e rwd -> do let rewards = Map.toList $ Generic.unRewards rwd - insertRewards syncEnv ntw (subFromCurrentEpoch 2) currentEpochNo cache (Map.toList $ Generic.unRewards rwd) + insertRewards syncEnv slotDetails ntw (subFromCurrentEpoch 2) currentEpochNo cache (Map.toList $ Generic.unRewards rwd) -- This event is only created when it's not empty, so we don't need to check for null here. liftIO . logInfo tracer $ "Inserted " <> show (length rewards) <> " Delta rewards" LedgerIncrementalRewards _ rwd -> do let rewards = Map.toList $ Generic.unRewards rwd - insertRewards syncEnv ntw (subFromCurrentEpoch 1) (EpochNo $ curEpoch + 1) cache rewards + insertRewards syncEnv slotDetails ntw (subFromCurrentEpoch 1) (EpochNo $ curEpoch + 1) cache rewards LedgerRestrainedRewards e rwd creds -> - lift $ adjustEpochRewards tracer ntw cache e rwd creds + lift $ adjustEpochRewards tracer ntw slotDetails cache e rwd creds LedgerTotalRewards _e rwd -> lift $ validateEpochRewards tracer ntw (subFromCurrentEpoch 2) currentEpochNo rwd LedgerAdaPots _ -> @@ -99,11 +100,11 @@ insertNewEpochLedgerEvents syncEnv currentEpochNo@(EpochNo curEpoch) = liftIO $ logInfo tracer $ "Found " <> textShow (Set.size uncl) <> " unclaimed proposal refunds" - updateDropped cache (EpochNo curEpoch) (garGovActionId <$> (dropped <> expired)) + updateDropped slotDetails cache (EpochNo curEpoch) (garGovActionId <$> (dropped <> expired)) let refunded = filter (\e -> Set.notMember (garGovActionId e) uncl) (enacted <> dropped <> expired) insertProposalRefunds tracer ntw (subFromCurrentEpoch 1) currentEpochNo cache refunded -- TODO: check if they are disjoint to avoid double entries. forM_ enacted $ \gar -> do - gaId <- resolveGovActionProposal cache (garGovActionId gar) + gaId <- resolveGovActionProposal slotDetails cache (garGovActionId gar) lift $ void $ DB.updateGovActionEnacted gaId (unEpochNo currentEpochNo) whenJust (garMTreasury gar) $ \treasuryMap -> do let rewards = Map.mapKeys Ledger.raCredential $ Map.map (Set.singleton . mkTreasuryReward) treasuryMap @@ -115,5 +116,5 @@ insertNewEpochLedgerEvents syncEnv currentEpochNo@(EpochNo curEpoch) = liftIO . logInfo tracer $ "Inserted " <> show (length rewards) <> " Mir rewards" LedgerPoolReap en drs -> unless (Map.null $ Generic.unRewards drs) $ do - insertPoolDepositRefunds syncEnv en drs + insertPoolDepositRefunds syncEnv slotDetails en drs LedgerDeposits {} -> pure () diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Pool.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Pool.hs index 2631c8a6c..90d40669a 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Pool.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Pool.hs @@ -29,7 +29,7 @@ import Cardano.DbSync.Cache ( import Cardano.DbSync.Cache.Types (CacheAction (..), CacheStatus (..)) import qualified Cardano.DbSync.Era.Shelley.Generic as Generic import Cardano.DbSync.Error -import Cardano.DbSync.Types (PoolKeyHash) +import Cardano.DbSync.Types (PoolKeyHash, SlotDetails) import Cardano.DbSync.Util import qualified Cardano.Ledger.Address as Ledger import Cardano.Ledger.BaseTypes @@ -110,13 +110,14 @@ insertPoolRetire :: (MonadBaseControl IO m, MonadIO m) => Trace IO Text -> DB.TxId -> + SlotDetails -> CacheStatus -> EpochNo -> Word16 -> Ledger.KeyHash 'Ledger.StakePool StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertPoolRetire trce txId cache epochNum idx keyHash = do - poolId <- lift $ queryPoolKeyOrInsert "insertPoolRetire" trce cache UpdateCache True keyHash +insertPoolRetire trce txId slotDetails cache epochNum idx keyHash = do + poolId <- lift $ queryPoolKeyOrInsert "insertPoolRetire" trce (Just slotDetails) cache UpdateCache True keyHash void . lift . DB.insertPoolRetire $ DB.PoolRetire { DB.poolRetireHashId = poolId @@ -198,6 +199,7 @@ insertPoolRelay updateId relay = insertPoolCert :: (MonadBaseControl IO m, MonadIO m) => Trace IO Text -> + SlotDetails -> CacheStatus -> IsPoolMember -> Maybe Generic.Deposits -> @@ -208,7 +210,7 @@ insertPoolCert :: Word16 -> PoolCert StandardCrypto -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertPoolCert tracer cache isMember mdeposits network epoch blkId txId idx pCert = +insertPoolCert tracer slotDetails cache isMember mdeposits network epoch blkId txId idx pCert = case pCert of RegPool pParams -> insertPoolRegister tracer cache isMember mdeposits network epoch blkId txId idx pParams - RetirePool keyHash epochNum -> insertPoolRetire tracer txId cache epochNum idx keyHash + RetirePool keyHash epochNum -> insertPoolRetire tracer txId slotDetails cache epochNum idx keyHash diff --git a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Tx.hs b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Tx.hs index 636003bc1..8581c5032 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Tx.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Era/Universal/Insert/Tx.hs @@ -45,6 +45,7 @@ import Cardano.DbSync.Era.Universal.Insert.Pool (IsPoolMember) import Cardano.DbSync.Era.Util (liftLookupFail, safeDecodeToJson) import Cardano.DbSync.Error import Cardano.DbSync.Ledger.Types (ApplyResult (..), getGovExpiresAt, lookupDepositsMap) +import Cardano.DbSync.Types (SlotDetails) import Cardano.DbSync.Util import Cardano.DbSync.Util.Cbor (serialiseTxMetadataToCbor) import qualified Cardano.Ledger.Address as Ledger @@ -67,6 +68,7 @@ import Ouroboros.Consensus.Cardano.Block (StandardCrypto) insertTx :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> + SlotDetails -> IsPoolMember -> DB.BlockId -> EpochNo -> @@ -76,7 +78,7 @@ insertTx :: Generic.Tx -> BlockGroupedData -> ExceptT SyncNodeError (ReaderT SqlBackend m) BlockGroupedData -insertTx syncEnv isMember blkId epochNo slotNo applyResult blockIndex tx grouped = do +insertTx syncEnv slotDetails isMember blkId epochNo slotNo applyResult blockIndex tx grouped = do let !txHash = Generic.txHash tx let !mdeposits = if not (Generic.txValidContract tx) then Just (Coin 0) else lookupDepositsMap txHash (apDepositsMap applyResult) let !outSum = fromIntegral $ unCoin $ Generic.txOutSum tx @@ -90,10 +92,10 @@ insertTx syncEnv isMember blkId epochNo slotNo applyResult blockIndex tx grouped (resolvedInputs, fees', deposits) <- case (disInOut, mdeposits, unCoin <$> Generic.txFees tx) of (True, _, _) -> pure ([], 0, unCoin <$> mdeposits) (_, Just deposits, Just fees) -> do - (resolvedInputs, _) <- splitLast <$> mapM (resolveTxInputs syncEnv hasConsumed False (fst <$> groupedTxOut grouped)) txIn + (resolvedInputs, _) <- splitLast <$> mapM (resolveTxInputs syncEnv slotDetails hasConsumed False (fst <$> groupedTxOut grouped)) txIn pure (resolvedInputs, fees, Just (unCoin deposits)) (_, Nothing, Just fees) -> do - (resolvedInputs, amounts) <- splitLast <$> mapM (resolveTxInputs syncEnv hasConsumed False (fst <$> groupedTxOut grouped)) txIn + (resolvedInputs, amounts) <- splitLast <$> mapM (resolveTxInputs syncEnv slotDetails hasConsumed False (fst <$> groupedTxOut grouped)) txIn if any isNothing amounts then pure (resolvedInputs, fees, Nothing) else @@ -101,7 +103,7 @@ insertTx syncEnv isMember blkId epochNo slotNo applyResult blockIndex tx grouped in pure (resolvedInputs, fees, Just $ fromIntegral (inSum + withdrawalSum) - fromIntegral outSum - fees - treasuryDonation) (_, _, Nothing) -> do -- Nothing in fees means a phase 2 failure - (resolvedInsFull, amounts) <- splitLast <$> mapM (resolveTxInputs syncEnv hasConsumed True (fst <$> groupedTxOut grouped)) txIn + (resolvedInsFull, amounts) <- splitLast <$> mapM (resolveTxInputs syncEnv slotDetails hasConsumed True (fst <$> groupedTxOut grouped)) txIn let !inSum = sum $ map unDbLovelace $ catMaybes amounts !diffSum = if inSum >= outSum then inSum - outSum else 0 !fees = maybe diffSum (fromIntegral . unCoin) (Generic.txFees tx) @@ -157,8 +159,8 @@ insertTx syncEnv isMember blkId epochNo slotNo applyResult blockIndex tx grouped when (ioPlutusExtra iopts) $ do mapM_ (insertDatum tracer cache txId) (Generic.txData tx) - mapM_ (insertCollateralTxIn syncEnv tracer txId) (Generic.txCollateralInputs tx) - mapM_ (insertReferenceTxIn syncEnv tracer txId) (Generic.txReferenceInputs tx) + mapM_ (insertCollateralTxIn syncEnv tracer slotDetails txId) (Generic.txCollateralInputs tx) + mapM_ (insertReferenceTxIn syncEnv tracer slotDetails txId) (Generic.txReferenceInputs tx) mapM_ (insertCollateralTxOut tracer cache iopts (txId, txHash)) (Generic.txCollateralOutputs tx) txMetadata <- @@ -169,7 +171,7 @@ insertTx syncEnv isMember blkId epochNo slotNo applyResult blockIndex tx grouped iopts (Generic.txMetadata tx) mapM_ - (insertCertificate syncEnv isMember mDeposits blkId txId epochNo slotNo redeemers) + (insertCertificate syncEnv slotDetails isMember mDeposits blkId txId epochNo slotNo redeemers) $ Generic.txCertificates tx when (ioShelley iopts) $ mapM_ (insertWithdrawals tracer cache txId redeemers) $ @@ -192,8 +194,8 @@ insertTx syncEnv isMember blkId epochNo slotNo applyResult blockIndex tx grouped Generic.txExtraKeyWitnesses tx when (ioGov iopts) $ do - mapM_ (insertGovActionProposal tracer cache blkId txId (getGovExpiresAt applyResult epochNo) (apGovActionState applyResult)) $ zip [0 ..] (Generic.txProposalProcedure tx) - mapM_ (insertVotingProcedures tracer cache blkId txId) (Generic.txVotingProcedure tx) + mapM_ (insertGovActionProposal tracer slotDetails cache blkId txId (getGovExpiresAt applyResult epochNo) (apGovActionState applyResult)) $ zip [0 ..] (Generic.txProposalProcedure tx) + mapM_ (insertVotingProcedures tracer slotDetails cache blkId txId) (Generic.txVotingProcedure tx) let !txIns = map (prepareTxIn txId redeemers) resolvedInputs pure (grouped <> BlockGroupedData txIns txOutsGrouped txMetadata maTxMint fees outSum) @@ -477,12 +479,13 @@ insertCollateralTxIn :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> Trace IO Text -> + SlotDetails -> DB.TxId -> Generic.TxIn -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertCollateralTxIn syncEnv _tracer txInId txIn = do +insertCollateralTxIn syncEnv _tracer slotDetails txInId txIn = do let txId = txInTxId txIn - txOutId <- liftLookupFail "insertCollateralTxIn" $ queryTxIdWithCache (envCache syncEnv) txId + txOutId <- liftLookupFail "insertCollateralTxIn" $ queryTxIdWithCache slotDetails (envCache syncEnv) txId void . lift . DB.insertCollateralTxIn @@ -496,12 +499,13 @@ insertReferenceTxIn :: (MonadBaseControl IO m, MonadIO m) => SyncEnv -> Trace IO Text -> + SlotDetails -> DB.TxId -> Generic.TxIn -> ExceptT SyncNodeError (ReaderT SqlBackend m) () -insertReferenceTxIn syncEnv _tracer txInId txIn = do +insertReferenceTxIn syncEnv _tracer slotDetails txInId txIn = do let txId = txInTxId txIn - txOutId <- liftLookupFail "insertReferenceTxIn" $ queryTxIdWithCache (envCache syncEnv) txId + txOutId <- liftLookupFail "insertReferenceTxIn" $ queryTxIdWithCache slotDetails (envCache syncEnv) txId void . lift . DB.insertReferenceTxIn diff --git a/cardano-db-sync/src/Cardano/DbSync/Fix/EpochStake.hs b/cardano-db-sync/src/Cardano/DbSync/Fix/EpochStake.hs index c1ff28caf..fa5659f77 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Fix/EpochStake.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Fix/EpochStake.hs @@ -12,14 +12,20 @@ import Cardano.DbSync.Era.Universal.Epoch import Cardano.DbSync.Error import Cardano.DbSync.Ledger.State import Cardano.DbSync.Ledger.Types +import Cardano.DbSync.Types (SlotDetails) import Cardano.Prelude import Control.Monad.Trans.Control import qualified Data.Map.Strict as Map import qualified Data.Strict.Maybe as Strict import Database.Persist.Sql (SqlBackend) -migrateStakeDistr :: (MonadIO m, MonadBaseControl IO m) => SyncEnv -> Strict.Maybe CardanoLedgerState -> ExceptT SyncNodeError (ReaderT SqlBackend m) Bool -migrateStakeDistr env mcls = +migrateStakeDistr :: + (MonadIO m, MonadBaseControl IO m) => + SyncEnv -> + Maybe SlotDetails -> + Strict.Maybe CardanoLedgerState -> + ExceptT SyncNodeError (ReaderT SqlBackend m) Bool +migrateStakeDistr env slotDetails mcls = case (envLedgerEnv env, mcls) of (HasLedger lenv, Strict.Just cls) -> do ems <- lift DB.queryAllExtraMigrations @@ -31,7 +37,7 @@ migrateStakeDistr env mcls = liftIO $ logInsert 0 Slice (StakeSlice _epochNo distr) isFinal -> do liftIO $ logInsert (Map.size distr) - insertStakeSlice env stakeSlice + insertStakeSlice env slotDetails stakeSlice (mminEpoch, mmaxEpoch) <- lift DB.queryMinMaxEpochStake liftIO $ logMinMax mminEpoch mmaxEpoch case (mminEpoch, mmaxEpoch) of diff --git a/cardano-db-sync/src/Cardano/DbSync/Util.hs b/cardano-db-sync/src/Cardano/DbSync/Util.hs index 961ad5546..fd6ef4972 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Util.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Util.hs @@ -13,7 +13,10 @@ module Cardano.DbSync.Util ( cardanoBlockSlotNo, fmap3, getSyncStatus, + getSyncStatusInHalfHour, isSyncedWithinSeconds, + isSyncedWithinTwoMin, + isSyncedWithinHalfHour, liftedLogException, logActionDuration, logException, @@ -75,9 +78,6 @@ cardanoBlockSlotNo = blockSlot fmap3 :: (Functor f, Functor g, Functor h) => (a -> b) -> f (g (h a)) -> f (g (h b)) fmap3 = fmap . fmap . fmap -getSyncStatus :: SlotDetails -> SyncState -getSyncStatus sd = isSyncedWithinSeconds sd 120 - isSyncedWithinSeconds :: SlotDetails -> Word -> SyncState isSyncedWithinSeconds sd target = -- diffUTCTime returns seconds. @@ -86,6 +86,18 @@ isSyncedWithinSeconds sd target = then SyncFollowing else SyncLagging +getSyncStatus :: SlotDetails -> SyncState +getSyncStatus sd = isSyncedWithinSeconds sd 120 + +getSyncStatusInHalfHour :: SlotDetails -> SyncState +getSyncStatusInHalfHour sd = isSyncedWithinSeconds sd 1800 + +isSyncedWithinTwoMin :: SlotDetails -> Bool +isSyncedWithinTwoMin sd = isSyncedWithinSeconds sd 120 == SyncFollowing + +isSyncedWithinHalfHour :: SlotDetails -> Bool +isSyncedWithinHalfHour sd = isSyncedWithinSeconds sd 1800 == SyncFollowing + textPrettyShow :: Show a => a -> Text textPrettyShow = Text.pack . ppShow