diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs index 268c85de5..eb692cf83 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs @@ -302,21 +302,21 @@ mkConfigFile :: FilePath -> FilePath -> ConfigFile mkConfigFile staticDir cliConfigFilename = ConfigFile $ staticDir cliConfigFilename -configPruneForceTxIn :: SyncNodeConfig -> SyncNodeConfig -configPruneForceTxIn cfg = do - cfg {dncInsertOptions = (dncInsertOptions cfg) {sioTxOut = TxOutConsumedPrune (ForceTxIn True) (UseTxOutAddress False)}} +configPruneForceTxIn :: Bool -> SyncNodeConfig -> SyncNodeConfig +configPruneForceTxIn useTxOutAddress cfg = do + cfg {dncInsertOptions = (dncInsertOptions cfg) {sioTxOut = TxOutConsumedPrune (ForceTxIn True) (UseTxOutAddress useTxOutAddress)}} -configPrune :: SyncNodeConfig -> SyncNodeConfig -configPrune cfg = do - cfg {dncInsertOptions = (dncInsertOptions cfg) {sioTxOut = TxOutConsumedPrune (ForceTxIn False) (UseTxOutAddress False)}} +configPrune :: Bool -> SyncNodeConfig -> SyncNodeConfig +configPrune useTxOutAddress cfg = do + cfg {dncInsertOptions = (dncInsertOptions cfg) {sioTxOut = TxOutConsumedPrune (ForceTxIn False) (UseTxOutAddress useTxOutAddress)}} -configConsume :: SyncNodeConfig -> SyncNodeConfig -configConsume cfg = do - cfg {dncInsertOptions = (dncInsertOptions cfg) {sioTxOut = TxOutConsumed (ForceTxIn False) (UseTxOutAddress False)}} +configConsume :: Bool -> SyncNodeConfig -> SyncNodeConfig +configConsume useTxOutAddress cfg = do + cfg {dncInsertOptions = (dncInsertOptions cfg) {sioTxOut = TxOutConsumed (ForceTxIn False) (UseTxOutAddress useTxOutAddress)}} -configBootstrap :: SyncNodeConfig -> SyncNodeConfig -configBootstrap cfg = do - cfg {dncInsertOptions = (dncInsertOptions cfg) {sioTxOut = TxOutConsumedBootstrap (ForceTxIn False) (UseTxOutAddress False)}} +configBootstrap :: Bool -> SyncNodeConfig -> SyncNodeConfig +configBootstrap useTxOutAddress cfg = do + cfg {dncInsertOptions = (dncInsertOptions cfg) {sioTxOut = TxOutConsumedBootstrap (ForceTxIn False) (UseTxOutAddress useTxOutAddress)}} configPlutusDisable :: SyncNodeConfig -> SyncNodeConfig configPlutusDisable cfg = do diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage.hs index 4d36830e5..07281e32c 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage.hs @@ -34,8 +34,9 @@ unitTests iom knownMigrations = [ testCase "default insert config" Config.defaultInsertConfig , testCase "insert config" Config.insertConfig , testGroup - "consumed-tx-out and prune-tx-out" + "tx-out" [ test "basic prune" MigrateConsumedPruneTxOut.basicPrune + , test "basic prune with address table" MigrateConsumedPruneTxOut.basicPruneWithAddress , test "prune with simple rollback" MigrateConsumedPruneTxOut.pruneWithSimpleRollback , test "prune with full tx rollback" MigrateConsumedPruneTxOut.pruneWithFullTxRollback , test "pruning should keep some tx" MigrateConsumedPruneTxOut.pruningShouldKeepSomeTx @@ -47,6 +48,20 @@ unitTests iom knownMigrations = , expectFailSilent "set prune flag, restart missing prune flag" $ MigrateConsumedPruneTxOut.pruneRestartMissingFlag iom knownMigrations , expectFailSilent "set bootstrap flag, restart missing bootstrap flag" $ MigrateConsumedPruneTxOut.bootstrapRestartMissingFlag iom knownMigrations ] + , testGroup + "tx-out using Address table" + [ test "basic prune with address table" MigrateConsumedPruneTxOut.basicPruneWithAddress + , test "prune with simple rollback with address table" MigrateConsumedPruneTxOut.pruneWithSimpleRollbackWithAddress + , test "prune with full tx rollback with address table" MigrateConsumedPruneTxOut.pruneWithFullTxRollbackWithAddress + , test "pruning should keep some tx with address table" MigrateConsumedPruneTxOut.pruningShouldKeepSomeTxWithAddress + , test "prune and rollback one block with address table" MigrateConsumedPruneTxOut.pruneAndRollBackOneBlockWithAddress + , test "no pruning and rollback with address table" MigrateConsumedPruneTxOut.noPruneAndRollBackWithAddress + , test "prune same block with address table" MigrateConsumedPruneTxOut.pruneSameBlockWithAddress + , test "no pruning same block with address table" MigrateConsumedPruneTxOut.noPruneSameBlockWithAddress + , expectFailSilent "restart with new consumed set to false, with address table" $ MigrateConsumedPruneTxOut.migrateAndPruneRestartWithAddress iom knownMigrations + , expectFailSilent "set prune flag, restart missing prune flag, with address table" $ MigrateConsumedPruneTxOut.pruneRestartMissingFlagWithAddress iom knownMigrations + , expectFailSilent "set bootstrap flag, restart missing bootstrap flag, with address table" $ MigrateConsumedPruneTxOut.bootstrapRestartMissingFlagWithAddress iom knownMigrations + ] ] , testGroup "simple" diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage/Config/MigrateConsumedPruneTxOut.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage/Config/MigrateConsumedPruneTxOut.hs index 954a424c1..6f385c06f 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage/Config/MigrateConsumedPruneTxOut.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage/Config/MigrateConsumedPruneTxOut.hs @@ -7,16 +7,27 @@ module Test.Cardano.Db.Mock.Unit.Babbage.Config.MigrateConsumedPruneTxOut ( basicPrune, + basicPruneWithAddress, pruneWithSimpleRollback, + pruneWithSimpleRollbackWithAddress, pruneWithFullTxRollback, + pruneWithFullTxRollbackWithAddress, pruningShouldKeepSomeTx, + pruningShouldKeepSomeTxWithAddress, pruneAndRollBackOneBlock, + pruneAndRollBackOneBlockWithAddress, noPruneAndRollBack, + noPruneAndRollBackWithAddress, pruneSameBlock, + pruneSameBlockWithAddress, noPruneSameBlock, + noPruneSameBlockWithAddress, migrateAndPruneRestart, + migrateAndPruneRestartWithAddress, pruneRestartMissingFlag, + pruneRestartMissingFlagWithAddress, bootstrapRestartMissingFlag, + bootstrapRestartMissingFlagWithAddress, ) where import Cardano.Db (TxOutTableType (..)) @@ -41,7 +52,7 @@ import Test.Cardano.Db.Mock.Config ( startDBSync, stopDBSync, txOutTableTypeFromConfig, - withCustomConfig, + withCustomConfigAndDropDB, ) import Test.Cardano.Db.Mock.Examples (mockBlock0, mockBlock1) import Test.Cardano.Db.Mock.UnifiedApi ( @@ -59,8 +70,14 @@ import Test.Tasty.HUnit (Assertion) -- Tests ------------------------------------------------------------------------------ basicPrune :: IOManager -> [(Text, Text)] -> Assertion -basicPrune = do - withCustomConfig cmdLineArgs (Just configPruneForceTxIn) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do +basicPrune = peformBasicPrune False + +basicPruneWithAddress :: IOManager -> [(Text, Text)] -> Assertion +basicPruneWithAddress = peformBasicPrune True + +peformBasicPrune :: Bool -> IOManager -> [(Text, Text)] -> Assertion +peformBasicPrune useTxOutAddress = do + withCustomConfigAndDropDB cmdLineArgs (Just $ configPruneForceTxIn useTxOutAddress) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do let txOutTableType = txOutTableTypeFromConfig dbSyncEnv startDBSync dbSyncEnv -- add 50 block @@ -84,8 +101,14 @@ basicPrune = do testLabel = "configPrune" pruneWithSimpleRollback :: IOManager -> [(Text, Text)] -> Assertion -pruneWithSimpleRollback = do - withCustomConfig cmdLineArgs (Just configPruneForceTxIn) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do +pruneWithSimpleRollback = peformPruneWithSimpleRollback False + +pruneWithSimpleRollbackWithAddress :: IOManager -> [(Text, Text)] -> Assertion +pruneWithSimpleRollbackWithAddress = peformPruneWithSimpleRollback True + +peformPruneWithSimpleRollback :: Bool -> IOManager -> [(Text, Text)] -> Assertion +peformPruneWithSimpleRollback useTxOutAddress = do + withCustomConfigAndDropDB cmdLineArgs (Just $ configPruneForceTxIn useTxOutAddress) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do let txOutTableType = txOutTableTypeFromConfig dbSyncEnv blk0 <- forgeNext interpreter mockBlock0 blk1 <- forgeNext interpreter mockBlock1 @@ -110,8 +133,14 @@ pruneWithSimpleRollback = do testLabel = "configPruneSimpleRollback" pruneWithFullTxRollback :: IOManager -> [(Text, Text)] -> Assertion -pruneWithFullTxRollback = do - withCustomConfig cmdLineArgs (Just configPruneForceTxIn) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do +pruneWithFullTxRollback = performPruneWithFullTxRollback False + +pruneWithFullTxRollbackWithAddress :: IOManager -> [(Text, Text)] -> Assertion +pruneWithFullTxRollbackWithAddress = performPruneWithFullTxRollback True + +performPruneWithFullTxRollback :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performPruneWithFullTxRollback useTxOutAddress = do + withCustomConfigAndDropDB cmdLineArgs (Just $ configPruneForceTxIn useTxOutAddress) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do let txOutTableType = txOutTableTypeFromConfig dbSyncEnv startDBSync dbSyncEnv blk0 <- forgeNextFindLeaderAndSubmit interpreter mockServer [] @@ -140,8 +169,14 @@ pruneWithFullTxRollback = do -- The tx in the last, 2 x securityParam worth of blocks should not be pruned. -- In these tests, 2 x securityParam = 20 blocks. pruningShouldKeepSomeTx :: IOManager -> [(Text, Text)] -> Assertion -pruningShouldKeepSomeTx = do - withCustomConfig cmdLineArgs (Just configPrune) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do +pruningShouldKeepSomeTx = performPruningShouldKeepSomeTx False + +pruningShouldKeepSomeTxWithAddress :: IOManager -> [(Text, Text)] -> Assertion +pruningShouldKeepSomeTxWithAddress = performPruningShouldKeepSomeTx True + +performPruningShouldKeepSomeTx :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performPruningShouldKeepSomeTx useTxOutAddress = do + withCustomConfigAndDropDB cmdLineArgs (Just $ configPrune useTxOutAddress) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do startDBSync dbSyncEnv let txOutTableType = txOutTableTypeFromConfig dbSyncEnv b1 <- forgeAndSubmitBlocks interpreter mockServer 80 @@ -164,8 +199,14 @@ pruningShouldKeepSomeTx = do -- prune with rollback pruneAndRollBackOneBlock :: IOManager -> [(Text, Text)] -> Assertion -pruneAndRollBackOneBlock = do - withCustomConfig cmdLineArgs (Just configPruneForceTxIn) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do +pruneAndRollBackOneBlock = performPruneAndRollBackOneBlock False + +pruneAndRollBackOneBlockWithAddress :: IOManager -> [(Text, Text)] -> Assertion +pruneAndRollBackOneBlockWithAddress = performPruneAndRollBackOneBlock True + +performPruneAndRollBackOneBlock :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performPruneAndRollBackOneBlock useTxOutAddress = do + withCustomConfigAndDropDB cmdLineArgs (Just $ configPruneForceTxIn useTxOutAddress) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do startDBSync dbSyncEnv let txOutTableType = txOutTableTypeFromConfig dbSyncEnv void $ forgeAndSubmitBlocks interpreter mockServer 98 @@ -197,8 +238,14 @@ pruneAndRollBackOneBlock = do -- consume with rollback noPruneAndRollBack :: IOManager -> [(Text, Text)] -> Assertion -noPruneAndRollBack = do - withCustomConfig cmdLineArgs (Just configConsume) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do +noPruneAndRollBack = performNoPruneAndRollBack False + +noPruneAndRollBackWithAddress :: IOManager -> [(Text, Text)] -> Assertion +noPruneAndRollBackWithAddress = performNoPruneAndRollBack True + +performNoPruneAndRollBack :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performNoPruneAndRollBack useTxOutAddress = do + withCustomConfigAndDropDB cmdLineArgs (Just $ configConsume useTxOutAddress) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do startDBSync dbSyncEnv let txOutTableType = txOutTableTypeFromConfig dbSyncEnv void $ forgeAndSubmitBlocks interpreter mockServer 98 @@ -229,8 +276,14 @@ noPruneAndRollBack = do testLabel = "configPruneAndRollBack" pruneSameBlock :: IOManager -> [(Text, Text)] -> Assertion -pruneSameBlock = - withCustomConfig cmdLineArgs (Just configPruneForceTxIn) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do +pruneSameBlock = performPruneSameBlock False + +pruneSameBlockWithAddress :: IOManager -> [(Text, Text)] -> Assertion +pruneSameBlockWithAddress = performPruneSameBlock True + +performPruneSameBlock :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performPruneSameBlock useTxOutAddress = + withCustomConfigAndDropDB cmdLineArgs (Just $ configPruneForceTxIn useTxOutAddress) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do startDBSync dbSyncEnv let txOutTableType = txOutTableTypeFromConfig dbSyncEnv void $ forgeAndSubmitBlocks interpreter mockServer 76 @@ -255,8 +308,14 @@ pruneSameBlock = testLabel = "configPruneSameBlock" noPruneSameBlock :: IOManager -> [(Text, Text)] -> Assertion -noPruneSameBlock = - withCustomConfig cmdLineArgs (Just configConsume) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do +noPruneSameBlock = performNoPruneSameBlock False + +noPruneSameBlockWithAddress :: IOManager -> [(Text, Text)] -> Assertion +noPruneSameBlockWithAddress = performNoPruneSameBlock True + +performNoPruneSameBlock :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performNoPruneSameBlock useTxOutAddress = + withCustomConfigAndDropDB cmdLineArgs (Just $ configConsume useTxOutAddress) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do startDBSync dbSyncEnv let txOutTableType = txOutTableTypeFromConfig dbSyncEnv void $ forgeAndSubmitBlocks interpreter mockServer 96 @@ -278,8 +337,14 @@ noPruneSameBlock = testLabel = "configNoPruneSameBlock" migrateAndPruneRestart :: IOManager -> [(Text, Text)] -> Assertion -migrateAndPruneRestart = do - withCustomConfig cmdLineArgs (Just configConsume) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do +migrateAndPruneRestart = performMigrateAndPruneRestart False + +migrateAndPruneRestartWithAddress :: IOManager -> [(Text, Text)] -> Assertion +migrateAndPruneRestartWithAddress = performMigrateAndPruneRestart True + +performMigrateAndPruneRestart :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performMigrateAndPruneRestart useTxOutAddress = do + withCustomConfigAndDropDB cmdLineArgs (Just $ configConsume useTxOutAddress) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do startDBSync dbSyncEnv void $ forgeAndSubmitBlocks interpreter mockServer 50 assertBlockNoBackoff dbSyncEnv 50 @@ -297,8 +362,14 @@ migrateAndPruneRestart = do testLabel = "configMigrateAndPruneRestart" pruneRestartMissingFlag :: IOManager -> [(Text, Text)] -> Assertion -pruneRestartMissingFlag = do - withCustomConfig cmdLineArgs (Just configPruneForceTxIn) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do +pruneRestartMissingFlag = performPruneRestartMissingFlag False + +pruneRestartMissingFlagWithAddress :: IOManager -> [(Text, Text)] -> Assertion +pruneRestartMissingFlagWithAddress = performPruneRestartMissingFlag True + +performPruneRestartMissingFlag :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performPruneRestartMissingFlag useTxOutAddress = do + withCustomConfigAndDropDB cmdLineArgs (Just $ configPruneForceTxIn useTxOutAddress) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do startDBSync dbSyncEnv void $ forgeAndSubmitBlocks interpreter mockServer 50 assertBlockNoBackoff dbSyncEnv 50 @@ -316,8 +387,14 @@ pruneRestartMissingFlag = do testLabel = "configPruneRestartMissingFlag" bootstrapRestartMissingFlag :: IOManager -> [(Text, Text)] -> Assertion -bootstrapRestartMissingFlag = do - withCustomConfig cmdLineArgs (Just configBootstrap) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do +bootstrapRestartMissingFlag = performBootstrapRestartMissingFlag False + +bootstrapRestartMissingFlagWithAddress :: IOManager -> [(Text, Text)] -> Assertion +bootstrapRestartMissingFlagWithAddress = performBootstrapRestartMissingFlag True + +performBootstrapRestartMissingFlag :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performBootstrapRestartMissingFlag useTxOutAddress = do + withCustomConfigAndDropDB cmdLineArgs (Just $ configBootstrap useTxOutAddress) babbageConfigDir testLabel $ \interpreter mockServer dbSyncEnv -> do startDBSync dbSyncEnv void $ forgeAndSubmitBlocks interpreter mockServer 50 assertBlockNoBackoff dbSyncEnv 50 diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway.hs index 3832fef3d..8afc4e590 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway.hs @@ -62,6 +62,29 @@ unitTests iom knownMigrations = "set bootstrap flag, restart missing bootstrap flag" $ MigrateConsumedPruneTxOut.bootstrapRestartMissingFlag iom knownMigrations ] + , testGroup + "tx-out with use_address_table config" + [ test "basic prune, with use_address_table config" MigrateConsumedPruneTxOut.basicPruneWithAddress + , test "prune with simple rollback, with use_address_table config" MigrateConsumedPruneTxOut.pruneWithSimpleRollbackWithAddress + , test "prune with full tx rollback, with use_address_table config" MigrateConsumedPruneTxOut.pruneWithFullTxRollbackWithAddress + , test "pruning should keep some tx, with use_address_table config" MigrateConsumedPruneTxOut.pruningShouldKeepSomeTxWithAddress + , test "prune and rollback one block, with use_address_table config" MigrateConsumedPruneTxOut.pruneAndRollBackOneBlockWithAddress + , test "no pruning and rollback, with use_address_table config" MigrateConsumedPruneTxOut.noPruneAndRollBackWithAddress + , test "prune same block, with use_address_table config" MigrateConsumedPruneTxOut.pruneSameBlockWithAddress + , test "no pruning same block, with use_address_table config" MigrateConsumedPruneTxOut.noPruneSameBlockWithAddress + , expectFailSilent + "restart with new consumed set to false, with use_address_table config" + $ MigrateConsumedPruneTxOut.migrateAndPruneRestartWithAddress iom knownMigrations + , expectFailSilent + "set prune flag, restart missing prune flag, with use_address_table config" + $ MigrateConsumedPruneTxOut.pruneRestartMissingFlagWithAddress iom knownMigrations + , expectFailSilent + "set bootstrap flag, restart missing bootstrap flag, with use_address_table config" + $ MigrateConsumedPruneTxOut.bootstrapRestartMissingFlagWithAddress iom knownMigrations + , expectFailSilent + "populate db then reset with use_address_table config config active" + $ MigrateConsumedPruneTxOut.populateDbRestartWithAddressConfig iom knownMigrations + ] ] , testGroup "simple" diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/MigrateConsumedPruneTxOut.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/MigrateConsumedPruneTxOut.hs index debf685d1..2d8f723f9 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/MigrateConsumedPruneTxOut.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/MigrateConsumedPruneTxOut.hs @@ -7,19 +7,32 @@ module Test.Cardano.Db.Mock.Unit.Conway.Config.MigrateConsumedPruneTxOut ( basicPrune, + basicPruneWithAddress, pruneWithSimpleRollback, + pruneWithSimpleRollbackWithAddress, pruneWithFullTxRollback, + pruneWithFullTxRollbackWithAddress, pruningShouldKeepSomeTx, + pruningShouldKeepSomeTxWithAddress, pruneAndRollBackOneBlock, + pruneAndRollBackOneBlockWithAddress, noPruneAndRollBack, + noPruneAndRollBackWithAddress, pruneSameBlock, + pruneSameBlockWithAddress, noPruneSameBlock, + noPruneSameBlockWithAddress, migrateAndPruneRestart, + migrateAndPruneRestartWithAddress, pruneRestartMissingFlag, + pruneRestartMissingFlagWithAddress, bootstrapRestartMissingFlag, + bootstrapRestartMissingFlagWithAddress, + populateDbRestartWithAddressConfig, ) where import qualified Cardano.Db as DB +import Cardano.DbSync.Config.Types (ForceTxIn (..), SyncInsertOptions (..), SyncNodeConfig (..), TxOutConfig (..), UseTxOutAddress (..)) import Cardano.Mock.ChainSync.Server (IOManager (), addBlock) import Cardano.Mock.Forging.Interpreter (forgeNext) import qualified Cardano.Mock.Forging.Tx.Conway as Conway @@ -38,8 +51,14 @@ import qualified Prelude -- Tests ----------------------------------------------------------------------------- basicPrune :: IOManager -> [(Text, Text)] -> Assertion -basicPrune = do - withCustomConfig args (Just configPruneForceTxIn) cfgDir testLabel $ \interpreter mockServer dbSync -> do +basicPrune = performBasicPrune False + +basicPruneWithAddress :: IOManager -> [(Text, Text)] -> Assertion +basicPruneWithAddress = performBasicPrune True + +performBasicPrune :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performBasicPrune useTxOutAddress = do + withCustomConfigAndDropDB args (Just $ configPruneForceTxIn useTxOutAddress) cfgDir testLabel $ \interpreter mockServer dbSync -> do startDBSync dbSync let txOutTableType = txOutTableTypeFromConfig dbSync @@ -72,8 +91,14 @@ basicPrune = do cfgDir = conwayConfigDir pruneWithSimpleRollback :: IOManager -> [(Text, Text)] -> Assertion -pruneWithSimpleRollback = - withCustomConfig cmdLineArgs (Just configPruneForceTxIn) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do +pruneWithSimpleRollback = performPruneWithSimpleRollback False + +pruneWithSimpleRollbackWithAddress :: IOManager -> [(Text, Text)] -> Assertion +pruneWithSimpleRollbackWithAddress = performPruneWithSimpleRollback True + +performPruneWithSimpleRollback :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performPruneWithSimpleRollback useTxOutAddress = + withCustomConfigAndDropDB cmdLineArgs (Just $ configPruneForceTxIn useTxOutAddress) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do let txOutTableType = txOutTableTypeFromConfig dbSync -- Forge some blocks blk0 <- forgeNext interpreter mockBlock0 @@ -109,8 +134,14 @@ pruneWithSimpleRollback = fullBlockSize b = fromIntegral $ length b + 4 pruneWithFullTxRollback :: IOManager -> [(Text, Text)] -> Assertion -pruneWithFullTxRollback = - withCustomConfig cmdLineArgs (Just configPruneForceTxIn) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do +pruneWithFullTxRollback = performPruneWithFullTxRollback False + +pruneWithFullTxRollbackWithAddress :: IOManager -> [(Text, Text)] -> Assertion +pruneWithFullTxRollbackWithAddress = performPruneWithFullTxRollback True + +performPruneWithFullTxRollback :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performPruneWithFullTxRollback useTxOutAddress = + withCustomConfigAndDropDB cmdLineArgs (Just $ configPruneForceTxIn useTxOutAddress) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do startDBSync dbSync let txOutTableType = txOutTableTypeFromConfig dbSync -- Forge a block @@ -146,9 +177,16 @@ pruneWithFullTxRollback = testLabel = "conwayConfigPruneOnFullRollback" -- The transactions in the last `2 * securityParam` blocks should not be pruned + pruningShouldKeepSomeTx :: IOManager -> [(Text, Text)] -> Assertion -pruningShouldKeepSomeTx = do - withCustomConfig cmdLineArgs (Just configPrune) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do +pruningShouldKeepSomeTx = performPruningShouldKeepSomeTx False + +pruningShouldKeepSomeTxWithAddress :: IOManager -> [(Text, Text)] -> Assertion +pruningShouldKeepSomeTxWithAddress = performPruningShouldKeepSomeTx True + +performPruningShouldKeepSomeTx :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performPruningShouldKeepSomeTx useTxOutAddress = do + withCustomConfigAndDropDB cmdLineArgs (Just $ configPrune useTxOutAddress) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do startDBSync dbSync let txOutTableType = txOutTableTypeFromConfig dbSync -- Forge some blocks @@ -177,8 +215,14 @@ pruningShouldKeepSomeTx = do testLabel = "conwayConfigPruneCorrectAmount" pruneAndRollBackOneBlock :: IOManager -> [(Text, Text)] -> Assertion -pruneAndRollBackOneBlock = - withCustomConfig cmdLineArgs (Just configPruneForceTxIn) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do +pruneAndRollBackOneBlock = performPruneAndRollBackOneBlock False + +pruneAndRollBackOneBlockWithAddress :: IOManager -> [(Text, Text)] -> Assertion +pruneAndRollBackOneBlockWithAddress = performPruneAndRollBackOneBlock True + +performPruneAndRollBackOneBlock :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performPruneAndRollBackOneBlock useTxOutAddress = + withCustomConfigAndDropDB cmdLineArgs (Just $ configPruneForceTxIn useTxOutAddress) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do startDBSync dbSync let txOutTableType = txOutTableTypeFromConfig dbSync @@ -217,8 +261,14 @@ pruneAndRollBackOneBlock = testLabel = "conwayConfigPruneAndRollBack" noPruneAndRollBack :: IOManager -> [(Text, Text)] -> Assertion -noPruneAndRollBack = - withCustomConfig cmdLineArgs (Just configConsume) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do +noPruneAndRollBack = performNoPruneAndRollBack False + +noPruneAndRollBackWithAddress :: IOManager -> [(Text, Text)] -> Assertion +noPruneAndRollBackWithAddress = performNoPruneAndRollBack True + +performNoPruneAndRollBack :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performNoPruneAndRollBack useTxOutAddress = + withCustomConfigAndDropDB cmdLineArgs (Just $ configConsume useTxOutAddress) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do startDBSync dbSync let txOutTableType = txOutTableTypeFromConfig dbSync @@ -257,8 +307,14 @@ noPruneAndRollBack = testLabel = "conwayConfigNoPruneAndRollBack" pruneSameBlock :: IOManager -> [(Text, Text)] -> Assertion -pruneSameBlock = - withCustomConfig cmdLineArgs (Just configPruneForceTxIn) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do +pruneSameBlock = performPruneSameBlock False + +pruneSameBlockWithAddress :: IOManager -> [(Text, Text)] -> Assertion +pruneSameBlockWithAddress = performPruneSameBlock True + +performPruneSameBlock :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performPruneSameBlock useTxOutAddress = + withCustomConfigAndDropDB cmdLineArgs (Just $ configPruneForceTxIn useTxOutAddress) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do startDBSync dbSync let txOutTableType = txOutTableTypeFromConfig dbSync @@ -294,8 +350,14 @@ pruneSameBlock = testLabel = "conwayConfigPruneSameBlock" noPruneSameBlock :: IOManager -> [(Text, Text)] -> Assertion -noPruneSameBlock = - withCustomConfig cmdLineArgs (Just configConsume) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do +noPruneSameBlock = performNoPruneSameBlock False + +noPruneSameBlockWithAddress :: IOManager -> [(Text, Text)] -> Assertion +noPruneSameBlockWithAddress = performNoPruneSameBlock True + +performNoPruneSameBlock :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performNoPruneSameBlock useTxOutAddress = + withCustomConfigAndDropDB cmdLineArgs (Just $ configConsume useTxOutAddress) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do startDBSync dbSync -- Forge some blocks @@ -326,8 +388,14 @@ noPruneSameBlock = testLabel = "conwayConfigNoPruneSameBlock" migrateAndPruneRestart :: IOManager -> [(Text, Text)] -> Assertion -migrateAndPruneRestart = - withCustomConfig cmdLineArgs (Just configConsume) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do +migrateAndPruneRestart = performMigrateAndPruneRestart False + +migrateAndPruneRestartWithAddress :: IOManager -> [(Text, Text)] -> Assertion +migrateAndPruneRestartWithAddress = performMigrateAndPruneRestart True + +performMigrateAndPruneRestart :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performMigrateAndPruneRestart useTxOutAddress = + withCustomConfigAndDropDB cmdLineArgs (Just $ configConsume useTxOutAddress) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do startDBSync dbSync -- Forge some blocks @@ -349,8 +417,14 @@ migrateAndPruneRestart = testLabel = "conwayConfigMigrateAndPruneRestart" pruneRestartMissingFlag :: IOManager -> [(Text, Text)] -> Assertion -pruneRestartMissingFlag = - withCustomConfig cmdLineArgs (Just configPruneForceTxIn) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do +pruneRestartMissingFlag = performPruneRestartMissingFlag False + +pruneRestartMissingFlagWithAddress :: IOManager -> [(Text, Text)] -> Assertion +pruneRestartMissingFlagWithAddress = performPruneRestartMissingFlag True + +performPruneRestartMissingFlag :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performPruneRestartMissingFlag useTxOutAddress = + withCustomConfigAndDropDB cmdLineArgs (Just $ configPruneForceTxIn useTxOutAddress) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do startDBSync dbSync -- Forge some blocks @@ -372,8 +446,14 @@ pruneRestartMissingFlag = testLabel = "conwayConfigPruneRestartMissingFlag" bootstrapRestartMissingFlag :: IOManager -> [(Text, Text)] -> Assertion -bootstrapRestartMissingFlag = - withCustomConfig cmdLineArgs (Just configBootstrap) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do +bootstrapRestartMissingFlag = performBootstrapRestartMissingFlag False + +bootstrapRestartMissingFlagWithAddress :: IOManager -> [(Text, Text)] -> Assertion +bootstrapRestartMissingFlagWithAddress = performBootstrapRestartMissingFlag True + +performBootstrapRestartMissingFlag :: Bool -> IOManager -> [(Text, Text)] -> Assertion +performBootstrapRestartMissingFlag useTxOutAddress = + withCustomConfigAndDropDB cmdLineArgs (Just $ configBootstrap useTxOutAddress) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do startDBSync dbSync -- Forge some blocks @@ -393,3 +473,36 @@ bootstrapRestartMissingFlag = where cmdLineArgs = initCommandLineArgs testLabel = "conwayConfigBootstrapRestartMissingFlag" + +populateDbRestartWithAddressConfig :: IOManager -> [(Text, Text)] -> Assertion +populateDbRestartWithAddressConfig = + withCustomConfigAndDropDB cmdLineArgs (Just $ configConsume False) conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do + startDBSync dbSync + + -- Forge some blocks + void $ forgeAndSubmitBlocks interpreter mockServer 50 + -- Wait for them to sync + assertBlockNoBackoff dbSync 50 + + stopDBSync dbSync + + let newDBSync = + dbSync + { dbSyncConfig = + (dbSyncConfig dbSync) + { dncInsertOptions = + (dncInsertOptions $ dbSyncConfig dbSync) + { sioTxOut = TxOutConsumedPrune (ForceTxIn False) (UseTxOutAddress True) + } + } + } + -- Start without tx-out=prune + newEnv <- replaceConfigFile "test-db-sync-config.json" newDBSync + startDBSync newEnv + -- There is a slight delay before the flag is checked + threadDelay 6_000_000 + -- Expected to fail + checkStillRuns newDBSync + where + cmdLineArgs = initCommandLineArgs + testLabel = "conwayPopulateDbRestartWithAddressConfig" diff --git a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Validate.hs b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Validate.hs index 574e033ef..907d18a5f 100644 --- a/cardano-chain-gen/test/Test/Cardano/Db/Mock/Validate.hs +++ b/cardano-chain-gen/test/Test/Cardano/Db/Mock/Validate.hs @@ -136,9 +136,10 @@ expectFailSilent name action = testCase name $ do -- checking that unspent count matches from tx_in to tx_out assertUnspentTx :: DBSyncEnv -> IO () -assertUnspentTx syncEnv = do - unspentTxCount <- queryDBSync syncEnv $ DB.queryTxOutConsumedNullCount TxOutCore - consumedNullCount <- queryDBSync syncEnv $ DB.queryTxOutUnspentCount TxOutCore +assertUnspentTx dbSyncEnv = do + let txOutTableType = txOutTableTypeFromConfig dbSyncEnv + unspentTxCount <- queryDBSync dbSyncEnv $ DB.queryTxOutConsumedNullCount txOutTableType + consumedNullCount <- queryDBSync dbSyncEnv $ DB.queryTxOutUnspentCount txOutTableType assertEqual "Unexpected tx unspent count between tx-in & tx-out" unspentTxCount consumedNullCount defaultDelays :: [Int] diff --git a/cardano-db-sync/src/Cardano/DbSync.hs b/cardano-db-sync/src/Cardano/DbSync.hs index 6bfa5b9a9..9df654d4c 100644 --- a/cardano-db-sync/src/Cardano/DbSync.hs +++ b/cardano-db-sync/src/Cardano/DbSync.hs @@ -100,7 +100,7 @@ runDbSync metricsSetters knownMigrations iomgr trce params syncNodeConfigFromFil logInfo trce $ "Running database migrations in mode " <> textShow mode logInfo trce msg when (mode `elem` [Db.Indexes, Db.Full]) $ logWarning trce indexesMsg - Db.runMigrations pgConfig True dbMigrationDir (Just $ Db.LogFileDir "/tmp") mode + Db.runMigrations pgConfig True dbMigrationDir (Just $ Db.LogFileDir "/tmp") mode (txOutConfigToTableType txOutConfig) (ranMigrations, unofficial) <- if enpForceIndexes params then runMigration Db.Full else runMigration Db.Initial unless (null unofficial) $ logWarning trce $ diff --git a/cardano-db-sync/src/Cardano/DbSync/Api.hs b/cardano-db-sync/src/Cardano/DbSync/Api.hs index 66162924c..e31e3d629 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Api.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Api.hs @@ -54,7 +54,6 @@ import Cardano.BM.Trace (Trace, logInfo, logWarning) import qualified Cardano.Chain.Genesis as Byron import Cardano.Crypto.ProtocolMagic (ProtocolMagicId (..)) import qualified Cardano.Db as DB -import qualified Cardano.Db as Multiplex (queryWrongConsumedBy) import Cardano.DbSync.Api.Types import Cardano.DbSync.Cache.Types (CacheCapacity (..), newEmptyCache, useNoCache) import Cardano.DbSync.Config.Cardano @@ -118,7 +117,7 @@ isConsistent env = do getIsConsumedFixed :: SyncEnv -> IO (Maybe Word64) getIsConsumedFixed env = case (DB.pcmPruneTxOut pcm, DB.pcmConsumedTxOut pcm) of - (False, True) -> Just <$> DB.runDbIohkNoLogging backend (Multiplex.queryWrongConsumedBy txOutTableType) + (False, True) -> Just <$> DB.runDbIohkNoLogging backend (DB.queryWrongConsumedBy txOutTableType) _ -> pure Nothing where txOutTableType = getTxOutTableType env diff --git a/cardano-db-sync/src/Cardano/DbSync/Config/Types.hs b/cardano-db-sync/src/Cardano/DbSync/Config/Types.hs index d55bfcaac..55ff83c89 100644 --- a/cardano-db-sync/src/Cardano/DbSync/Config/Types.hs +++ b/cardano-db-sync/src/Cardano/DbSync/Config/Types.hs @@ -692,10 +692,10 @@ instance ToJSON RemoveJsonbFromSchemaConfig where toJSON = boolToEnableDisable . isRemoveJsonbFromSchemaEnabled instance FromJSON TxOutTableTypeConfig where - parseJSON = Aeson.withText "add_address_table_to_txout" $ \v -> + parseJSON = Aeson.withText "use_address_table" $ \v -> case enableDisableToTxOutTableType v of Just g -> pure (TxOutTableTypeConfig g) - Nothing -> fail $ "unexpected add_address_table_to_txout: " <> show v + Nothing -> fail $ "unexpected use_address_table: " <> show v instance ToJSON TxOutTableTypeConfig where toJSON = addressTypeToEnableDisable . unTxOutTableTypeConfig diff --git a/cardano-db-tool/app/cardano-db-tool.hs b/cardano-db-tool/app/cardano-db-tool.hs index 16e2a9bf3..821d4cdcb 100644 --- a/cardano-db-tool/app/cardano-db-tool.hs +++ b/cardano-db-tool/app/cardano-db-tool.hs @@ -34,10 +34,10 @@ main = do -- ----------------------------------------------------------------------------- data Command - = CmdCreateMigration !MigrationDir + = CmdCreateMigration !MigrationDir !TxOutTableType | CmdReport !Report !TxOutTableType | CmdRollback !SlotNo !TxOutTableType - | CmdRunMigrations !MigrationDir !Bool !Bool !(Maybe LogFileDir) + | CmdRunMigrations !MigrationDir !Bool !Bool !(Maybe LogFileDir) !TxOutTableType | CmdTxOutMigration !TxOutTableType | CmdUtxoSetAtBlock !Word64 !TxOutTableType | CmdPrepareSnapshot !PrepareSnapshotArgs @@ -48,32 +48,32 @@ data Command runCommand :: Command -> IO () runCommand cmd = case cmd of - CmdCreateMigration mdir -> runCreateMigration mdir + CmdCreateMigration mdir txOutAddressType -> runCreateMigration mdir txOutAddressType CmdReport report txOutAddressType -> runReport report txOutAddressType CmdRollback slotNo txOutAddressType -> runRollback slotNo txOutAddressType - CmdRunMigrations mdir forceIndexes mockFix mldir -> do + CmdRunMigrations mdir forceIndexes mockFix mldir txOutTabletype -> do pgConfig <- runOrThrowIODb (readPGPass PGPassDefaultEnv) - unofficial <- snd <$> runMigrations pgConfig False mdir mldir Initial + unofficial <- snd <$> runMigrations pgConfig False mdir mldir Initial txOutTabletype unless (null unofficial) $ putStrLn $ "Unofficial migration scripts found: " ++ show unofficial when forceIndexes $ void $ - runMigrations pgConfig False mdir mldir Indexes + runMigrations pgConfig False mdir mldir Indexes txOutTabletype when mockFix $ void $ - runMigrations pgConfig False mdir mldir Fix + runMigrations pgConfig False mdir mldir Fix txOutTabletype CmdTxOutMigration txOutTableType -> do - runWithConnectionNoLogging PGPassDefaultEnv $ migrateTxOutTests txOutTableType + runWithConnectionNoLogging PGPassDefaultEnv $ migrateTxOutDbTool txOutTableType CmdUtxoSetAtBlock blkid txOutAddressType -> utxoSetAtSlot txOutAddressType blkid CmdPrepareSnapshot pargs -> runPrepareSnapshot pargs CmdValidateDb txOutAddressType -> runDbValidation txOutAddressType CmdValidateAddressBalance params txOutAddressType -> runLedgerValidation params txOutAddressType CmdVersion -> runVersionCommand -runCreateMigration :: MigrationDir -> IO () -runCreateMigration mdir = do - mfp <- createMigration PGPassDefaultEnv mdir +runCreateMigration :: MigrationDir -> TxOutTableType -> IO () +runCreateMigration mdir txOutTableType = do + mfp <- createMigration PGPassDefaultEnv mdir txOutTableType case mfp of Nothing -> putStrLn "No migration needed." Just fp -> putStrLn $ "New migration '" ++ fp ++ "' created." @@ -163,7 +163,7 @@ pCommand = where pCreateMigration :: Parser Command pCreateMigration = - CmdCreateMigration <$> pMigrationDir + CmdCreateMigration <$> pMigrationDir <*> pTxOutTableType pRunMigrations :: Parser Command pRunMigrations = @@ -172,6 +172,7 @@ pCommand = <*> pForceIndexes <*> pMockFix <*> optional pLogFileDir + <*> pTxOutTableType pRollback :: Parser Command pRollback = diff --git a/cardano-db/src/Cardano/Db/Migration.hs b/cardano-db/src/Cardano/Db/Migration.hs index ff2c57842..7e5e01396 100644 --- a/cardano-db/src/Cardano/Db/Migration.hs +++ b/cardano-db/src/Cardano/Db/Migration.hs @@ -29,10 +29,12 @@ import Cardano.Crypto.Hash (Blake2b_256, ByteString, Hash, hashToStringAsHex, ha import Cardano.Db.Migration.Haskell import Cardano.Db.Migration.Version import Cardano.Db.Operations.Query +import Cardano.Db.Operations.Types (TxOutTableType (..)) import Cardano.Db.PGConfig import Cardano.Db.Run import Cardano.Db.Schema.BaseSchema import Cardano.Db.Schema.Core.TxOut (migrateCoreTxOutCardanoDb) +import Cardano.Db.Schema.Variant.TxOut (migrateVariantAddressCardanoDb) import Cardano.Prelude (Typeable, textShow) import Control.Exception (Exception, SomeException, handle) import Control.Monad.Extra @@ -104,8 +106,8 @@ data MigrationToRun = Initial | Full | Fix | Indexes -- | Run the migrations in the provided 'MigrationDir' and write date stamped log file -- to 'LogFileDir'. It returns a list of file names of all non-official schema migration files. -runMigrations :: PGConfig -> Bool -> MigrationDir -> Maybe LogFileDir -> MigrationToRun -> IO (Bool, [FilePath]) -runMigrations pgconfig quiet migrationDir mLogfiledir mToRun = do +runMigrations :: PGConfig -> Bool -> MigrationDir -> Maybe LogFileDir -> MigrationToRun -> TxOutTableType -> IO (Bool, [FilePath]) +runMigrations pgconfig quiet migrationDir mLogfiledir mToRun txOutTableType = do allScripts <- getMigrationScripts migrationDir ranAll <- case (mLogfiledir, allScripts) of (_, []) -> @@ -140,23 +142,22 @@ runMigrations pgconfig quiet migrationDir mLogfiledir mToRun = do filterMigrations :: [(MigrationVersion, FilePath)] -> IO ([(MigrationVersion, FilePath)], Bool) filterMigrations scripts = case mToRun of - Full -> do - mVersion <- runWithConnectionNoLogging (PGPassCached pgconfig) querySchemaVersion - case mVersion of - Just (SchemaVersion _ v _) | v == hardCoded3_0 -> do - pure (filter (not . filterFix) scripts, False) - _ -> pure (scripts, True) - Initial -> do - mVersion <- runWithConnectionNoLogging (PGPassCached pgconfig) querySchemaVersion - case mVersion of - Just (SchemaVersion _ v _) | v == hardCoded3_0 -> do - pure (filter (\m -> not $ filterFix m || filterIndexes m) scripts, False) - _ -> pure (filter (not . filterIndexes) scripts, True) + Full -> pure (filter filterIndexesFull scripts, True) + Initial -> pure (filter filterInitial scripts, True) Fix -> pure (filter filterFix scripts, False) - Indexes -> pure (filter filterIndexes scripts, False) + Indexes -> do + pure (filter filterIndexes scripts, False) filterFix (mv, _) = mvStage mv == 2 && mvVersion mv > hardCoded3_0 - filterIndexes (mv, _) = mvStage mv == 4 + filterIndexesFull (mv, _) = do + case txOutTableType of + TxOutCore -> True + TxOutVariantAddress -> not $ mvStage mv == 4 && mvVersion mv == 1 + filterInitial (mv, _) = mvStage mv < 4 + filterIndexes (mv, _) = do + case txOutTableType of + TxOutCore -> mvStage mv == 4 + TxOutVariantAddress -> mvStage mv == 4 && mvVersion mv > 1 hardCoded3_0 :: Int hardCoded3_0 = 19 @@ -226,8 +227,8 @@ applyMigration (MigrationDir location) quiet pgconfig mLogFilename logHandle (ve -- | Create a database migration (using functionality built into Persistent). If no -- migration is needed return 'Nothing' otherwise return the migration as 'Text'. -createMigration :: PGPassSource -> MigrationDir -> IO (Maybe FilePath) -createMigration source (MigrationDir migdir) = do +createMigration :: PGPassSource -> MigrationDir -> TxOutTableType -> IO (Maybe FilePath) +createMigration source (MigrationDir migdir) txOutTableType = do mt <- runDbNoLogging source create case mt of Nothing -> pure Nothing @@ -239,10 +240,16 @@ createMigration source (MigrationDir migdir) = do create :: ReaderT SqlBackend (NoLoggingT IO) (Maybe (MigrationVersion, Text)) create = do ver <- getSchemaVersion - -- here is the place to combine any "core" schemas to the base schema statementsBase <- getMigration migrateBaseCardanoDb - statementsTxOut <- getMigration migrateCoreTxOutCardanoDb - let statements = statementsBase <> statementsTxOut + -- handle what type of migration to generate + statements <- + case txOutTableType of + TxOutCore -> do + statementsTxOut <- getMigration migrateCoreTxOutCardanoDb + pure $ statementsBase <> statementsTxOut + TxOutVariantAddress -> do + statementsTxOut <- getMigration migrateVariantAddressCardanoDb + pure $ statementsBase <> statementsTxOut if null statements then pure Nothing else do diff --git a/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs b/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs index 6cba99935..d6c77d745 100644 --- a/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs +++ b/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs @@ -23,7 +23,7 @@ import Cardano.Db.Schema.BaseSchema import qualified Cardano.Db.Schema.Core.TxOut as C import qualified Cardano.Db.Schema.Variant.TxOut as V import Cardano.Db.Types (ExtraMigration (..), MigrationValues (..), PruneConsumeMigration (..), processMigrationValues) -import Cardano.Prelude (textShow) +import Cardano.Prelude (textShow, void) import Control.Exception (throw) import Control.Exception.Lifted (handle, throwIO) import Control.Monad.Extra (unless, when, whenJust) @@ -52,63 +52,68 @@ data ConsumedTriplet = ConsumedTriplet -------------------------------------------------------------------------------------------------- -- Queries -------------------------------------------------------------------------------------------------- - --- | This is a count of the null consumed_by_tx_id -queryTxOutConsumedNullCount :: TxOutTableType -> MonadIO m => ReaderT SqlBackend m Word64 -queryTxOutConsumedNullCount = \case - TxOutCore -> query @'TxOutCore - TxOutVariantAddress -> query @'TxOutVariantAddress - where - query :: - forall (a :: TxOutTableType) m. - (MonadIO m, TxOutFields a) => - ReaderT SqlBackend m Word64 - query = do - res <- select $ do - txOut <- from $ table @(TxOutTable a) - where_ (isNothing $ txOut ^. txOutConsumedByTxIdField @a) - pure countRows - pure $ maybe 0 unValue (listToMaybe res) - -queryTxOutConsumedCount :: TxOutTableType -> MonadIO m => ReaderT SqlBackend m Word64 -queryTxOutConsumedCount = \case - TxOutCore -> query @'TxOutCore - TxOutVariantAddress -> query @'TxOutVariantAddress - where - query :: - forall (a :: TxOutTableType) m. - (MonadIO m, TxOutFields a) => - ReaderT SqlBackend m Word64 - query = do - res <- select $ do - txOut <- from $ table @(TxOutTable a) - where_ (not_ $ isNothing $ txOut ^. txOutConsumedByTxIdField @a) - pure countRows - pure $ maybe 0 unValue (listToMaybe res) - querySetNullTxOut :: MonadIO m => Trace IO Text -> TxOutTableType -> Maybe TxId -> ReaderT SqlBackend m () querySetNullTxOut trce txOutTableType mMinTxId = do whenJust mMinTxId $ \txId -> do - txOutIds <- getTxOutConsumedAfter txOutTableType txId - mapM_ (setNullTxOutConsumedAfter txOutTableType) txOutIds + txOutIds <- getTxOutConsumedAfter txId + mapM_ setNullTxOutConsumedAfter txOutIds let updatedEntries = length txOutIds liftIO $ logInfo trce $ "Set to null " <> textShow updatedEntries <> " tx_out.consumed_by_tx_id" - -updateListTxOutConsumedByTxId :: MonadIO m => [(TxOutIdW, TxId)] -> ReaderT SqlBackend m () -updateListTxOutConsumedByTxId ls = do - mapM_ (uncurry updateTxOutConsumedByTxId) ls + where + -- \| This requires an index at TxOutConsumedByTxId. + getTxOutConsumedAfter :: MonadIO m => TxId -> ReaderT SqlBackend m [TxOutIdW] + getTxOutConsumedAfter txId = + case txOutTableType of + TxOutCore -> wrapTxOutIds CTxOutIdW (queryConsumedTxOutIds @'TxOutCore txId) + TxOutVariantAddress -> wrapTxOutIds VTxOutIdW (queryConsumedTxOutIds @'TxOutVariantAddress txId) + where + wrapTxOutIds constructor = fmap (map constructor) + + queryConsumedTxOutIds :: + forall a m. + (TxOutFields a, MonadIO m) => + TxId -> + ReaderT SqlBackend m [TxOutIdFor a] + queryConsumedTxOutIds txId' = do + res <- select $ do + txOut <- from $ table @(TxOutTable a) + where_ (txOut ^. txOutConsumedByTxIdField @a >=. just (val txId')) + pure $ txOut ^. txOutIdField @a + pure $ map unValue res + + -- \| This requires an index at TxOutConsumedByTxId. + setNullTxOutConsumedAfter :: MonadIO m => TxOutIdW -> ReaderT SqlBackend m () + setNullTxOutConsumedAfter txOutId = + case txOutTableType of + TxOutCore -> setNull + TxOutVariantAddress -> setNull + where + setNull :: + (MonadIO m) => + ReaderT SqlBackend m () + setNull = do + case txOutId of + CTxOutIdW txOutId' -> update txOutId' [C.TxOutConsumedByTxId =. Nothing] + VTxOutIdW txOutId' -> update txOutId' [V.TxOutConsumedByTxId =. Nothing] runExtraMigrations :: (MonadBaseControl IO m, MonadIO m) => Trace IO Text -> TxOutTableType -> Word64 -> PruneConsumeMigration -> ReaderT SqlBackend m () runExtraMigrations trce txOutTableType blockNoDiff pcm = do ems <- queryAllExtraMigrations + isTxOutNull <- queryTxOutIsNull txOutTableType let migrationValues = processMigrationValues ems pcm - -- Make sure the config address_table is there if the migration wasn't previously set in teh db - when (not (isTxOutVariantAddress txOutTableType) && isTxOutAddressPreviouslySet migrationValues) $ throw $ DBExtraMigration "The configuration option 'tx_out.address_table' was previously set and the database updated. Unfortunately reverting this isn't possible." + + -- can only run "use_address_table" on a non populated database + when (isTxOutVariantAddress txOutTableType && not isTxOutNull) $ + throw $ + DBExtraMigration "runExtraMigrations: The use the config 'tx_out.use_address_table' can only be caried out on a non populated database." + -- Make sure the config "use_address_table" is there if the migration wasn't previously set in the past + when (not (isTxOutVariantAddress txOutTableType) && isTxOutAddressPreviouslySet migrationValues) $ + throw $ + DBExtraMigration "runExtraMigrations: The configuration option 'tx_out.use_address_table' was previously set and the database updated. Unfortunately reverting this isn't possible." -- Has the user given txout address config && the migration wasn't previously set when (isTxOutVariantAddress txOutTableType && not (isTxOutAddressPreviouslySet migrationValues)) $ do updateTxOutAndCreateAddress insertExtraMigration TxOutAddressPreviouslySet - -- first check if pruneTxOut flag is missing and it has previously been used when (isPruneTxOutPreviouslySet migrationValues && not (pcmPruneTxOut pcm)) $ throw $ @@ -159,54 +164,70 @@ queryWrongConsumedBy = \case pure $ maybe 0 unValue (listToMaybe res) -------------------------------------------------------------------------------------------------- --- Updates +-- Queries Tests -------------------------------------------------------------------------------------------------- -updateTxOutConsumedByTxId :: MonadIO m => TxOutIdW -> TxId -> ReaderT SqlBackend m () -updateTxOutConsumedByTxId txOutId txId = - case txOutId of - CTxOutIdW txOutId' -> update txOutId' [C.TxOutConsumedByTxId =. Just txId] - VTxOutIdW txOutId' -> update txOutId' [V.TxOutConsumedByTxId =. Just txId] - --- | This requires an index at TxOutConsumedByTxId. -getTxOutConsumedAfter :: MonadIO m => TxOutTableType -> TxId -> ReaderT SqlBackend m [TxOutIdW] -getTxOutConsumedAfter txOutTableType txId = - case txOutTableType of - TxOutCore -> wrapTxOutIds CTxOutIdW (queryConsumedTxOutIds @'TxOutCore txId) - TxOutVariantAddress -> wrapTxOutIds VTxOutIdW (queryConsumedTxOutIds @'TxOutVariantAddress txId) + +-- | This is a count of the null consumed_by_tx_id +queryTxOutConsumedNullCount :: TxOutTableType -> MonadIO m => ReaderT SqlBackend m Word64 +queryTxOutConsumedNullCount = \case + TxOutCore -> query @'TxOutCore + TxOutVariantAddress -> query @'TxOutVariantAddress + where + query :: + forall (a :: TxOutTableType) m. + (MonadIO m, TxOutFields a) => + ReaderT SqlBackend m Word64 + query = do + res <- select $ do + txOut <- from $ table @(TxOutTable a) + where_ (isNothing $ txOut ^. txOutConsumedByTxIdField @a) + pure countRows + pure $ maybe 0 unValue (listToMaybe res) + +queryTxOutConsumedCount :: TxOutTableType -> MonadIO m => ReaderT SqlBackend m Word64 +queryTxOutConsumedCount = \case + TxOutCore -> query @'TxOutCore + TxOutVariantAddress -> query @'TxOutVariantAddress where - wrapTxOutIds constructor = fmap (map constructor) - - queryConsumedTxOutIds :: - forall a m. - (TxOutFields a, MonadIO m) => - TxId -> - ReaderT SqlBackend m [TxOutIdFor a] - queryConsumedTxOutIds txId' = do + query :: + forall (a :: TxOutTableType) m. + (MonadIO m, TxOutFields a) => + ReaderT SqlBackend m Word64 + query = do res <- select $ do txOut <- from $ table @(TxOutTable a) - where_ (txOut ^. txOutConsumedByTxIdField @a >=. just (val txId')) - pure $ txOut ^. txOutIdField @a - pure $ map unValue res + where_ (not_ $ isNothing $ txOut ^. txOutConsumedByTxIdField @a) + pure countRows + pure $ maybe 0 unValue (listToMaybe res) --- | This requires an index at TxOutConsumedByTxId. -setNullTxOutConsumedAfter :: MonadIO m => TxOutTableType -> TxOutIdW -> ReaderT SqlBackend m () -setNullTxOutConsumedAfter txOutTableType txOutId = - case txOutTableType of - TxOutCore -> setNull - TxOutVariantAddress -> setNull +queryTxOutIsNull :: TxOutTableType -> MonadIO m => ReaderT SqlBackend m Bool +queryTxOutIsNull = \case + TxOutCore -> pure False + TxOutVariantAddress -> query @'TxOutVariantAddress where - setNull :: - (MonadIO m) => - ReaderT SqlBackend m () - setNull = do - case txOutId of - CTxOutIdW txOutId' -> update txOutId' [C.TxOutConsumedByTxId =. Nothing] - VTxOutIdW txOutId' -> update txOutId' [V.TxOutConsumedByTxId =. Nothing] + query :: + forall (a :: TxOutTableType) m. + (MonadIO m, TxOutFields a) => + ReaderT SqlBackend m Bool + query = do + res <- select $ do + _ <- from $ table @(TxOutTable a) + limit 1 + pure (val (1 :: Int)) + pure $ null res -migrateTxOutTests :: (MonadIO m, MonadBaseControl IO m) => TxOutTableType -> ReaderT SqlBackend m () -migrateTxOutTests txOutTableType = do - _ <- createConsumedIndexTxOut - migrateNextPageTxOut Nothing txOutTableType 0 +-------------------------------------------------------------------------------------------------- +-- Updates +-------------------------------------------------------------------------------------------------- +updateListTxOutConsumedByTxId :: MonadIO m => [(TxOutIdW, TxId)] -> ReaderT SqlBackend m () +updateListTxOutConsumedByTxId ls = do + mapM_ (uncurry updateTxOutConsumedByTxId) ls + where + updateTxOutConsumedByTxId :: MonadIO m => TxOutIdW -> TxId -> ReaderT SqlBackend m () + updateTxOutConsumedByTxId txOutId txId = + case txOutId of + CTxOutIdW txOutId' -> update txOutId' [C.TxOutConsumedByTxId =. Just txId] + VTxOutIdW txOutId' -> update txOutId' [V.TxOutConsumedByTxId =. Just txId] migrateTxOut :: ( MonadBaseControl IO m @@ -216,21 +237,34 @@ migrateTxOut :: TxOutTableType -> Maybe MigrationValues -> ReaderT SqlBackend m () -migrateTxOut trce txOutTableType _mMvs = do - liftIO $ logInfo trce "migrateTxOut:" - _ <- createConsumedIndexTxOut +migrateTxOut trce txOutTableType mMvs = do + whenJust mMvs $ \mvs -> do + liftIO $ logInfo trce $ "migrateTxOut: previously set: " <> textShow (not (isTxOutAddressPreviouslySet mvs)) + liftIO $ logInfo trce $ "migrateTxOut: pcmConsumedTxOut: " <> textShow (pcmConsumedTxOut (pruneConsumeMigration mvs)) + when (pcmConsumedTxOut (pruneConsumeMigration mvs) && not (isTxOutAddressPreviouslySet mvs)) $ do + liftIO $ logInfo trce "migrateTxOut: addeding consumed-by-id Index" + void createConsumedIndexTxOut + when (pcmPruneTxOut (pruneConsumeMigration mvs)) $ do + liftIO $ logInfo trce "migrateTxOut: adding prune contraint on tx_out table" + void createPruneConstraintTxOut migrateNextPageTxOut (Just trce) txOutTableType 0 migrateNextPageTxOut :: MonadIO m => Maybe (Trace IO Text) -> TxOutTableType -> Word64 -> ReaderT SqlBackend m () migrateNextPageTxOut mTrce txOutTableType offst = do whenJust mTrce $ \trce -> - liftIO $ logInfo trce $ "Handling input offset " <> textShow offst + liftIO $ logInfo trce $ "migrateNextPageTxOut: Handling input offset " <> textShow offst page <- getInputPage offst pageSize updatePageEntries txOutTableType page when (fromIntegral (length page) == pageSize) $ migrateNextPageTxOut mTrce txOutTableType $! (offst + pageSize) +-- TODO: cmdv put into tools or something +migrateTxOutDbTool :: (MonadIO m, MonadBaseControl IO m) => TxOutTableType -> ReaderT SqlBackend m () +migrateTxOutDbTool txOutTableType = do + _ <- createConsumedIndexTxOut + migrateNextPageTxOut Nothing txOutTableType 0 + -------------------------------------------------------------------------------------------------- -- Delete + Update -------------------------------------------------------------------------------------------------- @@ -247,8 +281,9 @@ deleteAndUpdateConsumedTxOut trce txOutTableType migrationValues blockNoDiff = d case maxTxId of Left errMsg -> do liftIO $ logInfo trce $ "No tx_out were deleted as no blocks found: " <> errMsg - liftIO $ logInfo trce "Now Running extra migration prune tx_out" + liftIO $ logInfo trce "deleteAndUpdateConsumedTxOut: Now Running extra migration prune tx_out" migrateTxOut trce txOutTableType $ Just migrationValues + insertExtraMigration ConsumeTxOutPreviouslySet Right mTxId -> do migrateNextPage mTxId False 0 where @@ -350,13 +385,13 @@ createConsumedIndexTxOut = do exceptHandler e = liftIO $ throwIO (DBPruneConsumed $ show e) -createConsumedConstraintTxOut :: +createPruneConstraintTxOut :: forall m. ( MonadBaseControl IO m , MonadIO m ) => ReaderT SqlBackend m () -createConsumedConstraintTxOut = do +createPruneConstraintTxOut = do handle exceptHandler $ rawExecute addConstraint [] where addConstraint = @@ -385,9 +420,18 @@ updateTxOutAndCreateAddress :: ) => ReaderT SqlBackend m () updateTxOutAndCreateAddress = do + handle exceptHandler $ rawExecute dropViewsQuery [] handle exceptHandler $ rawExecute alterTxOutQuery [] handle exceptHandler $ rawExecute createAddressTableQuery [] + handle exceptHandler $ rawExecute createIndexPaymentCredQuery [] + handle exceptHandler $ rawExecute createIndexRawQuery [] where + dropViewsQuery = + Text.unlines + [ "DROP VIEW IF EXISTS utxo_byron_view;" + , "DROP VIEW IF EXISTS utxo_view;" + ] + alterTxOutQuery = Text.unlines [ "ALTER TABLE \"tx_out\"" @@ -397,6 +441,7 @@ updateTxOutAndCreateAddress = do , " DROP COLUMN \"payment_cred\"," , " DROP COLUMN \"stake_address_id\"" ] + createAddressTableQuery = Text.unlines [ "CREATE TABLE \"address\" (" @@ -408,6 +453,13 @@ updateTxOutAndCreateAddress = do , " \"stake_address_id\" INT8 NULL" , ")" ] + + createIndexPaymentCredQuery = + "CREATE INDEX IF NOT EXISTS idx_address_payment_cred ON address(payment_cred);" + + createIndexRawQuery = + "CREATE INDEX IF NOT EXISTS idx_address_raw ON address(raw);" + exceptHandler :: SqlError -> ReaderT SqlBackend m a exceptHandler e = liftIO $ throwIO (DBPruneConsumed $ show e) diff --git a/cardano-db/src/Cardano/Db/Operations/Other/MinId.hs b/cardano-db/src/Cardano/Db/Operations/Other/MinId.hs index 3c03942a7..261c47064 100644 --- a/cardano-db/src/Cardano/Db/Operations/Other/MinId.hs +++ b/cardano-db/src/Cardano/Db/Operations/Other/MinId.hs @@ -108,7 +108,6 @@ minJust (Just x) (Just y) = Just (min x y) -------------------------------------------------------------------------------- -- CompleteMinId -------------------------------------------------------------------------------- --- example use case would be: `result <- completeMinId @'TxOutCore mTxId minIds` completeMinId :: (MonadIO m) => Maybe TxId -> diff --git a/cardano-db/test/Test/IO/Cardano/Db/Migration.hs b/cardano-db/test/Test/IO/Cardano/Db/Migration.hs index 642923c1c..640b68a45 100644 --- a/cardano-db/test/Test/IO/Cardano/Db/Migration.hs +++ b/cardano-db/test/Test/IO/Cardano/Db/Migration.hs @@ -12,6 +12,7 @@ import Cardano.Db ( MigrationValidateError (..), MigrationVersion (..), SchemaVersion (..), + TxOutTableType (..), getMigrationScripts, querySchemaVersion, readPGPassDefault, @@ -131,7 +132,7 @@ migrationTest :: IO () migrationTest = do let schemaDir = MigrationDir "../schema" pgConfig <- runOrThrowIODb readPGPassDefault - _ <- runMigrations pgConfig True schemaDir (Just $ LogFileDir "/tmp") Initial + _ <- runMigrations pgConfig True schemaDir (Just $ LogFileDir "/tmp") Initial TxOutVariantAddress expected <- readSchemaVersion schemaDir actual <- getDbSchemaVersion unless (expected == actual) $ diff --git a/doc/configuration.md b/doc/configuration.md index db3e86f91..be936af4a 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -194,20 +194,29 @@ Disables almost all data except `block` and `tx` tables. * Type: `object` + **Example** + ``` + "tx_out": { + "value": "consumed", + "force_tx_in": false, + "use_address_table": true, + }, + ``` + Tx Out Properties: | Property | Type | Required | | :------------------------------- | :-------- | :------- | | [value](#value) | `string` | Optional | | [force\_tx\_in](#force-tx-in) | `boolean` | Optional | -| [address\_table](#address-table) | `boolean` | Optional | +| [use\_address\_table](#address-table) | `boolean` | Optional | #### Value `tx_out.value` * Type: `string` - + **enum**: the value of this property must be equal to one of the following values: | Value | Explanation | @@ -270,11 +279,14 @@ can be changed. `tx_out.force_tx_in` * Type: `boolean` + +This value defaults to false. + ### Address Table -`tx_out.address_table` +`tx_out.use_address_table` * Type: `boolean` @@ -290,8 +302,14 @@ Key changes in the variant representation: - Replaces `address`, `address_has_script`, and `payment_cred` fields with a single `address_id` field - `addressId` references the new `Address` table +The address table can only be used on an empty database due to the schema restructuring which would cause data loss. + +The following indexes are added to the new `address` table: +1. `idx_address_payment_cred ON address(payment_cred)` +2. `idx_address_raw ON address(raw)` +Then `address.id` having a unique constraint. ## Ledger diff --git a/schema/migration-4-0001-20200702.sql b/schema/migration-4-0001-20200702.sql index 5fe18540a..f10c18c30 100644 --- a/schema/migration-4-0001-20200702.sql +++ b/schema/migration-4-0001-20200702.sql @@ -1,3 +1,10 @@ +-- +-- The following index creations only happen when not using "use_address_table" config in tx-out. +-- This is because the following uses columns that get moved to the `address` table . +-- --- This Index takes a lot of time to be created. -CREATE INDEX IF NOT EXISTS idx_tx_out_address ON tx_out USING hash (address); +CREATE INDEX IF NOT EXISTS idx_tx_out_payment_cred ON tx_out(payment_cred); +CREATE INDEX IF NOT EXISTS idx_tx_out_stake_address_id ON tx_out(stake_address_id) ; + +-- Left here for reference, it's removed to speed up restoring from a snapshot as this index is very slow to create. +-- CREATE INDEX IF NOT EXISTS idx_tx_out_address ON tx_out USING hash (address); diff --git a/schema/migration-4-0002-20200810.sql b/schema/migration-4-0002-20200810.sql index 4805612ca..e14a44bad 100644 --- a/schema/migration-4-0002-20200810.sql +++ b/schema/migration-4-0002-20200810.sql @@ -1,4 +1,3 @@ CREATE INDEX IF NOT EXISTS idx_block_time ON block(time); -CREATE INDEX IF NOT EXISTS idx_tx_out_payment_cred ON tx_out(payment_cred); CREATE INDEX IF NOT EXISTS idx_pool_update_hash_id ON pool_update(hash_id); diff --git a/schema/migration-4-0003-20210116.sql b/schema/migration-4-0003-20210116.sql index 11faa861e..617896a0e 100644 --- a/schema/migration-4-0003-20210116.sql +++ b/schema/migration-4-0003-20210116.sql @@ -34,7 +34,6 @@ CREATE INDEX IF NOT EXISTS idx_reward_addr_id ON reward(addr_id) ; CREATE INDEX IF NOT EXISTS idx_stake_deregistration_addr_id ON stake_deregistration(addr_id) ; CREATE INDEX IF NOT EXISTS idx_stake_registration_addr_id ON stake_registration(addr_id) ; CREATE INDEX IF NOT EXISTS idx_treasury_addr_id ON treasury(addr_id) ; -CREATE INDEX IF NOT EXISTS idx_tx_out_stake_address_id ON tx_out(stake_address_id) ; CREATE INDEX IF NOT EXISTS idx_withdrawal_addr_id ON withdrawal(addr_id) ; CREATE INDEX IF NOT EXISTS idx_ma_tx_out_tx_out_id ON ma_tx_out(tx_out_id) ; CREATE INDEX IF NOT EXISTS idx_pool_update_meta_id ON pool_update(meta_id) ;