update tests for txout variants and running txout migrations

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Plutus.hs

@@ -110,7 +110,7 @@ simpleScript =
           , V.txOutValue txout
           , V.txOutDataHash txout
           )
-        Nothing -> error "AlonzosimpleScript: expected an address"
+        Nothing -> error "AlonzoSimpleScript: expected an address"
     expectedFields =
       ( renderAddress alwaysSucceedsScriptAddr
       , True

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/Config/MigrateConsumedPruneTxOut.hs

@@ -112,7 +112,7 @@ basicPrune = do
 
 pruneWithSimpleRollback :: IOManager -> [(Text, Text)] -> Assertion
 pruneWithSimpleRollback =
-  withCustomConfig cmdLineArgs Nothing conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do
+  withCustomConfigAndLogs cmdLineArgs Nothing conwayConfigDir testLabel $ \interpreter mockServer dbSync -> do
     let txOutTableType = txOutTableTypeFromConfig dbSync
     -- Forge some blocks
     blk0 <- forgeNext interpreter mockBlock0

cardano-db-sync/src/Cardano/DbSync/Api.hs

@@ -168,6 +168,7 @@ initPruneConsumeMigration :: Bool -> Bool -> Bool -> Bool -> DB.PruneConsumeMigr
 initPruneConsumeMigration consumed pruneTxOut bootstrap forceTxIn' =
   DB.PruneConsumeMigration
     { DB.pcmPruneTxOut = pruneTxOut || bootstrap
+    , DB.pcmConsumedTxOut = consumed
     , DB.pcmConsumeOrPruneTxOut = consumed || pruneTxOut || bootstrap
     , DB.pcmSkipTxIn = not forceTxIn' && (consumed || pruneTxOut || bootstrap)
     }

cardano-db-sync/src/Cardano/DbSync/Config/Types.hs

@@ -470,6 +470,7 @@ optionsToList SyncInsertOptions {..} =
     , toJsonIfSet "offchain_pool_data" sioOffchainPoolData
     , toJsonIfSet "pool_stats" sioPoolStats
     , toJsonIfSet "json_type" sioJsonType
+    , toJsonIfSet "remove_jsonb_from_schema" sioRemoveJsonbFromSchema
     ]
 
 toJsonIfSet :: ToJSON a => Text -> a -> Maybe Pair
@@ -562,8 +563,8 @@ instance ToJSON TxOutConfig where
 instance FromJSON TxOutConfig where
   parseJSON = Aeson.withObject "tx_out" $ \obj -> do
     val <- obj .: "value"
-    useAddress' <- obj .: "use_address_table" .!= UseTxOutAddress False
     forceTxIn' <- obj .:? "force_tx_in" .!= ForceTxIn False
+    useAddress' <- obj .:? "use_address_table" .!= UseTxOutAddress False
 
     case val :: Text of
       "enable" -> pure (TxOutEnable useAddress')

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Query.hs

@@ -12,7 +12,7 @@ module Cardano.DbSync.Era.Shelley.Query (
 ) where
 
 import Cardano.Db
-import qualified Cardano.DbSync.Api as Db
+import Cardano.DbSync.Api (getTxOutTableType)
 import Cardano.DbSync.Api.Types (SyncEnv)
 import qualified Cardano.DbSync.Era.Shelley.Generic as Generic
 import Cardano.DbSync.Util
@@ -28,16 +28,16 @@ resolveStakeAddress addr = queryStakeAddress addr renderByteArray
 
 resolveInputTxOutId :: MonadIO m => SyncEnv -> Generic.TxIn -> ReaderT SqlBackend m (Either LookupFail (TxId, TxOutIdW))
 resolveInputTxOutId syncEnv txIn =
-  queryTxOutId (Db.getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
+  queryTxOutId (getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
 
 resolveInputValue :: MonadIO m => SyncEnv -> Generic.TxIn -> ReaderT SqlBackend m (Either LookupFail (TxId, DbLovelace))
 resolveInputValue syncEnv txIn =
-  queryTxOutValue (Db.getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
+  queryTxOutValue (getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
 
 resolveInputTxOutIdValue :: MonadIO m => SyncEnv -> Generic.TxIn -> ReaderT SqlBackend m (Either LookupFail (TxId, TxOutIdW, DbLovelace))
 resolveInputTxOutIdValue syncEnv txIn =
-  queryTxOutIdValue (Db.getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
+  queryTxOutIdValue (getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
 
 queryResolveInputCredentials :: MonadIO m => SyncEnv -> Generic.TxIn -> ReaderT SqlBackend m (Either LookupFail (Maybe ByteString, Bool))
 queryResolveInputCredentials syncEnv txIn = do
-  queryTxOutCredentials (Db.getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))
+  queryTxOutCredentials (getTxOutTableType syncEnv) (Generic.toTxHash txIn, fromIntegral (Generic.txInIndex txIn))

cardano-db-sync/src/Cardano/DbSync/Fix/ConsumedBy.hs

@@ -7,6 +7,8 @@ import qualified Cardano.Chain.Block as Byron hiding (blockHash)
 import qualified Cardano.Chain.UTxO as Byron
 import qualified Cardano.Crypto as Crypto (serializeCborHash)
 import qualified Cardano.Db as DB
+import Cardano.DbSync.Api (getTrace, getTxOutTableType)
+import Cardano.DbSync.Api.Types (SyncEnv)
 import Cardano.DbSync.Era.Byron.Insert
 import Cardano.DbSync.Era.Byron.Util (blockPayload, unTxHash)
 import Cardano.DbSync.Era.Util
@@ -16,8 +18,6 @@ import Cardano.Prelude hiding (length, (.))
 import Database.Persist.SqlBackend.Internal
 import Ouroboros.Consensus.Byron.Ledger (ByronBlock (..))
 import Ouroboros.Consensus.Cardano.Block (HardForkBlock (..))
-import Cardano.DbSync.Api.Types (SyncEnv)
-import Cardano.DbSync.Api (getTxOutTableType, getTrace)
 
 type FixEntry = (DB.TxOutIdW, DB.TxId)
 

cardano-db-sync/test/Cardano/DbSync/Config/TypesTest.hs

@@ -32,7 +32,6 @@ tests =
 prop_syncInsertConfigFromJSON :: Property
 prop_syncInsertConfigFromJSON = property $ do
   json <- forAll genDefaultJson
-
   Aeson.fromJSON json === Aeson.Success (def :: SyncInsertConfig)
 
 prop_syncInsertConfigRoundtrip :: Property
@@ -118,7 +117,9 @@ genDefaultJson =
     [ [aesonQQ|
         {
           "tx_out": {
-            "value": "enable"
+            "value": "enable",
+            "use_address_table": false,
+            "force_tx_in": false
           },
           "ledger": "enable",
           "shelley": {
@@ -148,7 +149,8 @@ genDefaultJson =
     , [aesonQQ|
         {
           "tx_out": {
-            "value": "enable"
+            "value": "enable",
+            "use_address_table": false
           },
           "ledger": "enable",
           "shelley": {

cardano-db-tool/app/cardano-db-tool.hs

@@ -64,7 +64,7 @@ runCommand cmd =
         void $
           runMigrations pgConfig False mdir mldir Fix
     CmdTxOutMigration txOutTableType -> do
-      runWithConnectionNoLogging PGPassDefaultEnv $ migrateTxOut Nothing txOutTableType
+      runWithConnectionNoLogging PGPassDefaultEnv $ migrateTxOutTests txOutTableType
     CmdUtxoSetAtBlock blkid txOutAddressType -> utxoSetAtSlot txOutAddressType blkid
     CmdPrepareSnapshot pargs -> runPrepareSnapshot pargs
     CmdValidateDb txOutAddressType -> runDbValidation txOutAddressType

cardano-db-tool/src/Cardano/DbTool/Validate/BlockTxs.hs

@@ -79,7 +79,7 @@ validateBlockCount (blockNo, txCountExpected) = do
       then Right ()
       else Left $ ValidateError blockNo txCountActual txCountExpected
 
--- This queries by BlockNo, the one in Cardano.Db.Operations.Core.Query queries by BlockId.
+-- This queries by BlockNo, the one in Cardano.Db.Operations.Query queries by BlockId.
 queryBlockTxCount :: MonadIO m => Word64 -> ReaderT SqlBackend m Word64
 queryBlockTxCount blockNo = do
   res <- select $ do

cardano-db/app/gen-schema-docs.hs

@@ -60,7 +60,7 @@ docHeader branchName =
             , "** which may not accurately reflect the version number)"
             ]
     , "\n"
-    , "**Note:** This file is auto-generated from the documentation in cardano-db/src/Cardano/Db/Schema.hs\
+    , "**Note:** This file is auto-generated from the documentation in cardano-db/src/Cardano/Db/Schema/BaseSchema.hs\
       \ by the command `cabal run -- gen-schema-docs doc/schema.md`. This document should only be updated\
       \ during the release process and updated on the release branch."
     , "\n"

cardano-db/cardano-db.cabal

@@ -40,18 +40,18 @@ library
                         Cardano.Db.Migration
                         Cardano.Db.Migration.Haskell
                         Cardano.Db.Migration.Version
-                        Cardano.Db.Operations.Core.AlterTable
-                        Cardano.Db.Operations.Core.Delete
-                        Cardano.Db.Operations.Core.Insert
-                        Cardano.Db.Operations.Core.MinId
-                        Cardano.Db.Operations.Core.Query
-                        Cardano.Db.Operations.Core.QueryHelper
+                        Cardano.Db.Operations.AlterTable
+                        Cardano.Db.Operations.Delete
+                        Cardano.Db.Operations.Insert
+                        Cardano.Db.Operations.Other.MinId
+                        Cardano.Db.Operations.Query
+                        Cardano.Db.Operations.QueryHelper
                         Cardano.Db.Operations.Types
-                        Cardano.Db.Operations.Variant.ConsumedTxOut
-                        Cardano.Db.Operations.Variant.JsonbQuery
-                        Cardano.Db.Operations.Variant.TxOutDelete
-                        Cardano.Db.Operations.Variant.TxOutInsert
-                        Cardano.Db.Operations.Variant.TxOutQuery
+                        Cardano.Db.Operations.Other.ConsumedTxOut
+                        Cardano.Db.Operations.Other.JsonbQuery
+                        Cardano.Db.Operations.TxOut.TxOutDelete
+                        Cardano.Db.Operations.TxOut.TxOutInsert
+                        Cardano.Db.Operations.TxOut.TxOutQuery
                         Cardano.Db.PGConfig
                         Cardano.Db.Run
                         Cardano.Db.Schema.BaseSchema

cardano-db/src/Cardano/Db.hs

@@ -6,34 +6,24 @@ module Cardano.Db (
   Tx (..),
   TxIn (..),
   gitRev,
-  -- CTX.migrateTxOut,
-  -- CTX.runExtraMigrations,
-  -- CTX.queryTxConsumedColumnExists,
-  -- CTX.queryTxOutConsumedNullCount,
-  -- CTX.queryTxOutConsumedCount,
-  -- CTX.querySetNullTxOut,
 ) where
 
 import Cardano.Db.Error as X
 import Cardano.Db.Git.Version (gitRev)
 import Cardano.Db.Migration as X
 import Cardano.Db.Migration.Version as X
-import Cardano.Db.Operations.Core.AlterTable as X
-import Cardano.Db.Operations.Core.Delete as X
-import Cardano.Db.Operations.Core.Insert as X
-import Cardano.Db.Operations.Core.MinId as X
-import Cardano.Db.Operations.Core.Query as X
-import Cardano.Db.Operations.Core.QueryHelper as X
+import Cardano.Db.Operations.AlterTable as X
+import Cardano.Db.Operations.Delete as X
+import Cardano.Db.Operations.Insert as X
+import Cardano.Db.Operations.Other.ConsumedTxOut as X
+import Cardano.Db.Operations.Other.JsonbQuery as X
+import Cardano.Db.Operations.Other.MinId as X
+import Cardano.Db.Operations.Query as X
+import Cardano.Db.Operations.QueryHelper as X
+import Cardano.Db.Operations.TxOut.TxOutDelete as X
+import Cardano.Db.Operations.TxOut.TxOutInsert as X
+import Cardano.Db.Operations.TxOut.TxOutQuery as X
 import Cardano.Db.Operations.Types as X
-
--- import qualified Cardano.Db.Operations.Variant.ConsumedTxOut as CTX
-import Cardano.Db.Operations.Variant.ConsumedTxOut as X
-
--- (migrateTxOut, queryTxConsumedColumnExists, queryTxOutConsumedCount, queryTxOutConsumedNullCount, runExtraMigrations, querySetNullTxOut)
-import Cardano.Db.Operations.Variant.JsonbQuery as X
-import Cardano.Db.Operations.Variant.TxOutDelete as X
-import Cardano.Db.Operations.Variant.TxOutInsert as X
-import Cardano.Db.Operations.Variant.TxOutQuery as X
 import Cardano.Db.PGConfig as X
 import Cardano.Db.Run as X
 import Cardano.Db.Schema.BaseSchema as X

cardano-db/src/Cardano/Db/Migration.hs

@@ -28,10 +28,11 @@ import Cardano.BM.Trace (Trace)
 import Cardano.Crypto.Hash (Blake2b_256, ByteString, Hash, hashToStringAsHex, hashWith)
 import Cardano.Db.Migration.Haskell
 import Cardano.Db.Migration.Version
-import Cardano.Db.Operations.Core.Query
+import Cardano.Db.Operations.Query
 import Cardano.Db.PGConfig
 import Cardano.Db.Run
 import Cardano.Db.Schema.BaseSchema
+import Cardano.Db.Schema.Core.TxOut (migrateCoreTxOutCardanoDb)
 import Cardano.Prelude (Typeable, textShow)
 import Control.Exception (Exception, SomeException, handle)
 import Control.Monad.Extra
@@ -238,7 +239,10 @@ createMigration source (MigrationDir migdir) = do
     create :: ReaderT SqlBackend (NoLoggingT IO) (Maybe (MigrationVersion, Text))
     create = do
       ver <- getSchemaVersion
-      statements <- getMigration migrateBaseCardanoDb
+      -- here is the place to combine any "core" schemas to the base schema
+      statementsBase <- getMigration migrateBaseCardanoDb
+      statementsTxOut <- getMigration migrateCoreTxOutCardanoDb
+      let statements = statementsBase <> statementsTxOut
       if null statements
         then pure Nothing
         else do