Merge remote-tracking branch 'upstream/master'

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Alonzo/Reward.hs

@@ -52,7 +52,7 @@ import Test.Cardano.Db.Mock.UnifiedApi (
   withAlonzoFindLeaderAndSubmit,
   withAlonzoFindLeaderAndSubmitTx,
  )
-import Test.Cardano.Db.Mock.Validate (assertBlockNoBackoff, assertRewardCount, assertRewardCounts)
+import Test.Cardano.Db.Mock.Validate (assertBlockNoBackoff, assertInstantRewardCount, assertRewardCount, assertRewardCounts)
 import Test.Tasty.HUnit (Assertion)
 
 simpleRewards :: IOManager -> [(Text, Text)] -> Assertion
@@ -481,6 +481,6 @@ singleMIRCertMultiOut =
     b <- fillUntilNextEpoch interpreter mockServer
 
     assertBlockNoBackoff dbSync (2 + length a + length b)
-    assertRewardCount dbSync 4
+    assertInstantRewardCount dbSync 4
   where
     testLabel = "singleMIRCertMultiOut-alonzo"

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage/CommandLineArg/ForceIndex.hs

@@ -19,7 +19,7 @@ checkForceIndexesArg =
   withCustomConfig commandLineForceIndexArgs babbageConfigDir testLabel $ \_ _ dbSyncEnv -> do
     startDBSync dbSyncEnv
     threadDelay 3_000_000
-    assertEqQuery dbSyncEnv DB.queryPgIndexesCount 165 "there wasn't the correct number of indexes"
+    assertEqQuery dbSyncEnv DB.queryPgIndexesCount 162 "there wasn't the correct number of indexes"
   where
     testLabel = "CLAcheckForceIndexesArg"
     commandLineForceIndexArgs =
@@ -32,7 +32,7 @@ checkNoForceIndexesArg =
   withCustomConfigAndDropDB commandLineNoForceIndexArgs babbageConfigDir testLabel $ \_ _ dbSyncEnv -> do
     startDBSync dbSyncEnv
     threadDelay 3_000_000
-    assertEqQuery dbSyncEnv DB.queryPgIndexesCount 100 "there wasn't the correct number of indexes"
+    assertEqQuery dbSyncEnv DB.queryPgIndexesCount 97 "there wasn't the correct number of indexes"
   where
     testLabel = "CLAcheckNoForceIndexesArg"
     commandLineNoForceIndexArgs =

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Babbage/Reward.hs

@@ -51,6 +51,7 @@ import Test.Cardano.Db.Mock.UnifiedApi (
 import Test.Cardano.Db.Mock.Validate (
   assertBlockNoBackoff,
   assertCurrentEpoch,
+  assertInstantRewardCount,
   assertRewardCount,
   assertRewardCounts,
  )
@@ -566,6 +567,6 @@ singleMIRCertMultiOut =
     b <- fillUntilNextEpoch interpreter mockServer
 
     assertBlockNoBackoff dbSync (2 + length a + length b)
-    assertRewardCount dbSync 4
+    assertInstantRewardCount dbSync 4
   where
     testLabel = "singleMIRCertMultiOut"

cardano-chain-gen/test/Test/Cardano/Db/Mock/Unit/Conway/CommandLineArg/ForceIndex.hs

@@ -17,7 +17,7 @@ checkForceIndexesArg =
     startDBSync dbSync
 
     -- Verify number of DB indexes
-    assertEqQuery dbSync DB.queryPgIndexesCount 165 "unexpected number of indexes"
+    assertEqQuery dbSync DB.queryPgIndexesCount 162 "unexpected number of indexes"
   where
     cliArgs = initCommandLineArgs {claForceIndexes = True}
     testLabel = "conwayCLACheckForceIndexesArg"
@@ -28,7 +28,7 @@ checkNoForceIndexesArg =
     startDBSync dbSync
 
     -- Verify number of DB indexes
-    assertEqQuery dbSync DB.queryPgIndexesCount 100 "unexpected number of indexes"
+    assertEqQuery dbSync DB.queryPgIndexesCount 97 "unexpected number of indexes"
   where
     cliArgs = initCommandLineArgs {claForceIndexes = False}
     testLabel = "conwayCLACheckNoForceIndexesArg"

cardano-chain-gen/test/Test/Cardano/Db/Mock/Validate.hs

@@ -14,6 +14,7 @@ module Test.Cardano.Db.Mock.Validate (
   assertTxInCount,
   assertUnspentTx,
   assertRewardCount,
+  assertInstantRewardCount,
   assertBlockNoBackoff,
   assertBlockNoBackoffTimes,
   assertEqQuery,
@@ -42,6 +43,7 @@ module Test.Cardano.Db.Mock.Validate (
 
 import Cardano.Db
 import qualified Cardano.Db as DB
+import qualified Cardano.DbSync.Era.Shelley.Generic as Generic
 import Cardano.DbSync.Era.Shelley.Generic.Util
 import qualified Cardano.Ledger.Address as Ledger
 import Cardano.Ledger.BaseTypes
@@ -78,7 +80,7 @@ import Database.Esqueleto.Legacy (
   (==.),
   (^.),
  )
-import Database.Persist.Sql (Entity, SqlBackend, entityVal)
+import Database.Persist.Sql (Entity, SqlBackend)
 import Database.PostgreSQL.Simple (SqlError (..))
 import Ouroboros.Consensus.Cardano.Block
 import Ouroboros.Consensus.Shelley.Ledger (ShelleyBlock)
@@ -111,6 +113,10 @@ assertRewardCount :: DBSyncEnv -> Word64 -> IO ()
 assertRewardCount env n =
   assertEqBackoff env queryRewardCount n defaultDelays "Unexpected rewards count"
 
+assertInstantRewardCount :: DBSyncEnv -> Word64 -> IO ()
+assertInstantRewardCount env n =
+  assertEqBackoff env queryInstantRewardCount n defaultDelays "Unexpected instant rewards count"
+
 assertBlockNoBackoff :: DBSyncEnv -> Int -> IO ()
 assertBlockNoBackoff = assertBlockNoBackoffTimes defaultDelays
 
@@ -197,8 +203,8 @@ assertAddrValues ::
   IO ()
 assertAddrValues env ix expected sta = do
   addr <- assertRight $ resolveAddress ix sta
-  let addrBs = Ledger.serialiseAddr addr
-      q = queryAddressOutputs addrBs
+  let address = Generic.renderAddress addr
+      q = queryAddressOutputs address
   assertEqBackoff env q expected defaultDelays "Unexpected Balance"
 
 assertRight :: Show err => Either err a -> IO a
@@ -241,7 +247,7 @@ assertRewardCounts env st filterAddr mEpoch expected = do
     expectedMap :: Map ByteString (Word64, Word64, Word64, Word64, Word64)
     expectedMap = Map.fromList $ fmap (first mkDBStakeAddress) expected
 
-    groupByAddress :: [(Reward, ByteString)] -> Map ByteString (Word64, Word64, Word64, Word64, Word64)
+    groupByAddress :: [(RewardSource, ByteString)] -> Map ByteString (Word64, Word64, Word64, Word64, Word64)
     groupByAddress rewards =
       let res = foldr updateMap Map.empty rewards
        in if filterAddr
@@ -254,39 +260,46 @@ assertRewardCounts env st filterAddr mEpoch expected = do
       Right cred -> Ledger.serialiseRewardAcnt $ Ledger.RewardAcnt Testnet cred
 
     updateAddrCounters ::
-      Reward ->
+      RewardSource ->
       Maybe (Word64, Word64, Word64, Word64, Word64) ->
       (Word64, Word64, Word64, Word64, Word64)
-    updateAddrCounters reward Nothing = updateCounters reward (0, 0, 0, 0, 0)
-    updateAddrCounters reward (Just cs) = updateCounters reward cs
+    updateAddrCounters rs Nothing = updateCounters rs (0, 0, 0, 0, 0)
+    updateAddrCounters rs (Just cs) = updateCounters rs cs
 
     updateCounters ::
-      Reward ->
+      RewardSource ->
       (Word64, Word64, Word64, Word64, Word64) ->
       (Word64, Word64, Word64, Word64, Word64)
-    updateCounters reward (a, b, c, d, e) = case rewardType reward of
+    updateCounters rs (a, b, c, d, e) = case rs of
       RwdLeader -> (a + 1, b, c, d, e)
       RwdMember -> (a, b + 1, c, d, e)
       RwdReserves -> (a, b, c + 1, d, e)
       RwdTreasury -> (a, b, c, d + 1, e)
       RwdDepositRefund -> (a, b, c, d, e + 1)
 
     updateMap ::
-      (Reward, ByteString) ->
+      (RewardSource, ByteString) ->
       Map ByteString (Word64, Word64, Word64, Word64, Word64) ->
       Map ByteString (Word64, Word64, Word64, Word64, Word64)
-    updateMap (rew, addr) = Map.alter (Just . updateAddrCounters rew) addr
+    updateMap (rs, addr) = Map.alter (Just . updateAddrCounters rs) addr
 
     filterEpoch rw = case mEpoch of
       Nothing -> val True
       Just e -> rw ^. RewardSpendableEpoch ==. val e
+    filterEpoch' rw = case mEpoch of
+      Nothing -> val True
+      Just e -> rw ^. InstantRewardSpendableEpoch ==. val e
 
     q = do
-      res <- select . from $ \(reward `InnerJoin` stake_addr) -> do
+      res1 <- select . from $ \(reward `InnerJoin` stake_addr) -> do
         on (reward ^. RewardAddrId ==. stake_addr ^. StakeAddressId)
         where_ (filterEpoch reward)
-        pure (reward, stake_addr ^. StakeAddressHashRaw)
-      pure $ fmap (bimap entityVal unValue) res
+        pure (reward ^. RewardType, stake_addr ^. StakeAddressHashRaw)
+      res2 <- select . from $ \(ireward `InnerJoin` stake_addr) -> do
+        on (ireward ^. InstantRewardAddrId ==. stake_addr ^. StakeAddressId)
+        where_ (filterEpoch' ireward)
+        pure (ireward ^. InstantRewardType, stake_addr ^. StakeAddressHashRaw)
+      pure $ fmap (bimap unValue unValue) (res1 <> res2)
 
 assertEpochStake :: DBSyncEnv -> Word64 -> IO ()
 assertEpochStake env expected =

cardano-db-sync/CHANGELOG.md

@@ -1,71 +1,6 @@
 # Revision history for cardano-db-sync
 
-## Sancho changelog
-`sancho-a-b-c` tags in db-sync are a series of incremental integration of conway feautures. These tags can follow the
-sanchonet https://book.world.dev.cardano.org/environments.html#sanchonet-testnet.
-Their enumeration has the following meaning:<br>
-a: schema changes<br>
-b: compatible with a respinned sanchonet or node<br>
-c: minor changes or fixes<br>
-
-It is advised to resync from genesis when using a new version a or b.
-
-Some useful links (adjust the numbers to the correct tag):
-
-- Changelog: https://github.com/IntersectMBO/cardano-db-sync/blob/sancho-2-0-0/cardano-db-sync/CHANGELOG.md#13200
-- Schema docs https://github.com/IntersectMBO/cardano-db-sync/blob/sancho-2-0-0/doc/schema.md
-- Postgres migrations https://github.com/IntersectMBO/cardano-db-sync/tree/sancho-2-0-0/schema
-
-In the schema docs, you can search for `13.2` or `Conway` for schema changes from the previous official release.
-
-### sancho-3.0.0
-- Renamed table `governance_action` to `gov_action_proposal`
-- Replaced `new_committee.quorum` to `new_committee.quorum_nominator` and `new_committee.quorum_denominator`
-- `always_abstain` and `always_no_confidence` `drep_hash` entries now have a null `raw` value and the unique key changed
-- `governance_action.description` now uses the json format
-- Added new offchain capabilites for voting anchor: the `off_chain_vote_data` and `off_chain_vote_fetch_error` are now populated
-- Added a constitution table
-
-### sancho-2.3.0
-- is compatible with node 8.7-pre
-- introduces flag `--only-gov` which allows db-sync to sync only the governance related data.
-- disable swagger in smash server temporarily
-
-### sancho-2.2.0
-- is compatible with node 8.6-pre
-- `governance_action.ratified_epoch` is now populated
-- `drep_distr.active_until` is now populated
-- Fixed typo `voting_procedure.comittee_voter` to `voting_procedure.committee_voter`
-- Added `committee_de_registration.voting_anchor_id` field
-
-### sancho-2.1.0
-- Fixed an issue with the size of Committee hash
-
-### sancho-2.0.1
-- Workaround an issue where delegation could target unregistered pools
-
-### sancho-2.0.0
-- is compatible with node 8.5-pre
-- adds the governance epoch param group in table `epoch_param` and their updates through governance proposal in
-`param_proposal`
-- adds a new field `governance_action.expiration` for the expected expiration of an action.
-- populates the `drep_distr` table and `governance.enacted_epoch` that used to remain empty.
-- fixes an issue with duplicated entries in `drep_hash` and `voting_anchor`.
-- merges drep registration, re-registration and de-registration on the same table `drep_registration` for easy use
-- the new field `drep_distr.active_until` is not populated yet.
-- change `new_committee` gov actions represantation.
-
-### sancho-1.1.0
-- is compatible with node-8.3-pre. There are no schema changes over sancho-1-0-0.
-
-### sancho-1.0.0
-The schema is quite close to the [initial design](https://github.com/IntersectMBO/cardano-db-sync/blob/conway-schema-design-13.2/doc/schema.md). You may find some differences: ie the `param_proposals` and `epoch_param` are not extended yet,
-Some tables/fields are created but are not populated yet: `anchor_offline_data`, `anchor_offline_fetch_error` , `drep_distr` , `governance_action.x_epoch` , `delegation_vote.redeemer_id`
-
-### sancho-0.0.0
-It's very early stage and is missing all Conway specific feautures and some Shelley features related to Rewards. Incremental tags will follow.
-
-## 13.2.0.0 - Unreleased
+## 13.2.0.0
 * Uses the cache for the computation of `epoch` table when following
 * `epoch_stake` is now computed earlier, before the epoch is reached
 * Exception handling and concurrency improved.
@@ -78,7 +13,12 @@ It's very early stage and is missing all Conway specific feautures and some Shel
 * References to `offline` for pool metadata are replaced by `offchain`
 * Added a new table `extra_migrations`
 * DBSync no longer does any query to the `tx_out` table during syncing
-* Added flags `--disable-in-out`, `--disable-shelley`, `--disable-gov`, `--only-gov`
+* Added flags `--disable-in-out`, `--disable-shelley`, `--disable-gov`, `--only-gov`, `--keep-tx-metadata`
+* Move MIR from reward to instant_reward. Done with a migration.
+* Remove redundant field `address_raw` of `tx_out` and `collateral_tx_out`
+* Partial Conway integration. You can read more in `doc/schema.md` (tagged as Conway). Also in files
+`schema/cardano-chain-gen/schema/migration-2-0032-20230815.sql` and `schema/cardano-chain-gen/schema/migration-2-0032-20230815.sql`
+for the raw sql migrations.
 
 ## 13.1.1.3
 * Adds compatibility with node-8.0.0 [#1403] and node-8.1.1 [#1455]

cardano-db-sync/src/Cardano/DbSync.hs

@@ -51,7 +51,6 @@ import Cardano.DbSync.Rollback (unsafeRollback)
 import Cardano.DbSync.Sync (runSyncNodeClient)
 import Cardano.DbSync.Tracing.ToObjectOrphans ()
 import Cardano.DbSync.Types
-import Cardano.DbSync.Util.Constraint (addRewardConstraintsIfNotExist)
 import Cardano.Prelude hiding (Nat, (%))
 import Cardano.Slotting.Slot (EpochNo (..))
 import Control.Concurrent.Async
@@ -180,7 +179,6 @@ runSyncNode metricsSetters trce iomgr dbConnString ranMigrations runMigrationFnc
                 syncNodeParams
                 ranMigrations
                 runMigrationFnc
-          liftIO $ runReaderT (addRewardConstraintsIfNotExist syncEnv trce) (envBackend syncEnv)
           liftIO $ runExtraMigrationsMaybe syncEnv
           unless (enpShouldUseLedger syncNodeParams) $ liftIO $ do
             logInfo trce "Migrating to a no ledger schema"

cardano-db-sync/src/Cardano/DbSync/Api.hs

@@ -376,10 +376,10 @@ mkSyncEnv trce backend connectionString syncOptions protoInfo nw nwMagic systemS
   bts <- getBootstrapInProgress trce (enpBootstrap syncNP) backend
   bootstrapVar <- newTVarIO bts
   -- Offline Pool + Anchor queues
-  opwq <- newTBQueueIO 100
-  oprq <- newTBQueueIO 100
-  oawq <- newTBQueueIO 100
-  oarq <- newTBQueueIO 100
+  opwq <- newTBQueueIO 1000
+  oprq <- newTBQueueIO 1000
+  oawq <- newTBQueueIO 1000
+  oarq <- newTBQueueIO 1000
   epochVar <- newTVarIO initEpochState
   epochSyncTime <- newTVarIO =<< getCurrentTime
   ledgerEnvType <-

cardano-db-sync/src/Cardano/DbSync/Default.hs

@@ -23,7 +23,7 @@ import Cardano.DbSync.Era.Cardano.Insert (insertEpochSyncTime)
 import Cardano.DbSync.Era.Shelley.Adjust (adjustEpochRewards)
 import qualified Cardano.DbSync.Era.Shelley.Generic as Generic
 import Cardano.DbSync.Era.Shelley.Insert (insertShelleyBlock, mkAdaPots)
-import Cardano.DbSync.Era.Shelley.Insert.Epoch (insertPoolDepositRefunds, insertRewards)
+import Cardano.DbSync.Era.Shelley.Insert.Epoch (insertInstantRewards, insertPoolDepositRefunds, insertRewards)
 import Cardano.DbSync.Era.Shelley.Validate (validateEpochRewards)
 import Cardano.DbSync.Error
 import Cardano.DbSync.Fix.EpochStake
@@ -293,7 +293,7 @@ insertLedgerEvents syncEnv currentEpochNo@(EpochNo curEpoch) =
         LedgerMirDist rwd -> do
           unless (Map.null rwd) $ do
             let rewards = Map.toList rwd
-            insertRewards syncEnv ntw (subFromCurrentEpoch 1) currentEpochNo cache rewards
+            insertInstantRewards ntw (subFromCurrentEpoch 1) currentEpochNo cache rewards
             liftIO . logInfo tracer $ "Inserted " <> show (length rewards) <> " Mir rewards"
         LedgerPoolReap en drs ->
           unless (Map.null $ Generic.unRewards drs) $ do

cardano-db-sync/src/Cardano/DbSync/Era/Byron/Genesis.hs

@@ -11,7 +11,6 @@ module Cardano.DbSync.Era.Byron.Genesis (
 ) where
 
 import Cardano.BM.Trace (Trace, logInfo)
-import qualified Cardano.Binary as Binary
 import qualified Cardano.Chain.Common as Byron
 import qualified Cardano.Chain.Genesis as Byron
 import qualified Cardano.Chain.UTxO as Byron
@@ -209,7 +208,6 @@ insertTxOuts hasConsumed disInOut blkId (address, value) = do
             { DB.txOutTxId = txId
             , DB.txOutIndex = 0
             , DB.txOutAddress = Text.decodeUtf8 $ Byron.addrToBase58 address
-            , DB.txOutAddressRaw = Binary.serialize' address
             , DB.txOutAddressHasScript = False
             , DB.txOutPaymentCred = Nothing
             , DB.txOutStakeAddressId = Nothing

cardano-db-sync/src/Cardano/DbSync/Era/Byron/Insert.hs

@@ -12,11 +12,6 @@ module Cardano.DbSync.Era.Byron.Insert (
 
 import Cardano.BM.Trace (Trace, logDebug, logInfo)
 import Cardano.Binary (serialize')
-import qualified Cardano.Binary as Binary
-
--- Import all 'cardano-ledger' functions and data types qualified so they do not
--- clash with the Cardano.Db functions and data types which are also imported
--- qualified.
 import qualified Cardano.Chain.Block as Byron hiding (blockHash)
 import qualified Cardano.Chain.Common as Byron
 import qualified Cardano.Chain.UTxO as Byron
@@ -327,7 +322,6 @@ insertTxOut _tracer hasConsumed bootStrap txId index txout =
       { DB.txOutTxId = txId
       , DB.txOutIndex = fromIntegral index
       , DB.txOutAddress = Text.decodeUtf8 $ Byron.addrToBase58 (Byron.txOutAddress txout)
-      , DB.txOutAddressRaw = Binary.serialize' (Byron.txOutAddress txout)
       , DB.txOutAddressHasScript = False
       , DB.txOutPaymentCred = Nothing -- Byron does not have a payment credential.
       , DB.txOutStakeAddressId = Nothing -- Byron does not have a stake address.

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Adjust.hs

@@ -22,7 +22,6 @@ import Cardano.Slotting.Slot (EpochNo (..))
 import Control.Monad.Trans.Control (MonadBaseControl)
 import qualified Data.Map.Strict as Map
 import qualified Data.Set as Set
-import qualified Data.Strict.Maybe as Strict
 import Database.Esqueleto.Experimental (
   SqlBackend,
   delete,
@@ -80,17 +79,15 @@ deleteReward ::
   ReaderT SqlBackend m ()
 deleteReward nw cache epochNo (cred, rwd) = do
   mAddrId <- queryStakeAddrWithCache cache DontCacheNew nw cred
-  eiPoolId <- case Generic.rewardPool rwd of
-    Strict.Nothing -> pure $ Left $ Db.DbLookupMessage "deleteReward.queryPoolKeyWithCache"
-    Strict.Just poolHash -> queryPoolKeyWithCache cache DontCacheNew poolHash
+  eiPoolId <- queryPoolKeyWithCache cache DontCacheNew (Generic.rewardPool rwd)
   case (mAddrId, eiPoolId) of
     (Right addrId, Right poolId) -> do
       delete $ do
         rwdDb <- from $ table @Db.Reward
         where_ (rwdDb ^. Db.RewardAddrId ==. val addrId)
         where_ (rwdDb ^. Db.RewardType ==. val (Generic.rewardSource rwd))
         where_ (rwdDb ^. Db.RewardSpendableEpoch ==. val (unEpochNo epochNo))
-        where_ (rwdDb ^. Db.RewardPoolId ==. val (Just poolId))
+        where_ (rwdDb ^. Db.RewardPoolId ==. val poolId)
     _ -> pure ()
 
 deleteOrphanedRewards :: MonadIO m => EpochNo -> [Db.StakeAddressId] -> ReaderT SqlBackend m ()