Skip to content

Commit

Permalink
1903 - extend logging
Browse files Browse the repository at this point in the history
  • Loading branch information
@Cmdv
Cmdv committed Nov 19, 2024
1 parent 5b131e9 commit e799ab3
Show file tree
Hide file tree
Showing 34 changed files with 857 additions and 523 deletions.
1 change: 1 addition & 0 deletions cardano-db-sync/cardano-db-sync.cabal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ library
Cardano.DbSync.Util.Bech32
Cardano.DbSync.Util.Cbor
Cardano.DbSync.Util.Constraint
Cardano.DbSync.Util.Logging

Paths_cardano_db_sync

Expand Down
84 changes: 51 additions & 33 deletions cardano-db-sync/src/Cardano/DbSync.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ module Cardano.DbSync (
extractSyncOptions,
) where

import Cardano.BM.Trace (Trace, logError, logInfo, logWarning)
import Cardano.BM.Trace (Trace)
import qualified Cardano.Crypto as Crypto
import qualified Cardano.Db as DB
import qualified Cardano.Db as Db
Expand All @@ -44,6 +44,7 @@ import Cardano.DbSync.Sync (runSyncNodeClient)
import Cardano.DbSync.Tracing.ToObjectOrphans ()
import Cardano.DbSync.Types
import Cardano.DbSync.Util.Constraint (queryIsJsonbInSchema)
import Cardano.DbSync.Util.Logging (LogContext (..), initLogCtx, logErrorCtx, logInfoCtx, logWarningCtx)
import Cardano.Prelude hiding (Nat, (%))
import Cardano.Slotting.Slot (EpochNo (..))
import Control.Concurrent.Async
Expand Down Expand Up @@ -79,41 +80,49 @@ runDbSync ::
Bool ->
IO ()
runDbSync metricsSetters knownMigrations iomgr trce params syncNodeConfigFromFile abortOnPanic = do
logInfo trce $ textShow syncOpts
let logCtx = initLogCtx "runDbSync" "Cardano.DbSync"
logInfoCtx trce $ logCtx {lcMessage = "Current sync options: " <> textShow syncOpts}

-- Read the PG connection info
pgConfig <- runOrThrowIO (Db.readPGPass $ enpPGPassSource params)

mErrors <- liftIO $ Db.validateMigrations dbMigrationDir knownMigrations
whenJust mErrors $ \(unknown, stage4orNewStage3) ->
if stage4orNewStage3
then logWarning trce $ Db.renderMigrationValidateError unknown
then logWarningCtx trce $ logCtx {lcMessage = Db.renderMigrationValidateError unknown}
else do
let msg = Db.renderMigrationValidateError unknown
logError trce msg
logErrorCtx trce $ logCtx {lcMessage = msg}
throwIO unknown

logInfo trce "Schema migration files validated"
logInfoCtx trce $ logCtx {lcMessage = "Schema migration files validated"}

let runMigration mode = do
msg <- Db.getMaintenancePsqlConf pgConfig
logInfo trce $ "Running database migrations in mode " <> textShow mode
logInfo trce msg
when (mode `elem` [Db.Indexes, Db.Full]) $ logWarning trce indexesMsg
logInfoCtx trce $ logCtx {lcMessage = "Running database migrations in mode " <> textShow mode}
logInfoCtx trce $ logCtx {lcMessage = msg}
when (mode `elem` [Db.Indexes, Db.Full]) $ logWarningCtx trce $ logCtx {lcMessage = indexesMsg}
Db.runMigrations pgConfig True dbMigrationDir (Just $ Db.LogFileDir "/tmp") mode (txOutConfigToTableType txOutConfig)
(ranMigrations, unofficial) <- if enpForceIndexes params then runMigration Db.Full else runMigration Db.Initial
unless (null unofficial) $
logWarning trce $
"Unofficial migration scripts found: "
<> textShow unofficial
logWarningCtx trce $
logCtx {lcMessage = "Unofficial migration scripts found: " <> textShow unofficial}

if ranMigrations
then logInfo trce "All migrations were executed"
else logInfo trce "Some migrations were not executed. They need to run when syncing has started."
logInfoCtx trce $
logCtx
{ lcMessage =
if ranMigrations
then "All migrations were executed"
else "Some migrations were not executed. They need to run when syncing has started."
}

if enpForceIndexes params
then logInfo trce "All user indexes were created"
else logInfo trce "New user indexes were not created. They may be created later if necessary."
logInfoCtx trce $
logCtx
{ lcMessage =
if enpForceIndexes params
then "All user indexes were created"
else "New user indexes were not created. They may be created later if necessary."
}

let connectionString = Db.toConnectionString pgConfig

Expand Down Expand Up @@ -162,12 +171,16 @@ runSyncNode ::
SyncOptions ->
IO ()
runSyncNode metricsSetters trce iomgr dbConnString ranMigrations runMigrationFnc syncNodeConfigFromFile syncNodeParams syncOptions = do
let logCtx = initLogCtx "runSyncNode" "Cardano.DbSync"
whenJust maybeLedgerDir $
\enpLedgerStateDir -> do
createDirectoryIfMissing True (unLedgerStateDir enpLedgerStateDir)
logInfo trce $ "Using byron genesis file from: " <> (show . unGenesisFile $ dncByronGenesisFile syncNodeConfigFromFile)
logInfo trce $ "Using shelley genesis file from: " <> (show . unGenesisFile $ dncShelleyGenesisFile syncNodeConfigFromFile)
logInfo trce $ "Using alonzo genesis file from: " <> (show . unGenesisFile $ dncAlonzoGenesisFile syncNodeConfigFromFile)
logInfoCtx trce $
logCtx {lcMessage = "Using byron genesis file from: " <> (show . unGenesisFile $ dncByronGenesisFile syncNodeConfigFromFile)}
logInfoCtx trce $
logCtx {lcMessage = "Using shelley genesis file from: " <> (show . unGenesisFile $ dncShelleyGenesisFile syncNodeConfigFromFile)}
logInfoCtx trce $
logCtx {lcMessage = "Using alonzo genesis file from: " <> (show . unGenesisFile $ dncAlonzoGenesisFile syncNodeConfigFromFile)}

let useLedger = shouldUseLedger (sioLedger $ dncInsertOptions syncNodeConfigFromFile)

Expand All @@ -193,16 +206,16 @@ runSyncNode metricsSetters trce iomgr dbConnString ranMigrations runMigrationFnc

-- Warn the user that jsonb datatypes are being removed from the database schema.
when (isJsonbInSchema && removeJsonbFromSchemaConfig) $ do
liftIO $ logWarning trce "Removing jsonb datatypes from the database. This can take time."
liftIO $ logWarningCtx trce $ logCtx {lcMessage = "Removing jsonb datatypes from the database. This can take time."}
liftIO $ runRemoveJsonbFromSchema syncEnv

-- Warn the user that jsonb datatypes are being added to the database schema.
when (not isJsonbInSchema && not removeJsonbFromSchemaConfig) $ do
liftIO $ logWarning trce "Adding jsonb datatypes back to the database. This can take time."
liftIO $ logWarningCtx trce $ logCtx {lcMessage = "Adding jsonb datatypes back to the database. This can take time."}
liftIO $ runAddJsonbToSchema syncEnv
liftIO $ runExtraMigrationsMaybe syncEnv
unless useLedger $ liftIO $ do
logInfo trce "Migrating to a no ledger schema"
logInfoCtx trce $ logCtx {lcMessage = "Migrating to a no ledger schema"}
Db.noLedgerMigrations backend trce
insertValidateGenesisDist syncEnv (dncNetworkName syncNodeConfigFromFile) genCfg (useShelleyInit syncNodeConfigFromFile)

Expand All @@ -227,13 +240,17 @@ runSyncNode metricsSetters trce iomgr dbConnString ranMigrations runMigrationFnc
maybeLedgerDir = enpMaybeLedgerStateDir syncNodeParams

logProtocolMagicId :: Trace IO Text -> Crypto.ProtocolMagicId -> ExceptT SyncNodeError IO ()
logProtocolMagicId tracer pm =
logProtocolMagicId tracer pm = do
let logCtx = initLogCtx "logProtocolMagicId" "Cardano.DbSync"
liftIO
. logInfo tracer
$ mconcat
[ "NetworkMagic: "
, textShow (Crypto.unProtocolMagicId pm)
]
. logInfoCtx tracer
$ logCtx
{ lcMessage =
mconcat
[ "NetworkMagic: "
, textShow (Crypto.unProtocolMagicId pm)
]
}

-- -------------------------------------------------------------------------------------------------

Expand Down Expand Up @@ -299,10 +316,11 @@ extractSyncOptions snp aop snc =

startupReport :: Trace IO Text -> Bool -> SyncNodeParams -> IO ()
startupReport trce aop params = do
logInfo trce $ mconcat ["Version number: ", Text.pack (showVersion version)]
logInfo trce $ mconcat ["Git hash: ", Db.gitRev]
logInfo trce $ mconcat ["Enviroment variable DbSyncAbortOnPanic: ", textShow aop]
logInfo trce $ textShow params
let logCtx = initLogCtx "runSyncNode" "Cardano.DbSync"
logInfoCtx trce $ logCtx {lcMessage = mconcat ["Version number: ", Text.pack (showVersion version)]}
logInfoCtx trce $ logCtx {lcMessage = mconcat ["Git hash: ", Db.gitRev]}
logInfoCtx trce $ logCtx {lcMessage = mconcat ["Enviroment variable DbSyncAbortOnPanic: ", textShow aop]}
logInfoCtx trce $ logCtx {lcMessage = textShow params}

txOutConfigToTableType :: TxOutConfig -> DB.TxOutTableType
txOutConfigToTableType config = case config of
Expand Down
89 changes: 56 additions & 33 deletions cardano-db-sync/src/Cardano/DbSync/Api.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ module Cardano.DbSync.Api (
convertToPoint,
) where

import Cardano.BM.Trace (Trace, logInfo, logWarning)
import Cardano.BM.Trace (Trace)
import qualified Cardano.Chain.Genesis as Byron
import Cardano.Crypto.ProtocolMagic (ProtocolMagicId (..))
import qualified Cardano.Db as DB
Expand All @@ -73,6 +73,7 @@ import Cardano.DbSync.LocalStateQuery
import Cardano.DbSync.Types
import Cardano.DbSync.Util
import Cardano.DbSync.Util.Constraint (dbConstraintNamesExists)
import Cardano.DbSync.Util.Logging (LogContext (..), initLogCtx, logInfoCtx, logWarningCtx)
import qualified Cardano.Ledger.BaseTypes as Ledger
import qualified Cardano.Ledger.Shelley.Genesis as Shelley
import Cardano.Prelude
Expand Down Expand Up @@ -101,7 +102,8 @@ import qualified Ouroboros.Network.Point as Point

setConsistentLevel :: SyncEnv -> ConsistentLevel -> IO ()
setConsistentLevel env cst = do
logInfo (getTrace env) $ "Setting ConsistencyLevel to " <> textShow cst
let logCtx = initLogCtx "setConsistentLevel" "Cardano.DbSync.Api"
logInfoCtx (getTrace env) $ logCtx {lcMessage = "Setting ConsistencyLevel to " <> textShow cst}
atomically $ writeTVar (envConsistentLevel env) cst

getConsistentLevel :: SyncEnv -> IO ConsistentLevel
Expand Down Expand Up @@ -158,10 +160,11 @@ getRanIndexes env = do

runIndexMigrations :: SyncEnv -> IO ()
runIndexMigrations env = do
let logCtx = initLogCtx "runIndexMigrations" "Cardano.DbSync.Api"
haveRan <- readTVarIO $ envIndexes env
unless haveRan $ do
envRunDelayedMigration env DB.Indexes
logInfo (getTrace env) "Indexes were created"
logInfoCtx (getTrace env) $ logCtx {lcMessage = "Indexes were created"}
atomically $ writeTVar (envIndexes env) True

initPruneConsumeMigration :: Bool -> Bool -> Bool -> Bool -> DB.PruneConsumeMigration
Expand All @@ -178,8 +181,9 @@ getPruneConsume = soptPruneConsumeMigration . envOptions
runExtraMigrationsMaybe :: SyncEnv -> IO ()
runExtraMigrationsMaybe syncEnv = do
let pcm = getPruneConsume syncEnv
logCtx = initLogCtx "runExtraMigrationsMaybe" "Cardano.DbSync.Api"
txOutTableType = getTxOutTableType syncEnv
logInfo (getTrace syncEnv) $ "runExtraMigrationsMaybe: " <> textShow pcm
logInfoCtx (getTrace syncEnv) $ logCtx {lcMessage = "runExtraMigrationsMaybe: " <> textShow pcm}
DB.runDbIohkNoLogging (envBackend syncEnv) $
DB.runExtraMigrations
(getTrace syncEnv)
Expand Down Expand Up @@ -306,12 +310,23 @@ getDbTipBlockNo env = do
mblk <- getDbLatestBlockInfo (envBackend env)
pure $ maybe Point.Origin (Point.At . bBlockNo) mblk

logDbState :: SyncEnv -> IO ()
logDbState env = do
getCurrentTipBlockNo :: SyncEnv -> IO (WithOrigin BlockNo)
getCurrentTipBlockNo env = do
maybeTip <- getDbLatestBlockInfo (envBackend env)
case maybeTip of
Just tip -> pure $ At (bBlockNo tip)
Nothing -> pure Origin

logDbState :: SyncEnv -> LogContext -> IO ()
logDbState env logCtx = do
mblk <- getDbLatestBlockInfo (envBackend env)
case mblk of
Nothing -> logInfo tracer "Database is empty"
Just tip -> logInfo tracer $ mconcat ["Database tip is at ", showTip tip]
Nothing ->
logInfoCtx tracer $
logCtx {lcMessage = "Database is empty"}
Just tip ->
logInfoCtx tracer $
logCtx {lcMessage = mconcat ["Database tip is at ", showTip tip]}
where
showTip :: TipInfo -> Text
showTip tipInfo =
Expand All @@ -325,13 +340,6 @@ logDbState env = do
tracer :: Trace IO Text
tracer = getTrace env

getCurrentTipBlockNo :: SyncEnv -> IO (WithOrigin BlockNo)
getCurrentTipBlockNo env = do
maybeTip <- getDbLatestBlockInfo (envBackend env)
case maybeTip of
Just tip -> pure $ At (bBlockNo tip)
Nothing -> pure Origin

mkSyncEnv ::
Trace IO Text ->
SqlBackend ->
Expand All @@ -347,6 +355,7 @@ mkSyncEnv ::
RunMigration ->
IO SyncEnv
mkSyncEnv trce backend connectionString syncOptions protoInfo nw nwMagic systemStart syncNodeConfigFromFile syncNP ranMigrations runMigrationFnc = do
let logCtx = initLogCtx "mkSyncEnv" "Cardano.DbSync.Api"
dbCNamesVar <- newTVarIO =<< dbConstraintNamesExists backend
cache <-
if soptCache syncOptions
Expand Down Expand Up @@ -384,9 +393,13 @@ mkSyncEnv trce backend connectionString syncOptions protoInfo nw nwMagic systemS
syncOptions
(Nothing, False) -> NoLedger <$> mkNoLedgerEnv trce protoInfo nw systemStart
(Just _, False) -> do
logWarning trce $
"Disabling the ledger doesn't require having a --state-dir."
<> " For more details view https://github.com/IntersectMBO/cardano-db-sync/blob/master/doc/configuration.md#ledger"
logWarningCtx trce $
logCtx
{ lcMessage =
"Disabling the ledger doesn't require having a --state-dir."
<> " For more details view "
<> " https://github.com/IntersectMBO/cardano-db-sync/blob/master/doc/configuration.md#ledger"
}
NoLedger <$> mkNoLedgerEnv trce protoInfo nw systemStart
-- This won't ever call because we error out this combination at parse time
(Nothing, True) -> NoLedger <$> mkNoLedgerEnv trce protoInfo nw systemStart
Expand Down Expand Up @@ -534,6 +547,7 @@ getBootstrapInProgress ::
SqlBackend ->
IO Bool
getBootstrapInProgress trce bootstrapFlag sqlBackend = do
let logCtx = initLogCtx "getBootstrapInProgress" "Cardano.DbSync.Api"
DB.runDbIohkNoLogging sqlBackend $ do
ems <- DB.queryAllExtraMigrations
let btsState = DB.bootstrapState ems
Expand All @@ -547,26 +561,35 @@ getBootstrapInProgress trce bootstrapFlag sqlBackend = do
liftIO $ DB.logAndThrowIO trce "Bootstrap flag not set, but still in progress"
(True, DB.BootstrapNotStarted) -> do
liftIO $
logInfo trce $
mconcat
[ "Syncing with bootstrap. "
, "This won't populate tx_out until the tip of the chain."
]
logInfoCtx trce $
logCtx
{ lcMessage =
mconcat
[ "Syncing with bootstrap. "
, "This won't populate tx_out until the tip of the chain."
]
}
DB.insertExtraMigration DB.BootstrapStarted
pure True
(True, DB.BootstrapInProgress) -> do
liftIO $
logInfo trce $
mconcat
[ "Syncing with bootstrap is in progress. "
, "This won't populate tx_out until the tip of the chain."
]
logInfoCtx trce $
logCtx
{ lcMessage =
mconcat
[ "Syncing with bootstrap is in progress. "
, "This won't populate tx_out until the tip of the chain."
]
}
pure True
(True, DB.BootstrapDone) -> do
liftIO $
logWarning trce $
mconcat
[ "Bootstrap flag is set, but it will be ignored, "
, "since bootstrap is already done."
]
logWarningCtx trce $
logCtx
{ lcMessage =
mconcat
[ "Bootstrap flag is set, but it will be ignored, "
, "since bootstrap is already done."
]
}
pure False
Loading

0 comments on commit e799ab3

Please sign in to comment.