From 7eea51a5ad7b3f15abd6e24a6a6ea36f9868b688 Mon Sep 17 00:00:00 2001 From: Dmitry <98899785+mdqst@users.noreply.github.com> Date: Tue, 12 Nov 2024 20:03:15 +0300 Subject: [PATCH 01/19] Typos Update README.md (#2032) "inquires" Error: Incorrect word usage. Correction: "inquiries" "IMPORTANT NODE" Error: Typo. Correction: "IMPORTANT NOTE" --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a71f122cfc..0e36cab7d1 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Read our [whitepapers](https://www.kadena.io/whitepapers): - [Chainweb: A Proof-of-Work Parallel-Chain Architecture for Massive Throughput](https://d31d887a-c1e0-47c2-aa51-c69f9f998b07.filesusr.com/ugd/86a16f_029c9991469e4565a7c334dd716345f4.pdf) - [Agent-based Simulations of Blockchain Protocols illustrated via Kadena's *Chainweb*](https://d31d887a-c1e0-47c2-aa51-c69f9f998b07.filesusr.com/ugd/86a16f_3b2d0c58179d4edd9df6df4d55d61dda.pdf) -For additional information, press, and development inquires, please refer to the Kadena [website](https://kadena.io) +For additional information, press, and development inquiries, please refer to the Kadena [website](https://kadena.io) ## Table of Contents @@ -103,7 +103,7 @@ At this point, you are ready to [run a Chainweb node](#configuring-running-and-m ## Building from Source -*IMPORTANT NODE: We recommend the use of officially released chainweb-node +*IMPORTANT NOTE: We recommend the use of officially released chainweb-node binaries or docker images, which can be found in the [release section of this repository](https://github.com/kadena-io/chainweb-node/releases). From e3447f2aaf5d1d1fc40348d6183d84d94400ef9c Mon Sep 17 00:00:00 2001 From: Lars Kuhtz Date: Sat, 2 Nov 2024 22:03:13 -0700 Subject: [PATCH 02/19] Revert "revert rosetta test changes based on hashable update" This reverts commit 9c1d9460d89703a3b3a041f13462f80c813ac004. --- test/unit/Chainweb/Test/Rosetta/RestAPI.hs | 45 ++++++++++++++++++---- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/test/unit/Chainweb/Test/Rosetta/RestAPI.hs b/test/unit/Chainweb/Test/Rosetta/RestAPI.hs index 1c6c46ffd1..f4a4bd8b58 100644 --- a/test/unit/Chainweb/Test/Rosetta/RestAPI.hs +++ b/test/unit/Chainweb/Test/Rosetta/RestAPI.hs @@ -217,20 +217,35 @@ blockTransactionTests tio envIo = step "send in block tx request" resp <- blockTransaction v cenv req - (fundtx,cred,deb,redeem,reward) <- + (fundtx,deb,cred,redeem,reward) <- case _transaction_operations $ _blockTransactionResp_transaction resp of [a,b,c,d,e] -> return (a,b,c,d,e) _ -> assertFailure "transfer should have resulted in 5 transactions" + -- The order in which operations are returned is flaky and may break. Use + -- the following to double check the order in case this test fails. + -- + -- print "fundtx: ----------------------" + -- print fundtx + -- print "cred: ----------------------" + -- print cred + -- print "deb: ----------------------" + -- print deb + -- print "redeem: ----------------------" + -- print redeem + -- print "reward: ----------------------" + -- print reward + -- print "----------------------" + step "validate initial gas buy at op index 0" validateOp 0 "FundTx" sender00ks Successful (negate defFundGas) fundtx - step "validate sender01 credit at op index 1" - validateOp 1 "TransferOrCreateAcct" sender01ks Successful 1.0 cred + step "validate sender00 debit at op index 1" + validateOp 1 "TransferOrCreateAcct" sender00ks Successful (negate 1.0) deb - step "validate sender00 debit at op index 2" - validateOp 2 "TransferOrCreateAcct" sender00ks Successful (negate 1.0) deb + step "validate sender01 credit at op index 2" + validateOp 2 "TransferOrCreateAcct" sender01ks Successful 1.0 cred step "validate sender00 gas redemption at op index 3" validateOp 3 "GasPayment" sender00ks Successful (defFundGas - transferGasCost) redeem @@ -295,7 +310,21 @@ blockTests testname tio envIo = testCaseSteps testname $ \step -> do validateBlock $ _blockResp_block resp - validateTxs remeds cbase fundtx cred deb gasRedeem gasReward = do + validateTxs remeds cbase fundtx deb cred gasRedeem gasReward = do + -- The order in which operations are returned is flaky and may break. Use + -- the following to double check the order in case this test fails. + -- + -- step $ "fundtx: ----------------------" + -- step $ debugShowOperation fundtx + -- step $ "deb: ----------------------" + -- step $ debugShowOperation deb + -- step $ "cred: ----------------------" + -- step $ debugShowOperation cred + -- step $ "redeem: ----------------------" + -- step $ debugShowOperation gasRedeem + -- step $ "reward: ----------------------" + -- step $ debugShowOperation gasReward + -- step $ "----------------------" -- coinbase is considered a separate tx list validateOp 0 "CoinbaseReward" noMinerks Successful defMiningReward cbase @@ -307,8 +336,8 @@ blockTests testname tio envIo = testCaseSteps testname $ \step -> do -- rest txs (i.e. transfer transaction) validateOp 0 "FundTx" sender00ks Successful (negate defFundGas) fundtx - validateOp 1 "TransferOrCreateAcct" sender01ks Successful 1.0 cred - validateOp 2 "TransferOrCreateAcct" sender00ks Successful (negate 1.0) deb + validateOp 1 "TransferOrCreateAcct" sender00ks Successful (negate 1.0) deb + validateOp 2 "TransferOrCreateAcct" sender01ks Successful 1.0 cred validateOp 3 "GasPayment" sender00ks Successful (defFundGas - transferGasCost) gasRedeem validateOp 4 "GasPayment" noMinerks Successful transferGasCost gasReward From e5a05c66201a841d20f50babf0b953b9d00594a4 Mon Sep 17 00:00:00 2001 From: Lars Kuhtz Date: Sat, 2 Nov 2024 22:16:12 -0700 Subject: [PATCH 03/19] pact version that support hashable-1.5 --- cabal.project | 7 +- cabal.project.freeze | 189 +++++++++++++----- src/Chainweb/Pact/Service/Types.hs | 10 +- src/Chainweb/Pact/TransactionExec.hs | 4 +- test/unit/Chainweb/Test/Pact/Checkpointer.hs | 4 +- .../Test/Pact/ModuleCacheOnRestart.hs | 14 +- 6 files changed, 162 insertions(+), 66 deletions(-) diff --git a/cabal.project b/cabal.project index a6f312e768..a092a70f46 100644 --- a/cabal.project +++ b/cabal.project @@ -80,8 +80,8 @@ package yet-another-logger source-repository-package type: git location: https://github.com/kadena-io/pact.git - tag: 058da315f7b427ad5dcaa334683600a1f7fd9588 - --sha256: 105jbsvrjv2zxsazd3vcbkml99w26xqimwjybfiznkg21456d6hn + tag: 8d51f4b251c22ecf623a2a49d26016c75bc728fd + --sha256: 18xgvzb3p8chch85747ln9a2191df09vwwrv9v3njr2h69n3rhxj source-repository-package type: git @@ -206,11 +206,10 @@ allow-newer: base-compat-batteries:* allow-newer: webauthn:* -- many packages use an spurious <1.5 upper bound on hashable --- allow-newer: *:hashable +allow-newer: *:hashable -- -------------------------------------------------------------------------- -- -- Upper Bounds -- remove when crypton-connection >0.4.1 got released constraints: http-client-tls <0.3.6.4 -constraints: hashable < 1.4.5 \ No newline at end of file diff --git a/cabal.project.freeze b/cabal.project.freeze index 8065484974..7987ad3ebb 100644 --- a/cabal.project.freeze +++ b/cabal.project.freeze @@ -7,58 +7,80 @@ constraints: any.Cabal ==3.10.2.0, any.OneTuple ==0.4.2, any.Only ==0.1, any.QuickCheck ==2.15.0.1, + QuickCheck -old-random +templatehaskell, any.RSA ==2.4.1, any.SHA ==1.6.4.4, + SHA -exe, any.StateVar ==1.2.2, any.adjunctions ==4.4.2, - any.aeson ==2.2.2.0, + any.aeson ==2.2.3.0, + aeson +ordered-keymap, any.aeson-pretty ==0.8.10, + aeson-pretty -lib-only, any.ansi-terminal ==1.1.1, + ansi-terminal -example, any.ansi-terminal-types ==1.1, any.ap-normalize ==0.1.0.1, + ap-normalize -test-with-clang, any.appar ==0.1.8, any.array ==0.5.6.0, any.asn1-encoding ==0.9.6, any.asn1-parse ==0.9.5, any.asn1-types ==0.3.4, any.assoc ==1.1.1, + assoc -tagged, any.async ==2.2.5, + async -bench, any.atomic-primops ==0.8.8, + atomic-primops -debug, any.attoparsec ==0.14.4, + attoparsec -developer, + any.attoparsec-aeson ==2.2.2.0, any.authenticate-oauth ==1.7, - any.auto-update ==0.2.1, + any.auto-update ==0.2.2, any.base ==4.19.1.0, any.base-compat ==0.14.0, any.base-compat-batteries ==0.14.0, any.base-orphans ==0.9.2, any.base-unicode-symbols ==0.2.4.2, + base-unicode-symbols +base-4-8 -old-base, any.base16-bytestring ==1.0.2.0, any.base64-bytestring ==1.2.1.0, any.base64-bytestring-kadena ==0.1, any.basement ==0.0.16, any.bifunctors ==5.6.2, + bifunctors +tagged, any.binary ==0.8.9.1, any.binary-orphans ==1.0.5, any.bitvec ==1.1.5.0, + bitvec +simd, any.blaze-builder ==0.4.2.3, any.blaze-html ==0.9.2.0, any.blaze-markup ==0.8.3.0, any.boring ==0.2.2, + boring +tagged, any.bound ==2.0.7, + bound +template-haskell, any.bsb-http-chunked ==0.0.0.4, any.bytebuild ==0.3.16.2, + bytebuild -checked, any.byteorder ==1.0.4, any.bytes ==0.17.3, any.byteslice ==0.2.13.2, + byteslice +avoid-rawmemchr, any.bytesmith ==0.3.11.1, any.bytestring ==0.12.1.0, any.bytestring-builder ==0.10.8.2.0, + bytestring-builder +bytestring_has_builder, + any.cabal-doctest ==1.0.10, any.cache ==0.1.3.0, any.call-stack ==0.4.0, any.case-insensitive ==1.2.1.0, - any.cassava ==0.5.3.1, + any.cassava ==0.5.3.2, any.cborg ==0.2.10.0, + cborg +optimize-gmp, any.cereal ==0.5.8.3, + cereal -bytestring-builder, chainweb -debug -ed25519 -ghc-flags, chainweb-node -debug -ed25519 -ghc-flags, any.chainweb-storage ==0.1.0.0, @@ -66,37 +88,46 @@ constraints: any.Cabal ==3.10.2.0, any.charset ==0.3.10, any.chronos ==1.1.6.1, any.clock ==0.8.4, + clock -llvm, + any.cmdargs ==0.10.22, + cmdargs +quotation -testprog, any.code-page ==0.2.1, any.colour ==2.3.6, any.comonad ==5.0.8, - any.conduit ==1.3.5, + comonad +containers +distributive +indexed-traversable, + any.conduit ==1.3.6, any.configuration-tools ==0.7.0, + configuration-tools -remote-configs, any.constraints ==0.14.2, any.containers ==0.6.8, any.contiguous ==0.6.4.2, any.contravariant ==1.5.5, + contravariant +semigroups +statevar +tagged, any.cookie ==0.5.0, - any.criterion ==1.6.3.0, - any.criterion-measurement ==0.2.2.0, + any.criterion ==1.6.4.0, + criterion -embed-data-files -fast, + any.criterion-measurement ==0.2.3.0, + criterion-measurement -fast, any.crypto-api ==0.13.3, + crypto-api -all_cpolys, any.crypto-pubkey-types ==0.4.3, - any.crypto-token ==0.1.1, + any.crypto-token ==0.1.2, any.cryptohash-md5 ==0.11.101.0, any.cryptohash-sha1 ==0.11.101.0, - any.crypton ==1.0.0, + any.crypton ==1.0.1, + crypton -check_alignment +integer-gmp -old_toolchain_inliner +support_aesni +support_deepseq +support_pclmuldq +support_rdrand -support_sse +use_target_attributes, any.crypton-connection ==0.4.1, any.crypton-x509 ==1.7.7, any.crypton-x509-store ==1.6.9, any.crypton-x509-system ==1.6.7, any.crypton-x509-validation ==1.6.12, any.cryptonite ==0.30, + cryptonite -check_alignment +integer-gmp -old_toolchain_inliner +support_aesni +support_deepseq -support_pclmuldq +support_rdrand -support_sse +use_target_attributes, any.cuckoo ==0.3.1, + cuckoo -mwc-random -pcg-random, any.data-bword ==0.1.0.2, - any.data-default ==0.7.1.1, - any.data-default-class ==0.1.2.0, - any.data-default-instances-containers ==0.0.1, - any.data-default-instances-dlist ==0.0.1, - any.data-default-instances-old-locale ==0.0.1, + any.data-default ==0.8.0.0, + any.data-default-class ==0.2.0.0, any.data-dword ==0.3.2.1, any.data-fix ==0.3.4, any.data-ordlist ==0.4.7.0, @@ -104,25 +135,31 @@ constraints: any.Cabal ==3.10.2.0, any.deepseq ==1.5.0.0, any.dense-linear-algebra ==0.1.0.0, any.deriving-compat ==0.6.6, + deriving-compat +base-4-9 +new-functor-classes +template-haskell-2-11, any.digraph ==0.3.0, any.direct-sqlite ==2.3.29, + direct-sqlite +dbstat +fulltextsearch +haveusleep +json1 -mathfunctions -systemlib +urifilenames, any.directory ==1.3.8.1, any.distributive ==0.6.2.1, + distributive +semigroups +tagged, any.dlist ==1.0, + dlist -werror, any.easy-file ==0.2.5, any.enclosed-exceptions ==1.0.3, any.entropy ==0.4.1.10, + entropy -donotgetentropy, any.errors ==2.3.0, any.ethereum ==0.1.0.2, ethereum -ethhash -openssl-use-pkg-config, any.exceptions ==0.10.7, - any.fast-logger ==3.2.3, + any.fast-logger ==3.2.5, any.file-embed ==0.0.16.0, any.filepath ==1.4.200.1, any.fingertree ==0.1.5.0, - any.finite-typelits ==0.2.0.1, + any.finite-typelits ==0.2.1.0, any.free ==5.2, - any.generic-data ==1.1.0.0, + any.generic-data ==1.1.0.1, + generic-data -enable-inspect, any.generically ==0.1.1, any.ghc-bignum ==1.3, any.ghc-boot-th ==9.8.2, @@ -132,66 +169,89 @@ constraints: any.Cabal ==3.10.2.0, any.groups ==0.5.3, any.growable-vector ==0.1, any.half ==0.3.1, - any.hashable ==1.4.4.0, - any.hashes ==0.3.0, - any.haskell-lexer ==1.1.1, + any.happy ==2.1.2, + any.happy-lib ==2.1.2, + any.hashable ==1.5.0.0, + hashable -arch-native -random-initial-seed, + any.hashes ==0.3.0.1, + hashes -benchmark-cryptonite -openssl-use-pkg-config -test-cryptonite +with-openssl, + any.haskell-lexer ==1.1.2, any.haskell-src-exts ==1.23.1, any.haskell-src-meta ==0.8.14, any.heaps ==0.4, any.hourglass ==0.2.12, + any.hsc2hs ==0.68.10, + hsc2hs -in-ghc-tree, any.http-api-data ==0.6.1, + http-api-data -use-text-show, any.http-client ==0.7.17, + http-client +network-uri, any.http-client-tls ==0.3.6.3, any.http-date ==0.0.11, any.http-media ==0.8.1.1, - any.http-semantics ==0.2.0, + any.http-semantics ==0.2.1, any.http-types ==0.12.4, - any.http2 ==5.3.1, + any.http2 ==5.3.4, + http2 -devel -h2spec, any.indexed-list-literals ==0.2.1.3, any.indexed-traversable ==0.1.4, any.indexed-traversable-instances ==0.1.2, any.integer-conversion ==0.1.1, any.integer-gmp ==1.1, any.integer-logarithms ==1.0.3.1, + integer-logarithms -check-bounds +integer-gmp, any.invariant ==0.6.3, - any.iproute ==1.7.14, + any.iproute ==1.7.15, any.ixset-typed ==0.5, any.js-chart ==2.9.4.1, any.kan-extensions ==5.2.6, any.lens ==5.3.2, + lens -benchmark-uniplate -dump-splices +inlining -j +test-hunit +test-properties +test-templates +trustworthy, any.lens-aeson ==1.2.3, any.libBF ==0.6.8, + libBF -system-libbf, any.libyaml ==0.1.4, + libyaml -no-unicode -system-libyaml, any.libyaml-clib ==0.2.5, any.lifted-base ==0.2.3.12, any.loglevel ==0.1.0.0, any.managed ==1.0.10, any.massiv ==1.0.4.0, + massiv -unsafe-checks, any.math-functions ==0.3.4.4, + math-functions +system-erf +system-expm1, any.megaparsec ==9.6.1, + megaparsec -dev, any.memory ==0.18.0, + memory +support_bytestring +support_deepseq, any.merkle-log ==0.2.0, any.microlens ==0.4.13.1, any.microstache ==1.0.3, any.mime-types ==0.1.2.0, any.mmorph ==1.2.0, any.mod ==0.2.0.1, + mod +semirings +vector, any.monad-control ==1.0.3.1, - any.mono-traversable ==1.0.17.0, + any.mono-traversable ==1.0.20.0, any.mtl ==2.3.1, any.mtl-compat ==0.2.2, + mtl-compat -two-point-one -two-point-two, any.mwc-probability ==2.3.1, any.mwc-random ==0.15.1.0, + mwc-random -benchpapi, any.natural-arithmetic ==0.2.1.0, - any.network ==3.1.4.0, + any.network ==3.2.4.0, + network -devel, any.network-byte-order ==0.1.7, any.network-control ==0.1.3, any.network-info ==0.2.1, any.network-uri ==2.6.4.2, - any.nothunks ==0.2.1.1, + any.nothunks ==0.3.0.0, + nothunks +bytestring +text +vector, any.old-locale ==1.0.0.7, any.old-time ==1.1.0.4, any.optparse-applicative ==0.18.1.0, + optparse-applicative +process, any.os-string ==2.0.6, any.pact ==4.13.1, pact -build-tool +cryptonite-ed25519 -tests-in-lib, @@ -201,13 +261,17 @@ constraints: any.Cabal ==3.10.2.0, any.parallel ==3.2.2.0, any.parsec ==3.1.17.0, any.parser-combinators ==1.3.0, - any.parsers ==0.12.11, + parser-combinators -dev, + any.parsers ==0.12.12, + parsers +attoparsec +binary +parsec, any.patience ==0.3, any.pem ==0.2.4, any.poly ==0.5.1.0, + poly +sparse, any.pretty ==1.1.3.6, any.pretty-show ==1.10, any.prettyprinter ==1.7.1, + prettyprinter -buildreadme +text, any.prettyprinter-ansi-terminal ==1.1.3, any.primitive ==0.9.0.0, any.primitive-addr ==0.1.0.3, @@ -222,11 +286,14 @@ constraints: any.Cabal ==3.10.2.0, any.recv ==0.1.0, any.reducers ==3.12.4, any.reflection ==2.1.8, + reflection -slow +template-haskell, any.regex-base ==0.94.0.2, any.regex-tdfa ==1.3.2.2, + regex-tdfa +doctest -force-o2, any.resource-pool ==0.4.0.0, any.resourcet ==1.3.0, any.retry ==0.9.3.1, + retry -lib-werror, any.rocksdb-haskell-kadena ==1.1.0, rocksdb-haskell-kadena -with-tbb, any.rosetta ==1.0.1, @@ -238,38 +305,50 @@ constraints: any.Cabal ==3.10.2.0, any.sbv ==9.2, any.scheduler ==2.0.0.1, any.scientific ==0.3.8.0, + scientific -integer-simple, any.semialign ==1.3.1, + semialign +semigroupoids, any.semigroupoids ==6.0.1, + semigroupoids +comonad +containers +contravariant +distributive +tagged +unordered-containers, any.semigroups ==0.20, + semigroups +binary +bytestring -bytestring-builder +containers +deepseq +hashable +tagged +template-haskell +text +transformers +unordered-containers, any.semirings ==0.7, + semirings +containers +unordered-containers, any.serialise ==0.2.6.1, - any.servant ==0.20.1, - any.servant-client ==0.20, - any.servant-client-core ==0.20, - any.servant-server ==0.20, + serialise +newtime15, + any.servant ==0.20.2, + any.servant-client ==0.20.2, + any.servant-client-core ==0.20.2, + any.servant-server ==0.20.2, any.sha-validation ==0.1.0.1, any.show-combinators ==0.2.0.0, any.simple-sendfile ==0.2.32, + simple-sendfile +allow-bsd -fallback, any.singleton-bool ==0.1.8, any.socks ==0.6.1, any.some ==1.0.6, + some +newtype-unsafe, any.sop-core ==0.5.0.2, any.split ==0.2.5, any.splitmix ==0.1.0.5, + splitmix -optimised-mixer, any.statistics ==0.16.2.1, any.stm ==2.5.2.1, any.stm-chans ==3.0.0.9, any.stopwatch ==0.1.0.6, + stopwatch -test_delay_upper_bound -test_threaded, any.streaming ==0.2.4.0, any.streaming-commons ==0.2.2.6, - any.strict ==0.5, + streaming-commons -use-bytestring-builder, + any.strict ==0.5.1, any.strict-concurrency ==0.2.4.3, - any.string-conversions ==0.4.0.1, any.syb ==0.7.2.4, - any.system-cxx-std-lib ==1.0, any.tagged ==0.8.8, + tagged +deepseq +transformers, any.tasty ==1.5, + tasty +unix, any.tasty-golden ==2.3.5, + tasty-golden -build-example, any.tasty-hunit ==0.10.2, any.tasty-json ==0.1.0.0, any.tasty-quickcheck ==0.11, @@ -278,63 +357,81 @@ constraints: any.Cabal ==3.10.2.0, any.text ==2.1.1, any.text-iso8601 ==0.1.1, any.text-short ==0.1.6, + text-short -asserts, any.th-abstraction ==0.7.0.0, any.th-compat ==0.1.5, any.th-expand-syns ==0.4.11.0, - any.th-lift ==0.8.4, - any.th-orphans ==0.13.14, + any.th-lift ==0.8.5, + any.th-orphans ==0.13.15, any.th-reify-many ==0.1.10, any.these ==1.2.1, any.time ==1.12.2, any.time-compat ==1.9.7, any.time-locale-compat ==0.1.1.5, + time-locale-compat +old-locale, any.time-manager ==0.1.0, - any.tls ==2.1.0, - any.tls-session-manager ==0.0.6, + any.tls ==2.1.2, + tls -devel, + any.tls-session-manager ==0.0.7, any.token-bucket ==0.1.0.1, + token-bucket +use-cbits, any.torsor ==0.1.0.1, any.transformers ==0.6.1.0, any.transformers-base ==0.4.6, + transformers-base +orphaninstances, any.transformers-compat ==0.7.2, + transformers-compat -five +five-three -four +generic-deriving +mtl -three -two, any.trifecta ==2.1.4, any.tuples ==0.1.0.0, - any.typed-process ==0.2.11.1, + any.typed-process ==0.2.12.0, any.uniplate ==1.6.13, any.unix ==2.8.4.0, - any.unix-compat ==0.7.2, - any.unix-time ==0.4.15, + any.unix-compat ==0.7.3, + any.unix-time ==0.4.16, any.unlifted ==0.2.2.0, any.unliftio ==0.2.25.0, any.unliftio-core ==0.2.1.0, any.unordered-containers ==0.2.20, + unordered-containers -debug, any.utf8-string ==1.0.2, any.uuid ==1.3.16, any.uuid-types ==1.0.6, any.validation ==1.1.3, any.vault ==0.3.1.5, - any.vector ==0.13.1.0, + vault +useghc, + any.vector ==0.13.2.0, + vector +boundschecks -internalchecks -unsafechecks -wall, any.vector-algorithms ==0.9.0.2, + vector-algorithms +bench +boundschecks -internalchecks -llvm +properties -unsafechecks, any.vector-binary-instances ==0.2.5.2, any.vector-sized ==1.6.1, any.vector-stream ==0.1.0.1, any.vector-th-unbox ==0.2.2, any.void ==0.7.3, + void -safe, any.wai ==3.2.4, any.wai-app-static ==3.1.9, + wai-app-static +crypton -print, any.wai-cors ==0.2.7, any.wai-extra ==3.1.15, - any.wai-logger ==2.4.1, + wai-extra -build-example, + any.wai-logger ==2.5.0, any.wai-middleware-throttle ==0.3.0.1, any.wai-middleware-validation ==0.1.0.2, - any.warp ==3.4.1, - any.warp-tls ==3.4.6, + any.warp ==3.4.3, + warp +allow-sendfilefd -network-bytestring -warp-debug +x509, + any.warp-tls ==3.4.7, any.wherefrom-compat ==0.1.1.1, any.wide-word ==0.1.6.0, any.witherable ==0.5, any.word8 ==0.1.3, any.wreq ==0.5.4.3, + wreq -aws -developer +doctest -httpbin, any.yaml ==0.11.11.2, + yaml +no-examples +no-exe, any.yet-another-logger ==0.4.2, + yet-another-logger -tbmqueue, any.zigzag ==0.1.0.0, - any.zlib ==0.7.1.0 -index-state: hackage.haskell.org 2024-10-31T14:36:55Z + any.zlib ==0.7.1.0, + zlib -bundled-c-zlib +non-blocking-ffi +pkg-config +index-state: hackage.haskell.org 2024-11-02T14:34:14Z diff --git a/src/Chainweb/Pact/Service/Types.hs b/src/Chainweb/Pact/Service/Types.hs index 916e83bc96..ea5edb8f9f 100644 --- a/src/Chainweb/Pact/Service/Types.hs +++ b/src/Chainweb/Pact/Service/Types.hs @@ -101,7 +101,6 @@ import Control.Applicative import Data.Aeson import qualified Data.ByteString.Short as SB import Data.HashMap.Strict (HashMap) -import qualified Data.HashMap.Strict as HM import qualified Data.List.NonEmpty as NE import Data.Text (Text, unpack) import qualified Data.Text.Encoding as T @@ -123,6 +122,7 @@ import Pact.Types.RowData import Pact.Types.Runtime hiding (ChainId) import qualified Pact.JSON.Encode as J import qualified Pact.JSON.Legacy.HashMap as LHM +import qualified Pact.Utils.StableHashMap as SHM -- internal chainweb modules @@ -523,14 +523,14 @@ filterModuleCacheByKey f (ModuleCache c) = ModuleCache $ moduleCacheToHashMap :: ModuleCache - -> HM.HashMap ModuleName (ModuleData Ref, Bool) -moduleCacheToHashMap (ModuleCache c) = HM.fromList $ LHM.toList c + -> SHM.StableHashMap ModuleName (ModuleData Ref, Bool) +moduleCacheToHashMap (ModuleCache c) = SHM.fromList $ LHM.toList c {-# INLINE moduleCacheToHashMap #-} moduleCacheFromHashMap - :: HM.HashMap ModuleName (ModuleData Ref, Bool) + :: SHM.StableHashMap ModuleName (ModuleData Ref, Bool) -> ModuleCache -moduleCacheFromHashMap = ModuleCache . LHM.fromList . HM.toList +moduleCacheFromHashMap = ModuleCache . LHM.fromList . SHM.toList {-# INLINE moduleCacheFromHashMap #-} moduleCacheKeys :: ModuleCache -> [ModuleName] diff --git a/src/Chainweb/Pact/TransactionExec.hs b/src/Chainweb/Pact/TransactionExec.hs index 27eb1c666d..00ceae4949 100644 --- a/src/Chainweb/Pact/TransactionExec.hs +++ b/src/Chainweb/Pact/TransactionExec.hs @@ -70,7 +70,6 @@ import qualified Data.ByteString.Short as SB import Data.Decimal (Decimal, roundTo) import Data.Foldable (fold, for_, traverse_) import Data.IORef -import qualified Data.HashMap.Strict as HM import qualified Data.List as List import qualified Data.Map.Strict as M import Data.Maybe @@ -105,6 +104,7 @@ import Pact.Types.SPV import Pact.Types.Verifier import Pact.Types.Util as PU +import qualified Pact.Utils.StableHashMap as SHM -- internal Chainweb modules @@ -1040,7 +1040,7 @@ findPayer txCtx cmd = runMaybeT $ do gasPayerIface = ModuleName "gas-payer-v1" Nothing lookupIfaceModRef (QualifiedName _ n _) (ModuleData (MDModule Module{..}) refs _) - | gasPayerIface `elem` _mInterfaces = HM.lookup n refs + | gasPayerIface `elem` _mInterfaces = SHM.lookup n refs lookupIfaceModRef _ _ = Nothing mkApp i r as = App (TVar r i) (map (liftTerm . fromPactValue) as) i diff --git a/test/unit/Chainweb/Test/Pact/Checkpointer.hs b/test/unit/Chainweb/Test/Pact/Checkpointer.hs index 573cc5bffe..b8bb098fd4 100644 --- a/test/unit/Chainweb/Test/Pact/Checkpointer.hs +++ b/test/unit/Chainweb/Test/Pact/Checkpointer.hs @@ -19,7 +19,6 @@ import Control.Monad.Reader import Data.Aeson (Value(..), object, (.=), Key) import Data.Function -import qualified Data.HashMap.Strict as HM import qualified Data.Map.Strict as M import Data.Text (Text) import qualified Data.Text as T @@ -40,6 +39,7 @@ import Pact.Types.Runtime hiding (ChainId) import Pact.Types.SPV (noSPVSupport) import Pact.Types.SQLite import qualified Pact.JSON.Encode as J +import qualified Pact.Utils.StableHashMap as SHM import Test.Tasty import Test.Tasty.HUnit @@ -785,7 +785,7 @@ loadModule = do fn = "test/pact/simple.repl" nativeLookup :: NativeDefName -> Maybe (Term Name) -nativeLookup (NativeDefName n) = case HM.lookup n nativeDefs of +nativeLookup (NativeDefName n) = case SHM.lookup n nativeDefs of Just (Direct t) -> Just t _ -> Nothing diff --git a/test/unit/Chainweb/Test/Pact/ModuleCacheOnRestart.hs b/test/unit/Chainweb/Test/Pact/ModuleCacheOnRestart.hs index 3576ce752c..613baa9d9d 100644 --- a/test/unit/Chainweb/Test/Pact/ModuleCacheOnRestart.hs +++ b/test/unit/Chainweb/Test/Pact/ModuleCacheOnRestart.hs @@ -16,7 +16,6 @@ import Control.Lens import Control.Monad import Control.Monad.IO.Class -import qualified Data.HashMap.Strict as HM import qualified Data.Map.Strict as M import Data.List (intercalate) import qualified Data.Text as T @@ -32,6 +31,7 @@ import System.LogLevel import Pact.Types.Runtime (mdModule) import Pact.Types.Term +import qualified Pact.Utils.StableHashMap as SHM -- chainweb imports @@ -69,7 +69,7 @@ type RewindPoint = (BlockHeader, PayloadWithOutputs) data RewindData = RewindData { afterV4 :: RewindPoint , beforeV4 :: RewindPoint - , v3Cache :: HM.HashMap ModuleName (Maybe ModuleHash) + , v3Cache :: SHM.StableHashMap ModuleName (Maybe ModuleHash) } deriving Generic instance NFData RewindData @@ -226,12 +226,12 @@ testCw217CoinOnly iobdb _rewindM = (go, go') Nothing -> assertFailure "failed to lookup block at 20" assertNoCacheMismatch - :: HM.HashMap ModuleName (Maybe ModuleHash) - -> HM.HashMap ModuleName (Maybe ModuleHash) + :: SHM.StableHashMap ModuleName (Maybe ModuleHash) + -> SHM.StableHashMap ModuleName (Maybe ModuleHash) -> Assertion assertNoCacheMismatch c1 c2 = assertBool msg $ c1 == c2 where - showCache = intercalate "\n" . map show . HM.toList + showCache = intercalate "\n" . map show . SHM.toList msg = mconcat [ "Module cache mismatch, found: \n" @@ -268,10 +268,10 @@ doNextCoinbaseN_ doNextCoinbaseN_ n iobdb = fmap last $ replicateM n $ doNextCoinbase iobdb -- | Interfaces can't be upgraded, but modules can, so verify hash in that case. -justModuleHashes :: ModuleInitCache -> HM.HashMap ModuleName (Maybe ModuleHash) +justModuleHashes :: ModuleInitCache -> SHM.StableHashMap ModuleName (Maybe ModuleHash) justModuleHashes = justModuleHashes' . snd . last . M.toList -justModuleHashes' :: ModuleCache -> HM.HashMap ModuleName (Maybe ModuleHash) +justModuleHashes' :: ModuleCache -> SHM.StableHashMap ModuleName (Maybe ModuleHash) justModuleHashes' = fmap (preview (_1 . mdModule . _MDModule . mHash)) . moduleCacheToHashMap From 3686deae7b0f651239c5841739fc47a3be4ff455 Mon Sep 17 00:00:00 2001 From: chessai Date: Mon, 4 Nov 2024 10:29:48 -0600 Subject: [PATCH 04/19] update cabal.project.freeze Change-Id: I53d40fc38a8b7cddeab3bb608ffc651dd8099ea4 --- cabal.project.freeze | 183 ++++++++++--------------------------------- 1 file changed, 43 insertions(+), 140 deletions(-) diff --git a/cabal.project.freeze b/cabal.project.freeze index 7987ad3ebb..84b4cdf658 100644 --- a/cabal.project.freeze +++ b/cabal.project.freeze @@ -7,80 +7,58 @@ constraints: any.Cabal ==3.10.2.0, any.OneTuple ==0.4.2, any.Only ==0.1, any.QuickCheck ==2.15.0.1, - QuickCheck -old-random +templatehaskell, any.RSA ==2.4.1, any.SHA ==1.6.4.4, - SHA -exe, any.StateVar ==1.2.2, any.adjunctions ==4.4.2, any.aeson ==2.2.3.0, - aeson +ordered-keymap, any.aeson-pretty ==0.8.10, - aeson-pretty -lib-only, any.ansi-terminal ==1.1.1, - ansi-terminal -example, any.ansi-terminal-types ==1.1, any.ap-normalize ==0.1.0.1, - ap-normalize -test-with-clang, any.appar ==0.1.8, any.array ==0.5.6.0, any.asn1-encoding ==0.9.6, any.asn1-parse ==0.9.5, any.asn1-types ==0.3.4, any.assoc ==1.1.1, - assoc -tagged, any.async ==2.2.5, - async -bench, any.atomic-primops ==0.8.8, - atomic-primops -debug, any.attoparsec ==0.14.4, - attoparsec -developer, - any.attoparsec-aeson ==2.2.2.0, any.authenticate-oauth ==1.7, - any.auto-update ==0.2.2, + any.auto-update ==0.2.1, any.base ==4.19.1.0, any.base-compat ==0.14.0, any.base-compat-batteries ==0.14.0, any.base-orphans ==0.9.2, any.base-unicode-symbols ==0.2.4.2, - base-unicode-symbols +base-4-8 -old-base, any.base16-bytestring ==1.0.2.0, any.base64-bytestring ==1.2.1.0, any.base64-bytestring-kadena ==0.1, any.basement ==0.0.16, any.bifunctors ==5.6.2, - bifunctors +tagged, any.binary ==0.8.9.1, any.binary-orphans ==1.0.5, any.bitvec ==1.1.5.0, - bitvec +simd, any.blaze-builder ==0.4.2.3, any.blaze-html ==0.9.2.0, any.blaze-markup ==0.8.3.0, any.boring ==0.2.2, - boring +tagged, any.bound ==2.0.7, - bound +template-haskell, any.bsb-http-chunked ==0.0.0.4, any.bytebuild ==0.3.16.2, - bytebuild -checked, any.byteorder ==1.0.4, any.bytes ==0.17.3, any.byteslice ==0.2.13.2, - byteslice +avoid-rawmemchr, any.bytesmith ==0.3.11.1, any.bytestring ==0.12.1.0, any.bytestring-builder ==0.10.8.2.0, - bytestring-builder +bytestring_has_builder, - any.cabal-doctest ==1.0.10, any.cache ==0.1.3.0, any.call-stack ==0.4.0, any.case-insensitive ==1.2.1.0, - any.cassava ==0.5.3.2, + any.cassava ==0.5.3.1, any.cborg ==0.2.10.0, - cborg +optimize-gmp, any.cereal ==0.5.8.3, - cereal -bytestring-builder, chainweb -debug -ed25519 -ghc-flags, chainweb-node -debug -ed25519 -ghc-flags, any.chainweb-storage ==0.1.0.0, @@ -88,46 +66,37 @@ constraints: any.Cabal ==3.10.2.0, any.charset ==0.3.10, any.chronos ==1.1.6.1, any.clock ==0.8.4, - clock -llvm, - any.cmdargs ==0.10.22, - cmdargs +quotation -testprog, any.code-page ==0.2.1, any.colour ==2.3.6, any.comonad ==5.0.8, - comonad +containers +distributive +indexed-traversable, - any.conduit ==1.3.6, + any.conduit ==1.3.5, any.configuration-tools ==0.7.0, - configuration-tools -remote-configs, any.constraints ==0.14.2, any.containers ==0.6.8, any.contiguous ==0.6.4.2, any.contravariant ==1.5.5, - contravariant +semigroups +statevar +tagged, any.cookie ==0.5.0, - any.criterion ==1.6.4.0, - criterion -embed-data-files -fast, - any.criterion-measurement ==0.2.3.0, - criterion-measurement -fast, + any.criterion ==1.6.3.0, + any.criterion-measurement ==0.2.2.0, any.crypto-api ==0.13.3, - crypto-api -all_cpolys, any.crypto-pubkey-types ==0.4.3, - any.crypto-token ==0.1.2, + any.crypto-token ==0.1.1, any.cryptohash-md5 ==0.11.101.0, any.cryptohash-sha1 ==0.11.101.0, - any.crypton ==1.0.1, - crypton -check_alignment +integer-gmp -old_toolchain_inliner +support_aesni +support_deepseq +support_pclmuldq +support_rdrand -support_sse +use_target_attributes, + any.crypton ==1.0.0, any.crypton-connection ==0.4.1, any.crypton-x509 ==1.7.7, any.crypton-x509-store ==1.6.9, any.crypton-x509-system ==1.6.7, any.crypton-x509-validation ==1.6.12, any.cryptonite ==0.30, - cryptonite -check_alignment +integer-gmp -old_toolchain_inliner +support_aesni +support_deepseq -support_pclmuldq +support_rdrand -support_sse +use_target_attributes, any.cuckoo ==0.3.1, - cuckoo -mwc-random -pcg-random, any.data-bword ==0.1.0.2, - any.data-default ==0.8.0.0, - any.data-default-class ==0.2.0.0, + any.data-default ==0.7.1.1, + any.data-default-class ==0.1.2.0, + any.data-default-instances-containers ==0.0.1, + any.data-default-instances-dlist ==0.0.1, + any.data-default-instances-old-locale ==0.0.1, any.data-dword ==0.3.2.1, any.data-fix ==0.3.4, any.data-ordlist ==0.4.7.0, @@ -135,31 +104,25 @@ constraints: any.Cabal ==3.10.2.0, any.deepseq ==1.5.0.0, any.dense-linear-algebra ==0.1.0.0, any.deriving-compat ==0.6.6, - deriving-compat +base-4-9 +new-functor-classes +template-haskell-2-11, any.digraph ==0.3.0, any.direct-sqlite ==2.3.29, - direct-sqlite +dbstat +fulltextsearch +haveusleep +json1 -mathfunctions -systemlib +urifilenames, any.directory ==1.3.8.1, any.distributive ==0.6.2.1, - distributive +semigroups +tagged, any.dlist ==1.0, - dlist -werror, any.easy-file ==0.2.5, any.enclosed-exceptions ==1.0.3, any.entropy ==0.4.1.10, - entropy -donotgetentropy, any.errors ==2.3.0, any.ethereum ==0.1.0.2, ethereum -ethhash -openssl-use-pkg-config, any.exceptions ==0.10.7, - any.fast-logger ==3.2.5, + any.fast-logger ==3.2.3, any.file-embed ==0.0.16.0, any.filepath ==1.4.200.1, any.fingertree ==0.1.5.0, - any.finite-typelits ==0.2.1.0, + any.finite-typelits ==0.2.0.1, any.free ==5.2, - any.generic-data ==1.1.0.1, - generic-data -enable-inspect, + any.generic-data ==1.1.0.0, any.generically ==0.1.1, any.ghc-bignum ==1.3, any.ghc-boot-th ==9.8.2, @@ -169,89 +132,66 @@ constraints: any.Cabal ==3.10.2.0, any.groups ==0.5.3, any.growable-vector ==0.1, any.half ==0.3.1, - any.happy ==2.1.2, - any.happy-lib ==2.1.2, any.hashable ==1.5.0.0, - hashable -arch-native -random-initial-seed, - any.hashes ==0.3.0.1, - hashes -benchmark-cryptonite -openssl-use-pkg-config -test-cryptonite +with-openssl, - any.haskell-lexer ==1.1.2, + any.hashes ==0.3.0, + any.haskell-lexer ==1.1.1, any.haskell-src-exts ==1.23.1, any.haskell-src-meta ==0.8.14, any.heaps ==0.4, any.hourglass ==0.2.12, - any.hsc2hs ==0.68.10, - hsc2hs -in-ghc-tree, any.http-api-data ==0.6.1, - http-api-data -use-text-show, any.http-client ==0.7.17, - http-client +network-uri, any.http-client-tls ==0.3.6.3, any.http-date ==0.0.11, any.http-media ==0.8.1.1, - any.http-semantics ==0.2.1, + any.http-semantics ==0.2.0, any.http-types ==0.12.4, - any.http2 ==5.3.4, - http2 -devel -h2spec, + any.http2 ==5.3.1, any.indexed-list-literals ==0.2.1.3, any.indexed-traversable ==0.1.4, any.indexed-traversable-instances ==0.1.2, any.integer-conversion ==0.1.1, any.integer-gmp ==1.1, any.integer-logarithms ==1.0.3.1, - integer-logarithms -check-bounds +integer-gmp, any.invariant ==0.6.3, - any.iproute ==1.7.15, + any.iproute ==1.7.14, any.ixset-typed ==0.5, any.js-chart ==2.9.4.1, any.kan-extensions ==5.2.6, any.lens ==5.3.2, - lens -benchmark-uniplate -dump-splices +inlining -j +test-hunit +test-properties +test-templates +trustworthy, any.lens-aeson ==1.2.3, any.libBF ==0.6.8, - libBF -system-libbf, any.libyaml ==0.1.4, - libyaml -no-unicode -system-libyaml, any.libyaml-clib ==0.2.5, any.lifted-base ==0.2.3.12, any.loglevel ==0.1.0.0, any.managed ==1.0.10, any.massiv ==1.0.4.0, - massiv -unsafe-checks, any.math-functions ==0.3.4.4, - math-functions +system-erf +system-expm1, any.megaparsec ==9.6.1, - megaparsec -dev, any.memory ==0.18.0, - memory +support_bytestring +support_deepseq, any.merkle-log ==0.2.0, any.microlens ==0.4.13.1, any.microstache ==1.0.3, any.mime-types ==0.1.2.0, any.mmorph ==1.2.0, any.mod ==0.2.0.1, - mod +semirings +vector, any.monad-control ==1.0.3.1, - any.mono-traversable ==1.0.20.0, + any.mono-traversable ==1.0.17.0, any.mtl ==2.3.1, any.mtl-compat ==0.2.2, - mtl-compat -two-point-one -two-point-two, any.mwc-probability ==2.3.1, any.mwc-random ==0.15.1.0, - mwc-random -benchpapi, any.natural-arithmetic ==0.2.1.0, - any.network ==3.2.4.0, - network -devel, + any.network ==3.1.4.0, any.network-byte-order ==0.1.7, any.network-control ==0.1.3, any.network-info ==0.2.1, any.network-uri ==2.6.4.2, - any.nothunks ==0.3.0.0, - nothunks +bytestring +text +vector, + any.nothunks ==0.2.1.1, any.old-locale ==1.0.0.7, any.old-time ==1.1.0.4, any.optparse-applicative ==0.18.1.0, - optparse-applicative +process, any.os-string ==2.0.6, any.pact ==4.13.1, pact -build-tool +cryptonite-ed25519 -tests-in-lib, @@ -261,17 +201,13 @@ constraints: any.Cabal ==3.10.2.0, any.parallel ==3.2.2.0, any.parsec ==3.1.17.0, any.parser-combinators ==1.3.0, - parser-combinators -dev, - any.parsers ==0.12.12, - parsers +attoparsec +binary +parsec, + any.parsers ==0.12.11, any.patience ==0.3, any.pem ==0.2.4, any.poly ==0.5.1.0, - poly +sparse, any.pretty ==1.1.3.6, any.pretty-show ==1.10, any.prettyprinter ==1.7.1, - prettyprinter -buildreadme +text, any.prettyprinter-ansi-terminal ==1.1.3, any.primitive ==0.9.0.0, any.primitive-addr ==0.1.0.3, @@ -286,14 +222,11 @@ constraints: any.Cabal ==3.10.2.0, any.recv ==0.1.0, any.reducers ==3.12.4, any.reflection ==2.1.8, - reflection -slow +template-haskell, any.regex-base ==0.94.0.2, any.regex-tdfa ==1.3.2.2, - regex-tdfa +doctest -force-o2, any.resource-pool ==0.4.0.0, any.resourcet ==1.3.0, any.retry ==0.9.3.1, - retry -lib-werror, any.rocksdb-haskell-kadena ==1.1.0, rocksdb-haskell-kadena -with-tbb, any.rosetta ==1.0.1, @@ -305,50 +238,38 @@ constraints: any.Cabal ==3.10.2.0, any.sbv ==9.2, any.scheduler ==2.0.0.1, any.scientific ==0.3.8.0, - scientific -integer-simple, any.semialign ==1.3.1, - semialign +semigroupoids, any.semigroupoids ==6.0.1, - semigroupoids +comonad +containers +contravariant +distributive +tagged +unordered-containers, any.semigroups ==0.20, - semigroups +binary +bytestring -bytestring-builder +containers +deepseq +hashable +tagged +template-haskell +text +transformers +unordered-containers, any.semirings ==0.7, - semirings +containers +unordered-containers, any.serialise ==0.2.6.1, - serialise +newtime15, - any.servant ==0.20.2, - any.servant-client ==0.20.2, - any.servant-client-core ==0.20.2, - any.servant-server ==0.20.2, + any.servant ==0.20.1, + any.servant-client ==0.20, + any.servant-client-core ==0.20, + any.servant-server ==0.20, any.sha-validation ==0.1.0.1, any.show-combinators ==0.2.0.0, any.simple-sendfile ==0.2.32, - simple-sendfile +allow-bsd -fallback, any.singleton-bool ==0.1.8, any.socks ==0.6.1, any.some ==1.0.6, - some +newtype-unsafe, any.sop-core ==0.5.0.2, any.split ==0.2.5, any.splitmix ==0.1.0.5, - splitmix -optimised-mixer, any.statistics ==0.16.2.1, any.stm ==2.5.2.1, any.stm-chans ==3.0.0.9, any.stopwatch ==0.1.0.6, - stopwatch -test_delay_upper_bound -test_threaded, any.streaming ==0.2.4.0, any.streaming-commons ==0.2.2.6, - streaming-commons -use-bytestring-builder, any.strict ==0.5.1, any.strict-concurrency ==0.2.4.3, + any.string-conversions ==0.4.0.1, any.syb ==0.7.2.4, + any.system-cxx-std-lib ==1.0, any.tagged ==0.8.8, - tagged +deepseq +transformers, any.tasty ==1.5, - tasty +unix, any.tasty-golden ==2.3.5, - tasty-golden -build-example, any.tasty-hunit ==0.10.2, any.tasty-json ==0.1.0.0, any.tasty-quickcheck ==0.11, @@ -357,81 +278,63 @@ constraints: any.Cabal ==3.10.2.0, any.text ==2.1.1, any.text-iso8601 ==0.1.1, any.text-short ==0.1.6, - text-short -asserts, any.th-abstraction ==0.7.0.0, any.th-compat ==0.1.5, any.th-expand-syns ==0.4.11.0, - any.th-lift ==0.8.5, - any.th-orphans ==0.13.15, + any.th-lift ==0.8.4, + any.th-orphans ==0.13.14, any.th-reify-many ==0.1.10, any.these ==1.2.1, any.time ==1.12.2, any.time-compat ==1.9.7, any.time-locale-compat ==0.1.1.5, - time-locale-compat +old-locale, any.time-manager ==0.1.0, - any.tls ==2.1.2, - tls -devel, - any.tls-session-manager ==0.0.7, + any.tls ==2.1.0, + any.tls-session-manager ==0.0.6, any.token-bucket ==0.1.0.1, - token-bucket +use-cbits, any.torsor ==0.1.0.1, any.transformers ==0.6.1.0, any.transformers-base ==0.4.6, - transformers-base +orphaninstances, any.transformers-compat ==0.7.2, - transformers-compat -five +five-three -four +generic-deriving +mtl -three -two, any.trifecta ==2.1.4, any.tuples ==0.1.0.0, - any.typed-process ==0.2.12.0, + any.typed-process ==0.2.11.1, any.uniplate ==1.6.13, any.unix ==2.8.4.0, - any.unix-compat ==0.7.3, - any.unix-time ==0.4.16, + any.unix-compat ==0.7.2, + any.unix-time ==0.4.15, any.unlifted ==0.2.2.0, any.unliftio ==0.2.25.0, any.unliftio-core ==0.2.1.0, any.unordered-containers ==0.2.20, - unordered-containers -debug, any.utf8-string ==1.0.2, any.uuid ==1.3.16, any.uuid-types ==1.0.6, any.validation ==1.1.3, any.vault ==0.3.1.5, - vault +useghc, - any.vector ==0.13.2.0, - vector +boundschecks -internalchecks -unsafechecks -wall, + any.vector ==0.13.1.0, any.vector-algorithms ==0.9.0.2, - vector-algorithms +bench +boundschecks -internalchecks -llvm +properties -unsafechecks, any.vector-binary-instances ==0.2.5.2, any.vector-sized ==1.6.1, any.vector-stream ==0.1.0.1, any.vector-th-unbox ==0.2.2, any.void ==0.7.3, - void -safe, any.wai ==3.2.4, any.wai-app-static ==3.1.9, - wai-app-static +crypton -print, any.wai-cors ==0.2.7, any.wai-extra ==3.1.15, - wai-extra -build-example, - any.wai-logger ==2.5.0, + any.wai-logger ==2.4.1, any.wai-middleware-throttle ==0.3.0.1, any.wai-middleware-validation ==0.1.0.2, - any.warp ==3.4.3, - warp +allow-sendfilefd -network-bytestring -warp-debug +x509, - any.warp-tls ==3.4.7, + any.warp ==3.4.1, + any.warp-tls ==3.4.6, any.wherefrom-compat ==0.1.1.1, any.wide-word ==0.1.6.0, any.witherable ==0.5, any.word8 ==0.1.3, any.wreq ==0.5.4.3, - wreq -aws -developer +doctest -httpbin, any.yaml ==0.11.11.2, - yaml +no-examples +no-exe, any.yet-another-logger ==0.4.2, - yet-another-logger -tbmqueue, any.zigzag ==0.1.0.0, - any.zlib ==0.7.1.0, - zlib -bundled-c-zlib +non-blocking-ffi +pkg-config -index-state: hackage.haskell.org 2024-11-02T14:34:14Z + any.zlib ==0.7.1.0 +index-state: hackage.haskell.org 2024-11-04T15:41:43Z From 343b6e3d9b2c771bb6a9b757a337fca1ec824e91 Mon Sep 17 00:00:00 2001 From: Lars Kuhtz Date: Mon, 4 Nov 2024 08:49:44 -0800 Subject: [PATCH 05/19] update pact pin --- cabal.project | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cabal.project b/cabal.project index a092a70f46..24684c045b 100644 --- a/cabal.project +++ b/cabal.project @@ -80,7 +80,7 @@ package yet-another-logger source-repository-package type: git location: https://github.com/kadena-io/pact.git - tag: 8d51f4b251c22ecf623a2a49d26016c75bc728fd + tag: 1027a1f5fd0439c58522921e3a0532c4f5867a24 --sha256: 18xgvzb3p8chch85747ln9a2191df09vwwrv9v3njr2h69n3rhxj source-repository-package From cd7b4dc189ea080c71f8e97fe8afff719ecea8ab Mon Sep 17 00:00:00 2001 From: chessai Date: Mon, 4 Nov 2024 10:56:49 -0600 Subject: [PATCH 06/19] update nix hackage index Change-Id: I1f311862b5f2e66d659a17ac40b3d2844218b422 --- cabal.project.freeze | 78 +++++++++++++++++++++----------------------- flake.lock | 6 ++-- 2 files changed, 40 insertions(+), 44 deletions(-) diff --git a/cabal.project.freeze b/cabal.project.freeze index 84b4cdf658..235fa252f3 100644 --- a/cabal.project.freeze +++ b/cabal.project.freeze @@ -26,7 +26,7 @@ constraints: any.Cabal ==3.10.2.0, any.atomic-primops ==0.8.8, any.attoparsec ==0.14.4, any.authenticate-oauth ==1.7, - any.auto-update ==0.2.1, + any.auto-update ==0.2.2, any.base ==4.19.1.0, any.base-compat ==0.14.0, any.base-compat-batteries ==0.14.0, @@ -56,7 +56,7 @@ constraints: any.Cabal ==3.10.2.0, any.cache ==0.1.3.0, any.call-stack ==0.4.0, any.case-insensitive ==1.2.1.0, - any.cassava ==0.5.3.1, + any.cassava ==0.5.3.2, any.cborg ==0.2.10.0, any.cereal ==0.5.8.3, chainweb -debug -ed25519 -ghc-flags, @@ -69,21 +69,21 @@ constraints: any.Cabal ==3.10.2.0, any.code-page ==0.2.1, any.colour ==2.3.6, any.comonad ==5.0.8, - any.conduit ==1.3.5, + any.conduit ==1.3.6, any.configuration-tools ==0.7.0, any.constraints ==0.14.2, any.containers ==0.6.8, any.contiguous ==0.6.4.2, any.contravariant ==1.5.5, any.cookie ==0.5.0, - any.criterion ==1.6.3.0, - any.criterion-measurement ==0.2.2.0, + any.criterion ==1.6.4.0, + any.criterion-measurement ==0.2.3.0, any.crypto-api ==0.13.3, any.crypto-pubkey-types ==0.4.3, - any.crypto-token ==0.1.1, + any.crypto-token ==0.1.2, any.cryptohash-md5 ==0.11.101.0, any.cryptohash-sha1 ==0.11.101.0, - any.crypton ==1.0.0, + any.crypton ==1.0.1, any.crypton-connection ==0.4.1, any.crypton-x509 ==1.7.7, any.crypton-x509-store ==1.6.9, @@ -92,11 +92,8 @@ constraints: any.Cabal ==3.10.2.0, any.cryptonite ==0.30, any.cuckoo ==0.3.1, any.data-bword ==0.1.0.2, - any.data-default ==0.7.1.1, - any.data-default-class ==0.1.2.0, - any.data-default-instances-containers ==0.0.1, - any.data-default-instances-dlist ==0.0.1, - any.data-default-instances-old-locale ==0.0.1, + any.data-default ==0.8.0.0, + any.data-default-class ==0.2.0.0, any.data-dword ==0.3.2.1, any.data-fix ==0.3.4, any.data-ordlist ==0.4.7.0, @@ -116,13 +113,13 @@ constraints: any.Cabal ==3.10.2.0, any.ethereum ==0.1.0.2, ethereum -ethhash -openssl-use-pkg-config, any.exceptions ==0.10.7, - any.fast-logger ==3.2.3, + any.fast-logger ==3.2.5, any.file-embed ==0.0.16.0, any.filepath ==1.4.200.1, any.fingertree ==0.1.5.0, - any.finite-typelits ==0.2.0.1, + any.finite-typelits ==0.2.1.0, any.free ==5.2, - any.generic-data ==1.1.0.0, + any.generic-data ==1.1.0.1, any.generically ==0.1.1, any.ghc-bignum ==1.3, any.ghc-boot-th ==9.8.2, @@ -133,8 +130,8 @@ constraints: any.Cabal ==3.10.2.0, any.growable-vector ==0.1, any.half ==0.3.1, any.hashable ==1.5.0.0, - any.hashes ==0.3.0, - any.haskell-lexer ==1.1.1, + any.hashes ==0.3.0.1, + any.haskell-lexer ==1.1.2, any.haskell-src-exts ==1.23.1, any.haskell-src-meta ==0.8.14, any.heaps ==0.4, @@ -144,9 +141,9 @@ constraints: any.Cabal ==3.10.2.0, any.http-client-tls ==0.3.6.3, any.http-date ==0.0.11, any.http-media ==0.8.1.1, - any.http-semantics ==0.2.0, + any.http-semantics ==0.2.1, any.http-types ==0.12.4, - any.http2 ==5.3.1, + any.http2 ==5.3.4, any.indexed-list-literals ==0.2.1.3, any.indexed-traversable ==0.1.4, any.indexed-traversable-instances ==0.1.2, @@ -154,7 +151,7 @@ constraints: any.Cabal ==3.10.2.0, any.integer-gmp ==1.1, any.integer-logarithms ==1.0.3.1, any.invariant ==0.6.3, - any.iproute ==1.7.14, + any.iproute ==1.7.15, any.ixset-typed ==0.5, any.js-chart ==2.9.4.1, any.kan-extensions ==5.2.6, @@ -177,18 +174,18 @@ constraints: any.Cabal ==3.10.2.0, any.mmorph ==1.2.0, any.mod ==0.2.0.1, any.monad-control ==1.0.3.1, - any.mono-traversable ==1.0.17.0, + any.mono-traversable ==1.0.20.0, any.mtl ==2.3.1, any.mtl-compat ==0.2.2, any.mwc-probability ==2.3.1, any.mwc-random ==0.15.1.0, any.natural-arithmetic ==0.2.1.0, - any.network ==3.1.4.0, + any.network ==3.2.4.0, any.network-byte-order ==0.1.7, any.network-control ==0.1.3, any.network-info ==0.2.1, any.network-uri ==2.6.4.2, - any.nothunks ==0.2.1.1, + any.nothunks ==0.3.0.0, any.old-locale ==1.0.0.7, any.old-time ==1.1.0.4, any.optparse-applicative ==0.18.1.0, @@ -201,7 +198,7 @@ constraints: any.Cabal ==3.10.2.0, any.parallel ==3.2.2.0, any.parsec ==3.1.17.0, any.parser-combinators ==1.3.0, - any.parsers ==0.12.11, + any.parsers ==0.12.12, any.patience ==0.3, any.pem ==0.2.4, any.poly ==0.5.1.0, @@ -243,10 +240,10 @@ constraints: any.Cabal ==3.10.2.0, any.semigroups ==0.20, any.semirings ==0.7, any.serialise ==0.2.6.1, - any.servant ==0.20.1, - any.servant-client ==0.20, - any.servant-client-core ==0.20, - any.servant-server ==0.20, + any.servant ==0.20.2, + any.servant-client ==0.20.2, + any.servant-client-core ==0.20.2, + any.servant-server ==0.20.2, any.sha-validation ==0.1.0.1, any.show-combinators ==0.2.0.0, any.simple-sendfile ==0.2.32, @@ -264,11 +261,10 @@ constraints: any.Cabal ==3.10.2.0, any.streaming-commons ==0.2.2.6, any.strict ==0.5.1, any.strict-concurrency ==0.2.4.3, - any.string-conversions ==0.4.0.1, any.syb ==0.7.2.4, any.system-cxx-std-lib ==1.0, any.tagged ==0.8.8, - any.tasty ==1.5, + any.tasty ==1.5.2, any.tasty-golden ==2.3.5, any.tasty-hunit ==0.10.2, any.tasty-json ==0.1.0.0, @@ -281,16 +277,16 @@ constraints: any.Cabal ==3.10.2.0, any.th-abstraction ==0.7.0.0, any.th-compat ==0.1.5, any.th-expand-syns ==0.4.11.0, - any.th-lift ==0.8.4, - any.th-orphans ==0.13.14, + any.th-lift ==0.8.5, + any.th-orphans ==0.13.15, any.th-reify-many ==0.1.10, any.these ==1.2.1, any.time ==1.12.2, any.time-compat ==1.9.7, any.time-locale-compat ==0.1.1.5, any.time-manager ==0.1.0, - any.tls ==2.1.0, - any.tls-session-manager ==0.0.6, + any.tls ==2.1.3, + any.tls-session-manager ==0.0.7, any.token-bucket ==0.1.0.1, any.torsor ==0.1.0.1, any.transformers ==0.6.1.0, @@ -298,11 +294,11 @@ constraints: any.Cabal ==3.10.2.0, any.transformers-compat ==0.7.2, any.trifecta ==2.1.4, any.tuples ==0.1.0.0, - any.typed-process ==0.2.11.1, + any.typed-process ==0.2.12.0, any.uniplate ==1.6.13, any.unix ==2.8.4.0, - any.unix-compat ==0.7.2, - any.unix-time ==0.4.15, + any.unix-compat ==0.7.3, + any.unix-time ==0.4.16, any.unlifted ==0.2.2.0, any.unliftio ==0.2.25.0, any.unliftio-core ==0.2.1.0, @@ -312,7 +308,7 @@ constraints: any.Cabal ==3.10.2.0, any.uuid-types ==1.0.6, any.validation ==1.1.3, any.vault ==0.3.1.5, - any.vector ==0.13.1.0, + any.vector ==0.13.2.0, any.vector-algorithms ==0.9.0.2, any.vector-binary-instances ==0.2.5.2, any.vector-sized ==1.6.1, @@ -323,11 +319,11 @@ constraints: any.Cabal ==3.10.2.0, any.wai-app-static ==3.1.9, any.wai-cors ==0.2.7, any.wai-extra ==3.1.15, - any.wai-logger ==2.4.1, + any.wai-logger ==2.5.0, any.wai-middleware-throttle ==0.3.0.1, any.wai-middleware-validation ==0.1.0.2, - any.warp ==3.4.1, - any.warp-tls ==3.4.6, + any.warp ==3.4.3, + any.warp-tls ==3.4.7, any.wherefrom-compat ==0.1.1.1, any.wide-word ==0.1.6.0, any.witherable ==0.5, diff --git a/flake.lock b/flake.lock index af40055c27..5a719104cd 100644 --- a/flake.lock +++ b/flake.lock @@ -85,11 +85,11 @@ "hackage": { "flake": false, "locked": { - "lastModified": 1722385397, - "narHash": "sha256-rccC2HsYG7SUEo5dhLRhwx7RWRotvlzeF/TZ3IU4mZY=", + "lastModified": 1730680170, + "narHash": "sha256-CUPGIJ4PMrGKVC30bZfCrlzvTvlAjvz2bQ091DKqNNk=", "owner": "input-output-hk", "repo": "hackage.nix", - "rev": "cb8f6dc140b3bcfe14589f5a191fa912bef2464f", + "rev": "3f9db9843a52f45de2e9884838de31fe7c526c75", "type": "github" }, "original": { From c0312c13a731b0cf8ac8227e9d18ee51442ac326 Mon Sep 17 00:00:00 2001 From: chessai Date: Tue, 12 Nov 2024 19:45:09 -0600 Subject: [PATCH 07/19] correct node shutdown message due to old ChainwebVersion Change-Id: Id45bc33f9fbdc799126df13c091961b03b5b1024 --- node/src/ChainwebNode.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/node/src/ChainwebNode.hs b/node/src/ChainwebNode.hs index a0c375ee3c..2cca19a610 100644 --- a/node/src/ChainwebNode.hs +++ b/node/src/ChainwebNode.hs @@ -520,7 +520,7 @@ withServiceDate v lf msd inner = case msd of shutdownMessage :: Text shutdownMessage = T.concat - [ "Shutting down. This version of chainweb was only valid until" <> sshow t <> "." + [ "Shutting down. This version of chainweb was only valid until " <> sshow t <> "." , " Please upgrade to a new version." ] From f7846228795ec93be82ae08ffce60ce4999e19f5 Mon Sep 17 00:00:00 2001 From: chessai Date: Wed, 6 Nov 2024 14:05:43 -0600 Subject: [PATCH 08/19] increment p2p successive failure count of failed peer syncs Change-Id: Ieca25025c414918bf1e7b2b2306b0764b0110a0e --- src/P2P/Node.hs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/P2P/Node.hs b/src/P2P/Node.hs index 429be73827..353d92076f 100644 --- a/src/P2P/Node.hs +++ b/src/P2P/Node.hs @@ -445,6 +445,10 @@ syncFromPeer node info = do return False | otherwise -> do logg node Warn $ "failed to sync peers from " <> showInfo info <> ": " <> showClientError e + -- incrementSuccessiveFailures here helps reduce redundant synchronisation attempts + -- by a significant margin, from limited experimentation. More in-depth experimentation + -- is still to be done. + incrementSuccessiveFailures peerDb info return False Right p -> do peers <- peerDbSnapshot peerDb @@ -510,7 +514,7 @@ findNextPeer conf node = do -- random circular shift of a set let shift i = uncurry (++) . swap - . splitAt (fromIntegral i) + . splitAt i shiftR s = do i <- nodeRandomR node (0, max 1 (length s) - 1) From 5290cf1bc1144524789ce96988428d491090620b Mon Sep 17 00:00:00 2001 From: Edmund Noble Date: Mon, 4 Nov 2024 13:59:57 -0500 Subject: [PATCH 09/19] relocate base64-bytestring-kadena to kadena fork --- cabal.project | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cabal.project b/cabal.project index 24684c045b..a82cc62fa3 100644 --- a/cabal.project +++ b/cabal.project @@ -139,9 +139,9 @@ source-repository-package -- Required for non-canonical decode in base64-bytestring (remove after 2.20 fork) source-repository-package type: git - location: https://github.com/emilypi/base64-bytestring-kadena - tag: 174af3523616c8fe01449da5ccbb9f16df097ac3 - --sha256: 0grssxgjygi4ijiqaa87diwv6sdqz77hldf3lr4k7p93w35lhlci + location: https://github.com/kadena-io/base64-bytestring-kadena + tag: 90247042ab3b8662809210af2a78e6dee0f9b4ac + --sha256: 0dqsrjxm0cm35xcihm49dhwdvmz79vsv4sd5qs2izc4sbnd0d8n6 -- -------------------------------------------------------------------------- -- -- Relaxed Bounds From 5f67ad35a44db0ef05ad77e5d705b3f964ee8b69 Mon Sep 17 00:00:00 2001 From: Edmund Noble Date: Wed, 30 Oct 2024 13:15:55 -0400 Subject: [PATCH 10/19] delete some unnecessary indices in the peerdb --- src/P2P/Node/PeerDB.hs | 8 +------- src/P2P/Node/RestAPI/Server.hs | 10 +++++----- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/P2P/Node/PeerDB.hs b/src/P2P/Node/PeerDB.hs index 12e677d0b8..35d4a250a6 100644 --- a/src/P2P/Node/PeerDB.hs +++ b/src/P2P/Node/PeerDB.hs @@ -194,11 +194,8 @@ hostAddressIdx = HostAddressIdx . xor pdNonce . hash {-# INLINE hostAddressIdx #-} type PeerEntryIxs = - '[ HostAddressIdx - , HostAddress + '[ HostAddress -- a primary index - , Maybe PeerId - -- unique index in the 'Just' values, but not in the 'Nothing' values , SuccessiveFailures , LastSuccess , NetworkId @@ -207,9 +204,7 @@ type PeerEntryIxs = instance Indexable PeerEntryIxs PeerEntry where indices = ixList - (ixFun $ \e -> [hostAddressIdx $ _peerAddr $ _peerEntryInfo e]) (ixFun $ \e -> [_peerAddr $ _peerEntryInfo e]) - (ixFun $ \e -> [_peerId $ _peerEntryInfo e]) (ixFun $ \e -> [_peerEntrySuccessiveFailures e]) (ixFun $ \e -> [_peerEntryLastSuccess e]) (ixFun $ \e -> F.toList (_peerEntryNetworkIds e)) @@ -466,4 +461,3 @@ somePeerDbVal (FromSingChainwebVersion (SChainwebVersion :: Sing v)) n db = f n f (FromSingNetworkId (SChainNetwork SChainId :: Sing n)) = SomePeerDb $ PeerDbT @v @n db f (FromSingNetworkId (SMempoolNetwork SChainId :: Sing n)) = SomePeerDb $ PeerDbT @v @n db f (FromSingNetworkId (SCutNetwork :: Sing n)) = SomePeerDb $ PeerDbT @v @n db - diff --git a/src/P2P/Node/RestAPI/Server.hs b/src/P2P/Node/RestAPI/Server.hs index e3f07ed153..7e37e79ca1 100644 --- a/src/P2P/Node/RestAPI/Server.hs +++ b/src/P2P/Node/RestAPI/Server.hs @@ -44,7 +44,7 @@ import Control.Monad.Except (throwError) import Control.Monad.IO.Class import Data.Bifunctor -import Data.IxSet.Typed (getEQ, toAscList) +import qualified Data.IxSet.Typed as IXS import Data.Proxy import qualified Data.Text.IO as T @@ -95,9 +95,10 @@ peerGetHandler db nid limit next = do . SP.map (second _peerEntryInfo) . SP.zip (SP.each [0..]) . SP.each - . toAscList (Proxy @HostAddressIdx) - . getEQ (SuccessiveFailures 0) - $ getEQ nid sn + . IXS.toList + . IXS.getEQ (SuccessiveFailures 0) + . IXS.getEQ nid + $ sn return $! over pageItems (fmap snd) page where effectiveLimit = min @@ -192,4 +193,3 @@ serveP2pSocket -> [(NetworkId, PeerDb)] -> IO () serveP2pSocket s sock v = runSettingsSocket s sock . someServerApplication . someP2pServers v - From cab1674ce9e25c2aba3690c219351eb7c049988a Mon Sep 17 00:00:00 2001 From: Edmund Noble Date: Wed, 30 Oct 2024 12:47:42 -0400 Subject: [PATCH 11/19] use M.size for session count --- src/P2P/Node.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/P2P/Node.hs b/src/P2P/Node.hs index 353d92076f..09974faf32 100644 --- a/src/P2P/Node.hs +++ b/src/P2P/Node.hs @@ -559,7 +559,7 @@ findNextPeer conf node = do peers <- peerDbSnapshotSTM peerDbVar !sessions <- readTVar sessionsVar let peerCount = length peers - let sessionCount = length sessions + let sessionCount = M.size sessions -- Check that there are peers -- From c3130c995dcb4cd3923d8d6771acd892cf5d18b1 Mon Sep 17 00:00:00 2001 From: Edmund Noble Date: Wed, 30 Oct 2024 12:47:42 -0400 Subject: [PATCH 12/19] order peersList by SuccessiveFailures --- src/P2P/Node.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/P2P/Node.hs b/src/P2P/Node.hs index 09974faf32..61e77d4e9e 100644 --- a/src/P2P/Node.hs +++ b/src/P2P/Node.hs @@ -128,6 +128,7 @@ import P2P.Node.PeerDB import P2P.Node.RestAPI.Client import P2P.Peer import P2P.Session +import Data.Proxy -- -------------------------------------------------------------------------- -- -- P2pNodeStats @@ -580,7 +581,7 @@ findNextPeer conf node = do -- transation we only check that the result is not empty, which -- is expected to be much cheaper than forcing the full list. -- - peersList = IXS.toList peers + peersList = IXS.toAscList (Proxy @SuccessiveFailures) peers candidates = flip filter peersList $ \peer -> let addr = _peerAddr (_peerEntryInfo peer) in From 4ecc5a999742e55d61c15d1c42af37362b4d4a23 Mon Sep 17 00:00:00 2001 From: Edmund Noble Date: Mon, 11 Nov 2024 12:09:13 -0500 Subject: [PATCH 13/19] Set server name --- src/Chainweb/Chainweb.hs | 1 + src/Chainweb/Chainweb/CheckReachability.hs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Chainweb/Chainweb.hs b/src/Chainweb/Chainweb.hs index f596ffc8b9..7308a6e3da 100644 --- a/src/Chainweb/Chainweb.hs +++ b/src/Chainweb/Chainweb.hs @@ -892,6 +892,7 @@ runChainweb cw nowServing = do & setOnException (logWarpException "Service API" clientClosedConnectionsCounter) & setBeforeMainLoop (nowServing (nowServingServiceAPI .~ True)) + & setServerName "Chainweb Service API" serviceApiHost = _serviceApiConfigInterface $ _configServiceApi $ _chainwebConfig cw diff --git a/src/Chainweb/Chainweb/CheckReachability.hs b/src/Chainweb/Chainweb/CheckReachability.hs index dd5b654197..1480ef00f5 100644 --- a/src/Chainweb/Chainweb/CheckReachability.hs +++ b/src/Chainweb/Chainweb/CheckReachability.hs @@ -149,4 +149,5 @@ peerServerSettings :: Peer -> W.Settings peerServerSettings peer = W.setPort (int . _hostAddressPort . _peerAddr $ _peerInfo peer) . W.setHost (_peerInterface peer) + . W.setServerName "Chainweb P2P API" $ W.defaultSettings From 362ade6592724d0d256a50c6da921af7ae2c1eda Mon Sep 17 00:00:00 2001 From: Edmund Noble Date: Tue, 5 Nov 2024 11:38:27 -0500 Subject: [PATCH 14/19] Make RuleZipper to replace Measurement This generalizes the "measurement" idea so that measurements of rules are represented by zippers, allowing the caller to navigate around a measurement themselves. --- src/Chainweb/BlockHeader/Internal.hs | 4 +- src/Chainweb/RestAPI/NodeInfo.hs | 5 +- src/Chainweb/Utils/Rule.hs | 148 ++++++++++----------- src/Chainweb/Version.hs | 37 +++--- src/Chainweb/Version/Development.hs | 8 +- src/Chainweb/Version/Guards.hs | 6 +- src/Chainweb/Version/Mainnet.hs | 9 +- src/Chainweb/Version/RecapDevelopment.hs | 6 +- src/Chainweb/Version/Testnet.hs | 6 +- src/Chainweb/Version/Utils.hs | 9 +- test/lib/Chainweb/Test/Orphans/Internal.hs | 2 +- test/lib/Chainweb/Test/TestVersions.hs | 12 +- test/unit/Chainweb/Test/Version.hs | 7 +- 13 files changed, 128 insertions(+), 131 deletions(-) diff --git a/src/Chainweb/BlockHeader/Internal.hs b/src/Chainweb/BlockHeader/Internal.hs index 99f4a39668..4efa013a20 100644 --- a/src/Chainweb/BlockHeader/Internal.hs +++ b/src/Chainweb/BlockHeader/Internal.hs @@ -699,7 +699,7 @@ makeGenesisBlockHeader' makeGenesisBlockHeader' v p ct@(BlockCreationTime t) n = fromLog @ChainwebMerkleHashAlgorithm mlog where - g = genesisGraph v p + (h, g) = genesisHeightAndGraph v p cid = _chainId p mlog = newMerkleLog @@ -710,7 +710,7 @@ makeGenesisBlockHeader' v p ct@(BlockCreationTime t) n = :+: genesisBlockPayloadHash v cid :+: cid :+: BlockWeight 0 - :+: genesisBlockHeight v cid -- because of chain graph changes (new chains) not all chains start at 0 + :+: h -- because of chain graph changes (new chains) not all chains start at 0 :+: _versionCode v :+: EpochStartTime t :+: n diff --git a/src/Chainweb/RestAPI/NodeInfo.hs b/src/Chainweb/RestAPI/NodeInfo.hs index 945083b231..60de9b0aa8 100644 --- a/src/Chainweb/RestAPI/NodeInfo.hs +++ b/src/Chainweb/RestAPI/NodeInfo.hs @@ -95,7 +95,7 @@ nodeInfoHandler v (SomeCutDb (CutDbT db :: CutDbT cas v)) = do , nodeGraphHistory = graphs , nodeLatestBehaviorHeight = latestBehaviorAt v , nodeGenesisHeights = map (\c -> (chainIdToText c, genesisHeight v c)) $ HS.toList (chainIds v) - , nodeHistoricalChains = ruleElems 0 $ fmap (HM.toList . HM.map HS.toList . toAdjacencySets) $ _versionGraphs v + , nodeHistoricalChains = ruleElems $ fmap (HM.toList . HM.map HS.toList . toAdjacencySets) $ _versionGraphs v , nodeServiceDate = T.pack <$> _versionServiceDate v , nodeBlockDelay = _versionBlockDelay v } @@ -105,7 +105,6 @@ nodeInfoHandler v (SomeCutDb (CutDbT db :: CutDbT cas v)) = do unpackGraphs :: ChainwebVersion -> [(BlockHeight, [(Int, [Int])])] unpackGraphs v = gs where - gs = map (second graphAdjacencies) $ NE.toList $ ruleElems (BlockHeight 0) $ _versionGraphs v + gs = map (second graphAdjacencies) $ NE.toList $ ruleElems $ _versionGraphs v graphAdjacencies = map unChain . HM.toList . fmap HS.toList . G.adjacencySets . view chainGraphGraph unChain (a, bs) = (chainIdInt a, map chainIdInt bs) - diff --git a/src/Chainweb/Utils/Rule.hs b/src/Chainweb/Utils/Rule.hs index b6abe72f20..9499749dbf 100644 --- a/src/Chainweb/Utils/Rule.hs +++ b/src/Chainweb/Utils/Rule.hs @@ -6,7 +6,21 @@ {-# language LambdaCase #-} {-# language TupleSections #-} -module Chainweb.Utils.Rule where +module Chainweb.Utils.Rule + ( Rule(..) + , ruleHead + , ruleDropWhile + , ruleTakeWhile + , ruleValid + , ruleElems + , RuleZipper(..) + , ruleZipperHere + , unzipRule + , ruleZipperFind + , ruleSeek + , ruleZipperDown + + ) where import Control.DeepSeq @@ -29,7 +43,7 @@ import GHC.Generics -- we often lookup chain properties (e.g. forks) where we are interested in the -- latest occurrence. -- -data Rule h a = Above (h, a) (Rule h a) | End a +data Rule h a = Above (h, a) (Rule h a) | Bottom (h, a) deriving stock (Eq, Ord, Show, Foldable, Functor, Generic, Generic1, Traversable) deriving anyclass (Hashable, NFData) @@ -39,29 +53,26 @@ instance Bifunctor Rule where where go = \case Above (h, a) r -> Above (fh h, fa a) (go r) - End a -> End (fa a) + Bottom (h, a) -> Bottom (fh h, fa a) instance Foldable1 (Rule h) where foldMap1 = foldMap1Default instance Traversable1 (Rule h) where - traverse1 f (Above (h, a) t) = Above <$> ((h,) <$> f a) <.> traverse1 f t - traverse1 f (End a) = End <$> f a + traverse1 f (Above (h, a) t) = Above . (h,) <$> f a <.> traverse1 f t + traverse1 f (Bottom (h, a)) = Bottom . (h,) <$> f a instance (ToJSON h, ToJSON a) => ToJSON (Rule h a) where - toJSON = toJSON . go - where - go (Above (h, a) t) = toJSON (toJSON h, toJSON a) : go t - go (End a) = [toJSON a] + toJSON = toJSON . ruleElems instance (FromJSON h, FromJSON a) => FromJSON (Rule h a) where parseJSON = withArray "Rule" $ go . V.toList where go [] = fail "empty list" - go [a] = End <$> parseJSON a + go [a] = Bottom <$> parseJSON a go (x:xs) = Above <$> parseJSON x <*> go xs -ruleHead :: Rule h a -> (Maybe h, a) -ruleHead (Above (h, a) _) = (Just h, a) -ruleHead (End a) = (Nothing, a) +ruleHead :: Rule h a -> (h, a) +ruleHead (Above (h, a) _) = (h, a) +ruleHead (Bottom (h, a)) = (h, a) ruleTakeWhile :: (h -> Bool) -> Rule h a -> Rule h a ruleTakeWhile p (Above (h, a) t) @@ -75,73 +86,58 @@ ruleDropWhile p (Above (h, a) t) | otherwise = Above (h, a) t ruleDropWhile _ t = t --- | A measurement on a rule tells you where a condition starts to be true; at --- the Top, at the Bottom, or Between lower and upper. --- -data Measurement h a = Bottom a | Top (h, a) | Between (h, a) (h, a) - --- | Takes a measurement on a rule using a monotone function. --- -measureRule' :: (h -> Bool) -> Rule h a -> Measurement h a -measureRule' p ((topH, topA) `Above` topTail) - | p topH = Top (topH, topA) - | otherwise = go topH topA topTail - where - go lh la (Above (h, a) t) - | p h = Between (h, a) (lh, la) - | otherwise = go h a t - go _ _ (End a) = Bottom a -measureRule' _ (End a) = Bottom a - -measureRule :: Ord h => h -> Rule h a -> Measurement h a -measureRule h = - measureRule' (\hc -> h >= hc) - --- | Returns the elements of the Rule. --- -ruleElems :: h -> Rule h a -> NE.NonEmpty (h, a) -ruleElems h (End a) = (h, a) NE.:| [] -ruleElems he (Above (h, a) t) = (h, a) `NE.cons` ruleElems he t - --- | Returns the elements of the Rule in ascending order. --- --- This does not stream. Accessing the first element takes O(n) time. --- -ruleElemsAsc :: h -> Rule h a -> NE.NonEmpty (h, a) -ruleElemsAsc he = go [] - where - go acc (End a) = (he, a) NE.:| acc - go acc (Above (h, a) t) = go ((h, a) : acc) t - --- | Measures a monotone condition on a value. It tells you when a condition --- started to be true most recently (it ignores any previous history). --- --- Note that 'Top' provides a lower bound for a possible change and 'Bottom' is --- an upper bound for a change of the evaluation of the condition: It returns --- 'Top' when the condition is false at the latest/current grade. It returns --- 'Bottom' if the the condition was always true. --- -measureValue' :: Num h => (a -> Bool) -> Rule h a -> Measurement h a -measureValue' p ((topH, topA) `Above` topTail) - | not (p topA) = Top (topH, topA) - | otherwise = go topH topA topTail +-- | A zipper on a rule represents a measurement on the rule, either at some +-- point on the rule (including the top) or at the bottom of the rule. +-- Leftmost fields are "below", rightmost fields are "above". +data RuleZipper h a + = BetweenZipper (Rule h a) [(h, a)] + deriving Show + +ruleZipperHere :: RuleZipper h a -> (h, a) +ruleZipperHere (BetweenZipper r _) = ruleHead r + +-- | Construct a zipper at the top of the Rule, O(1). +unzipRule :: Rule h a -> RuleZipper h a +unzipRule r = BetweenZipper r [] +{-# inline unzipRule #-} + +ruleZipperDown :: RuleZipper h a -> RuleZipper h a +ruleZipperDown = \case + BetweenZipper (Bottom t) above -> BetweenZipper (Bottom t) above + BetweenZipper (justBelow `Above` below) above -> + BetweenZipper below (justBelow : above) + +-- | Find the place in the rule zipper that satisfies the condition. +-- Note that if the condition is never reached, the rule is returned reversed. +-- O(length(untrue prefix)). +ruleZipperFind :: (h -> a -> Bool) -> RuleZipper h a -> (Bool, RuleZipper h a) +ruleZipperFind p = go where - go lh la (Above (h, a) t) - | not (p a) = Between (h, a) (lh, la) - | otherwise = go h a t - go lh la (End a) - | not (p a) = Between (0, a) (lh, la) - | otherwise = Bottom a -measureValue' _ (End a) = Bottom a - --- | Measures when a value most recently dropped blow the given threshold. + go pl@(BetweenZipper below above) + | (h, a) <- ruleHead below, p h a = + (True, pl) + | Bottom {} <- below = + (False, pl) + | justBelow `Above` wayBelow <- below = + go (BetweenZipper wayBelow (justBelow : above)) +{-# inline ruleZipperFind #-} + +-- | Find the place in the rule that satisfies the condition, and +-- return it as a zipper. +-- Note that if it reaches the bottom, the bottom is returned. +-- O(length(untrue prefix)). +ruleSeek :: (h -> a -> Bool) -> Rule h a -> (Bool, RuleZipper h a) +ruleSeek p = ruleZipperFind p . unzipRule +{-# inline ruleSeek #-} + +-- | Returns the elements of the Rule. O(n) and lazily produces elements. -- -measureValue :: Num h => Ord a => a -> Rule h a -> Measurement h a -measureValue a = - measureValue' (\ac -> a >= ac) +ruleElems :: Rule h a -> NE.NonEmpty (h, a) +ruleElems (Bottom (h, a)) = (h, a) NE.:| [] +ruleElems (Above (h, a) t) = (h, a) `NE.cons` ruleElems t -- | Checks that a Rule is decreasing, and thus valid. --- +-- O(n). ruleValid :: Ord h => Rule h a -> Bool ruleValid (Above (h, _) t@(Above (h', _) _)) = h > h' && ruleValid t ruleValid _ = True diff --git a/src/Chainweb/Version.hs b/src/Chainweb/Version.hs index 8942edc1e7..dbf6d196c2 100644 --- a/src/Chainweb/Version.hs +++ b/src/Chainweb/Version.hs @@ -74,7 +74,7 @@ module Chainweb.Version , genesisBlockTarget , genesisTime , genesisBlockHeight - , genesisGraph + , genesisHeightAndGraph -- * Typelevel ChainwebVersionName , ChainwebVersionT(..) @@ -603,11 +603,8 @@ instance HasChainGraph (ChainwebVersion, BlockHeight) where -- It is enforced via the 'mkChainId' smart constructor for ChainId.) -- genesisBlockHeight :: HasCallStack => ChainwebVersion -> ChainId -> BlockHeight -genesisBlockHeight v c = - case measureValue' (flip isWebChain c) $ _versionGraphs v of - Top _ -> error $ "Invalid ChainId " <> show c - Between _ (h, _) -> h - Bottom _ -> BlockHeight 0 +genesisBlockHeight v c = fst $ genesisHeightAndGraph v c +{-# inlinable genesisBlockHeight #-} -- | The genesis graph for a given Chain -- @@ -617,18 +614,29 @@ genesisBlockHeight v c = -- (We generally assume that this invariant holds throughout the code base. -- It is enforced via the 'mkChainId' smart constructor for ChainId.) -- -genesisGraph +genesisHeightAndGraph :: HasCallStack => HasChainwebVersion v => HasChainId c => v -> c - -> ChainGraph -genesisGraph v c = - case measureValue' (flip isWebChain c) $ _versionGraphs (_chainwebVersion v) of - Top _ -> error $ "Invalid ChainId " <> show (_chainId c) - Between _ (_, a) -> a - Bottom a -> a + -> (BlockHeight, ChainGraph) +genesisHeightAndGraph v c = + case ruleSeek (\_ g -> not (isWebChain g c)) (_versionGraphs (_chainwebVersion v)) of + -- the chain was in every graph down to the bottom, + -- so the bottom has the genesis graph + (False, z) -> ruleZipperHere z + (True, (BetweenZipper _ above)) + -- the chain is not in this graph, and there is no graph above + -- which could have it + | [] <- above -> missingChainError + -- the chain is not in this graph but the graph above does have it + | (t:_) <- above -> t + where + missingChainError = error + $ "Invalid ChainId " <> show (_chainId c) + <> " for chainweb version " <> show (_versionName (_chainwebVersion v)) +{-# inlinable genesisHeightAndGraph #-} -------------------------------------------------------------------------- -- -- Utilities for constructing chainweb versions @@ -667,6 +675,5 @@ latestBehaviorAt v = foldlOf' behaviorChanges max 0 v + 1 behaviorChanges = fold [ versionForks . folded . folded . _ForkAtBlockHeight , versionUpgrades . folded . ifolded . asIndex - , versionGraphs . to ruleHead . _1 . _Just + , versionGraphs . to ruleHead . _1 ] - diff --git a/src/Chainweb/Version/Development.hs b/src/Chainweb/Version/Development.hs index aec4736721..6867a0c650 100644 --- a/src/Chainweb/Version/Development.hs +++ b/src/Chainweb/Version/Development.hs @@ -33,7 +33,7 @@ devnet = ChainwebVersion , _versionName = ChainwebVersionName "development" , _versionForks = tabulateHashMap $ \_ -> AllChains ForkAtGenesis , _versionUpgrades = AllChains mempty - , _versionGraphs = End twentyChainGraph + , _versionGraphs = Bottom (minBound, twentyChainGraph) , _versionBlockDelay = BlockDelay 30_000_000 , _versionWindow = WindowWidth 120 , _versionHeaderBaseSizeBytes = 318 - 110 @@ -49,7 +49,7 @@ devnet = ChainwebVersion -- still the *default* block gas limit is set, see -- defaultChainwebConfiguration._configBlockGasLimit - , _versionMaxBlockGasLimit = End Nothing + , _versionMaxBlockGasLimit = Bottom (minBound, Nothing) , _versionCheats = VersionCheats { _disablePow = True , _fakeFirstEpochStart = True @@ -59,8 +59,8 @@ devnet = ChainwebVersion { _disablePeerValidation = True , _disableMempoolSync = False } - , _versionVerifierPluginNames = AllChains $ End - $ Set.fromList $ map VerifierName ["hyperlane_v3_message", "allow"] + , _versionVerifierPluginNames = AllChains $ Bottom + (minBound, Set.fromList $ map VerifierName ["hyperlane_v3_message", "allow"]) , _versionQuirks = noQuirks , _versionServiceDate = Nothing } diff --git a/src/Chainweb/Version/Guards.hs b/src/Chainweb/Version/Guards.hs index 0a56207a0f..acc8f05c47 100644 --- a/src/Chainweb/Version/Guards.hs +++ b/src/Chainweb/Version/Guards.hs @@ -267,10 +267,8 @@ pactParserVersion v cid bh | otherwise = PactParserGenesis maxBlockGasLimit :: ChainwebVersion -> BlockHeight -> Maybe Natural -maxBlockGasLimit v bh = case measureRule bh $ _versionMaxBlockGasLimit v of - Bottom limit -> limit - Top (_, limit) -> limit - Between (_, limit) _ -> limit +maxBlockGasLimit v bh = snd $ ruleZipperHere $ snd + $ ruleSeek (\h _ -> bh >= h) (_versionMaxBlockGasLimit v) -- | Different versions of Chainweb allow different PPKSchemes. diff --git a/src/Chainweb/Version/Mainnet.hs b/src/Chainweb/Version/Mainnet.hs index f8f9cfb75d..6aae0d9556 100644 --- a/src/Chainweb/Version/Mainnet.hs +++ b/src/Chainweb/Version/Mainnet.hs @@ -155,13 +155,13 @@ mainnet = ChainwebVersion , _versionGraphs = (to20ChainsMainnet, twentyChainGraph) `Above` - End petersonChainGraph + Bottom (minBound, petersonChainGraph) , _versionBlockDelay = BlockDelay 30_000_000 , _versionWindow = WindowWidth 120 , _versionHeaderBaseSizeBytes = 318 - 110 , _versionMaxBlockGasLimit = (succ $ mainnet ^?! versionForks . at Chainweb216Pact . _Just . onChain (unsafeChainId 0) . _ForkAtBlockHeight, Just 180_000) `Above` - End Nothing + Bottom (minBound, Nothing) , _versionBootstraps = domainAddr2PeerInfo mainnetBootstrapHosts , _versionGenesis = VersionGenesis { _genesisBlockTarget = OnChains $ HM.fromList $ concat @@ -213,8 +213,9 @@ mainnet = ChainwebVersion { _disablePeerValidation = False , _disableMempoolSync = False } - , _versionVerifierPluginNames = AllChains $ (4_577_530, Set.fromList $ map VerifierName ["hyperlane_v3_message"]) `Above` - End mempty + , _versionVerifierPluginNames = AllChains $ + (4_577_530, Set.fromList $ map VerifierName ["hyperlane_v3_message"]) `Above` + Bottom (minBound, mempty) , _versionQuirks = VersionQuirks { _quirkGasFees = HM.fromList [ (fromJuste (decodeStrictOrThrow' "\"s9fUspNaCHoV4rNI-Tw-JYU1DxqZAOXS-80oEy7Zfbo\""), Gas 67_618) diff --git a/src/Chainweb/Version/RecapDevelopment.hs b/src/Chainweb/Version/RecapDevelopment.hs index 0fbd6236dd..c671106df3 100644 --- a/src/Chainweb/Version/RecapDevelopment.hs +++ b/src/Chainweb/Version/RecapDevelopment.hs @@ -89,7 +89,7 @@ recapDevnet = ChainwebVersion , _versionGraphs = (to20ChainsHeight, twentyChainGraph) `Above` - End petersonChainGraph + Bottom (minBound, petersonChainGraph) , _versionBlockDelay = BlockDelay 30_000_000 , _versionWindow = WindowWidth 120 @@ -108,7 +108,7 @@ recapDevnet = ChainwebVersion ] } - , _versionMaxBlockGasLimit = End (Just 180_000) + , _versionMaxBlockGasLimit = Bottom (minBound, Just 180_000) , _versionCheats = VersionCheats { _disablePow = False , _fakeFirstEpochStart = True @@ -120,7 +120,7 @@ recapDevnet = ChainwebVersion } , _versionVerifierPluginNames = AllChains $ (600, Set.fromList $ map VerifierName ["hyperlane_v3_message", "allow"]) `Above` - End mempty + Bottom (minBound, mempty) , _versionQuirks = noQuirks , _versionServiceDate = Nothing } diff --git a/src/Chainweb/Version/Testnet.hs b/src/Chainweb/Version/Testnet.hs index 296877fb52..7cf11ab8bf 100644 --- a/src/Chainweb/Version/Testnet.hs +++ b/src/Chainweb/Version/Testnet.hs @@ -135,13 +135,13 @@ testnet = ChainwebVersion , _versionGraphs = (to20ChainsTestnet, twentyChainGraph) `Above` - End petersonChainGraph + Bottom (minBound, petersonChainGraph) , _versionBlockDelay = BlockDelay 30_000_000 , _versionWindow = WindowWidth 120 , _versionHeaderBaseSizeBytes = 318 - 110 , _versionMaxBlockGasLimit = (succ $ testnet ^?! versionForks . at Chainweb216Pact . _Just . onChain (unsafeChainId 0) . _ForkAtBlockHeight, Just 180_000) `Above` - End Nothing + Bottom (minBound, Nothing) , _versionBootstraps = domainAddr2PeerInfo testnetBootstrapHosts , _versionGenesis = VersionGenesis { _genesisBlockTarget = OnChains $ HM.fromList $ concat @@ -185,7 +185,7 @@ testnet = ChainwebVersion , _disableMempoolSync = False } , _versionVerifierPluginNames = AllChains $ (4_100_681, Set.fromList $ map VerifierName ["hyperlane_v3_message"]) `Above` - End mempty + Bottom (minBound, mempty) , _versionQuirks = VersionQuirks { _quirkGasFees = HM.fromList [ (fromJuste (decodeStrictOrThrow' "\"myHrgVbYCXlAk8KJbmWHs3TEDSlRKRuzxpFa9yaC7cQ\""), Gas 66239) diff --git a/src/Chainweb/Version/Utils.hs b/src/Chainweb/Version/Utils.hs index 1029ed666d..32674460b9 100644 --- a/src/Chainweb/Version/Utils.hs +++ b/src/Chainweb/Version/Utils.hs @@ -131,9 +131,9 @@ chainGraphs :: HasChainwebVersion v => v -> M.Map BlockHeight ChainGraph chainGraphs = \case (_chainwebVersion -> v) | _versionCode v == _versionCode mainnet -> mainnetGraphs - | otherwise -> M.fromDistinctDescList . toList . ruleElems minBound $ _versionGraphs v + | otherwise -> M.fromDistinctDescList . toList . ruleElems $ _versionGraphs v where - mainnetGraphs = M.fromDistinctDescList . toList . ruleElems minBound $ _versionGraphs mainnet + mainnetGraphs = M.fromDistinctDescList . toList . ruleElems $ _versionGraphs mainnet -- | BlockHeight intervals for the chain graphs of a chainweb version up to a -- given block height. @@ -462,10 +462,7 @@ verifiersAt v cid bh = M.restrictKeys allVerifierPlugins activeVerifierNames where activeVerifierNames = - case measureRule bh $ _versionVerifierPluginNames v ^?! onChain cid of - Bottom vs -> vs - Top (_, vs) -> vs - Between (_, vs) _ -> vs + snd $ ruleZipperHere $ snd $ ruleSeek (\h _ -> bh >= h) $ _versionVerifierPluginNames v ^?! onChain cid -- the mappings from names to verifier plugins is global. the list of verifier -- plugins active in any particular block validation context is the only thing diff --git a/test/lib/Chainweb/Test/Orphans/Internal.hs b/test/lib/Chainweb/Test/Orphans/Internal.hs index 777b854097..f7438d891d 100644 --- a/test/lib/Chainweb/Test/Orphans/Internal.hs +++ b/test/lib/Chainweb/Test/Orphans/Internal.hs @@ -296,7 +296,7 @@ instance Arbitrary NodeInfo where , nodeGraphHistory = graphs , nodeLatestBehaviorHeight = latestBehaviorAt v , nodeGenesisHeights = map (\c -> (chainIdToText c, genesisHeight v c)) $ HS.toList $ chainIds v - , nodeHistoricalChains = ruleElems 0 $ fmap (HM.toList . HM.map HS.toList . toAdjacencySets) $ _versionGraphs v + , nodeHistoricalChains = ruleElems $ fmap (HM.toList . HM.map HS.toList . toAdjacencySets) $ _versionGraphs v , nodeServiceDate = T.pack <$> _versionServiceDate v , nodeBlockDelay = _versionBlockDelay v } diff --git a/test/lib/Chainweb/Test/TestVersions.hs b/test/lib/Chainweb/Test/TestVersions.hs index e21ab14d55..06ee7e2fe2 100644 --- a/test/lib/Chainweb/Test/TestVersions.hs +++ b/test/lib/Chainweb/Test/TestVersions.hs @@ -124,9 +124,9 @@ testVersionTemplate v = v & versionCode .~ ChainwebVersionCode (int (fromJuste $ List.elemIndex (_versionName v) testVersions) + 0x80000000) & versionHeaderBaseSizeBytes .~ 318 - 110 & versionWindow .~ WindowWidth 120 - & versionMaxBlockGasLimit .~ End (Just 2_000_000) + & versionMaxBlockGasLimit .~ Bottom (minBound, Just 2_000_000) & versionBootstraps .~ [testBootstrapPeerInfos] - & versionVerifierPluginNames .~ AllChains (End mempty) + & versionVerifierPluginNames .~ AllChains (Bottom (minBound, mempty)) & versionQuirks .~ noQuirks & versionServiceDate .~ Nothing @@ -173,7 +173,7 @@ barebonesTestVersion g = buildTestVersion $ \v -> & versionWindow .~ WindowWidth 120 & versionBlockDelay .~ BlockDelay 1_000_000 & versionName .~ ChainwebVersionName ("test-" <> toText g) - & versionGraphs .~ End g + & versionGraphs .~ Bottom (minBound, g) & versionCheats .~ VersionCheats { _disablePow = True , _fakeFirstEpochStart = True @@ -204,7 +204,7 @@ timedConsensusVersion g1 g2 = buildTestVersion $ \v -> v _ -> AllChains ForkAtGenesis ) & versionUpgrades .~ AllChains HM.empty - & versionGraphs .~ (BlockHeight 8, g2) `Above` (End g1) + & versionGraphs .~ (BlockHeight 8, g2) `Above` Bottom (minBound, g1) & versionCheats .~ VersionCheats { _disablePow = True , _fakeFirstEpochStart = True @@ -228,7 +228,7 @@ cpmTestVersion g v = v & testVersionTemplate & versionWindow .~ WindowWidth 120 & versionBlockDelay .~ BlockDelay (Micros 100_000) - & versionGraphs .~ End g + & versionGraphs .~ Bottom (minBound, g) & versionCheats .~ VersionCheats { _disablePow = True , _fakeFirstEpochStart = True @@ -297,7 +297,7 @@ slowForkingCpmTestVersion g = buildTestVersion $ \v -> v & versionName .~ ChainwebVersionName ("slowfork-CPM-" <> toText g) & versionForks .~ slowForks & versionVerifierPluginNames .~ AllChains - (End $ Set.fromList $ map VerifierName ["allow", "hyperlane_v3_announcement", "hyperlane_v3_message"]) + (Bottom (minBound, Set.fromList $ map VerifierName ["allow", "hyperlane_v3_announcement", "hyperlane_v3_message"])) -- | CPM version (see `cpmTestVersion`) with forks and upgrades slowly enabled, -- and with a gas fee quirk. diff --git a/test/unit/Chainweb/Test/Version.hs b/test/unit/Chainweb/Test/Version.hs index 942bcc0af2..cee53ded9b 100644 --- a/test/unit/Chainweb/Test/Version.hs +++ b/test/unit/Chainweb/Test/Version.hs @@ -28,7 +28,6 @@ import Test.Tasty.QuickCheck (testProperty) import Chainweb.BlockHash import Chainweb.BlockHeader -import Chainweb.BlockHeight import Chainweb.Graph import Chainweb.Test.Orphans.Internal import Chainweb.Test.Orphans.Internal () @@ -75,7 +74,7 @@ prop_chainGraphs_sorted v prop_chainGraphs_order :: ChainwebVersion -> Property prop_chainGraphs_order v = orders === NE.reverse (NE.sort orders) where - orders = ruleElems (BlockHeight 0) $ fmap order $ _versionGraphs v + orders = ruleElems $ fmap order $ _versionGraphs v prop_genesisHeight :: ChainwebVersion -> Property prop_genesisHeight v = property $ all ((>= 0) . genesisHeight v) $ chainIds v @@ -113,12 +112,12 @@ prop_headerBaseSizeBytes v = property $ do prop_headerSizes_sorted :: ChainwebVersion -> Property prop_headerSizes_sorted v - = NE.reverse (NE.sort (ruleElems (BlockHeight 0) (headerSizes v))) === ruleElems (BlockHeight 0) (headerSizes v) + = NE.reverse (NE.sort (ruleElems (headerSizes v))) === ruleElems (headerSizes v) prop_headerSizes_order :: ChainwebVersion -> Property prop_headerSizes_order v = orders === NE.reverse (NE.sort orders) where - orders = ruleElems (BlockHeight 0) $ fmap order $ _versionGraphs v + orders = ruleElems $ fmap order $ _versionGraphs v prop_headerSizeBytes_gen :: ChainwebVersion -> Property prop_headerSizeBytes_gen v = property $ do From 19f7d63cce2c5eed0d473047d4169417e61e1735 Mon Sep 17 00:00:00 2001 From: Edmund Noble Date: Sun, 25 Aug 2024 00:38:31 -0400 Subject: [PATCH 15/19] stop going through PactService to pre-insert check an empty tx batch It's extra latency that doesn't get us anything. --- src/Chainweb/Chainweb.hs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Chainweb/Chainweb.hs b/src/Chainweb/Chainweb.hs index 7308a6e3da..71a2dc7465 100644 --- a/src/Chainweb/Chainweb.hs +++ b/src/Chainweb/Chainweb.hs @@ -319,10 +319,12 @@ validatingMempoolConfig cid v gl gp mv = Mempool.InMemConfig :: V.Vector (T2 Mempool.TransactionHash ChainwebTransaction) -> IO (V.Vector (Either (T2 Mempool.TransactionHash Mempool.InsertError) (T2 Mempool.TransactionHash ChainwebTransaction))) - preInsertBatch txs = do - pex <- readMVar mv - rs <- _pactPreInsertCheck pex cid (V.map ssnd txs) - pure $ alignWithV f rs txs + preInsertBatch txs + | V.null txs = return V.empty + | otherwise = do + pex <- readMVar mv + rs <- _pactPreInsertCheck pex cid (V.map ssnd txs) + pure $ alignWithV f rs txs where f (These r (T2 h t)) = case r of Left e -> Left (T2 h e) From 9a957388a0fac736abdb45c1cbd0d29694731247 Mon Sep 17 00:00:00 2001 From: Edmund Noble Date: Wed, 30 Oct 2024 12:47:42 -0400 Subject: [PATCH 16/19] compute peerCount using ixset's size function Should be faster than using the Foldable `length` because it's using the default impl --- src/P2P/Node.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/P2P/Node.hs b/src/P2P/Node.hs index 61e77d4e9e..3a0ff9dfe9 100644 --- a/src/P2P/Node.hs +++ b/src/P2P/Node.hs @@ -559,7 +559,7 @@ findNextPeer conf node = do -- peers <- peerDbSnapshotSTM peerDbVar !sessions <- readTVar sessionsVar - let peerCount = length peers + let peerCount = IXS.size peers let sessionCount = M.size sessions -- Check that there are peers From 75ab5b5adc4843084623cfcbba6cb6683a71ac6d Mon Sep 17 00:00:00 2001 From: Edmund Noble Date: Thu, 7 Nov 2024 12:15:39 -0500 Subject: [PATCH 17/19] optimize peer pruning algorithm, deleting instead of unioning --- cabal.project | 8 ++-- cabal.project.freeze | 2 +- src/P2P/Node.hs | 2 +- src/P2P/Node/PeerDB.hs | 50 ++++++++++++++-------- test/lib/Chainweb/Test/Orphans/Internal.hs | 1 + 5 files changed, 40 insertions(+), 23 deletions(-) diff --git a/cabal.project b/cabal.project index a82cc62fa3..0da734cb89 100644 --- a/cabal.project +++ b/cabal.project @@ -121,12 +121,12 @@ source-repository-package -- Required for backward compatibility: --- ixset-typed FIX (no PR yet) +-- ixset-typed add difference operator (https://github.com/well-typed/ixset-typed/pull/24) source-repository-package type: git - location: https://github.com/larskuhtz/ixset-typed - tag: d8019c3404d6f3b3c0b0416e9899cfdf614ef425 - --sha256: 09msayidg23rsdz97fcfqqalm4pbawx3c1qihgab8hnlmjxby103 + location: https://github.com/kadena-io/ixset-typed + tag: 8b3227c59305f7e5ca6af7c0b9bc5435e78875a0 + --sha256: 08xpx3vwl8w7gn6z9s5smc383b962mxal3nav0fd5qh55yxqsp97 -- Patch merged into master (upcoming verison 10.0). We are currently using 9.2. -- This fork contains additional fixes for using 9.2 with recent compilers. diff --git a/cabal.project.freeze b/cabal.project.freeze index 235fa252f3..fe8c848a9d 100644 --- a/cabal.project.freeze +++ b/cabal.project.freeze @@ -152,7 +152,7 @@ constraints: any.Cabal ==3.10.2.0, any.integer-logarithms ==1.0.3.1, any.invariant ==0.6.3, any.iproute ==1.7.15, - any.ixset-typed ==0.5, + any.ixset-typed ==0.5.1.0, any.js-chart ==2.9.4.1, any.kan-extensions ==5.2.6, any.lens ==5.3.2, diff --git a/src/P2P/Node.hs b/src/P2P/Node.hs index 3a0ff9dfe9..7443c1cc73 100644 --- a/src/P2P/Node.hs +++ b/src/P2P/Node.hs @@ -437,7 +437,7 @@ peerClientEnv node = peerInfoClientEnv (_p2pNodeManager node) -- syncFromPeer :: P2pNode -> PeerInfo -> IO Bool syncFromPeer node info = do - prunePeerDb peerDb + prunePeerDb (_p2pNodeLogFunction node) peerDb runClientM sync env >>= \case Left e | isCertMismatch e -> do diff --git a/src/P2P/Node/PeerDB.hs b/src/P2P/Node/PeerDB.hs index 35d4a250a6..7ec4f193e3 100644 --- a/src/P2P/Node/PeerDB.hs +++ b/src/P2P/Node/PeerDB.hs @@ -14,6 +14,7 @@ {-# LANGUAGE TemplateHaskell #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE RankNTypes #-} -- | -- Module: P2P.Node.PeerDB @@ -31,6 +32,7 @@ module P2P.Node.PeerDB , SuccessiveFailures(..) , AddedTime(..) , ActiveSessionCount(..) +, PeerEntrySticky(..) , HostAddressIdx , hostAddressIdx @@ -81,7 +83,7 @@ import Control.Concurrent.MVar import Control.Concurrent.STM.TVar import Control.DeepSeq import Control.Lens hiding (Indexable) -import Control.Monad ((<$!>)) +import Control.Monad ((<$!>), unless) import Control.Monad.STM import Data.Aeson @@ -115,6 +117,10 @@ import Chainweb.Version import Data.Singletons import P2P.Peer +import qualified Data.IxSet.Typed as IxSet +import Data.LogMessage +import System.LogLevel +import Data.Text (Text) -- -------------------------------------------------------------------------- -- -- Peer Database Entry @@ -135,6 +141,10 @@ newtype ActiveSessionCount = ActiveSessionCount { _getActiveSessionCount :: Natu deriving (Show, Eq, Ord, Generic) deriving newtype (ToJSON, FromJSON, Num, Enum, NFData) +newtype PeerEntrySticky = PeerEntrySticky { _getPeerEntrySticky :: Bool } + deriving (Show, Eq, Ord, Generic) + deriving newtype (ToJSON, FromJSON, Enum, NFData) + data PeerEntry = PeerEntry { _peerEntryInfo :: !PeerInfo -- ^ There must be only one peer per peer address. A peer id @@ -162,7 +172,7 @@ data PeerEntry = PeerEntry -- -- ^ Count the number of sessions. When this number becomes to high -- -- we should - , _peerEntrySticky :: !Bool + , _peerEntrySticky :: !PeerEntrySticky -- ^ A flag that indicates whether this entry can not be pruned form the -- db -- @@ -176,7 +186,8 @@ newPeerEntry :: NetworkId -> PeerInfo -> PeerEntry newPeerEntry = newPeerEntry_ False newPeerEntry_ :: Bool -> NetworkId -> PeerInfo -> PeerEntry -newPeerEntry_ sticky nid i = PeerEntry i 0 (LastSuccess Nothing) (S.singleton nid) 0 sticky +newPeerEntry_ sticky nid i = + PeerEntry i 0 (LastSuccess Nothing) (S.singleton nid) 0 (PeerEntrySticky sticky) -- -------------------------------------------------------------------------- -- -- Peer Entry Set @@ -200,6 +211,7 @@ type PeerEntryIxs = , LastSuccess , NetworkId , ActiveSessionCount + , PeerEntrySticky ] instance Indexable PeerEntryIxs PeerEntry where @@ -209,6 +221,7 @@ instance Indexable PeerEntryIxs PeerEntry where (ixFun $ \e -> [_peerEntryLastSuccess e]) (ixFun $ \e -> F.toList (_peerEntryNetworkIds e)) (ixFun $ \e -> [_peerEntryActiveSessionCount e]) + (ixFun $ \e -> [_peerEntrySticky e]) type PeerSet = IxSet PeerEntryIxs PeerEntry @@ -251,7 +264,7 @@ addPeerEntry b m = m & case getOne (getEQ addr m) of , _peerEntryLastSuccess = max (_peerEntryLastSuccess a) (_peerEntryLastSuccess b) , _peerEntryNetworkIds = _peerEntryNetworkIds a <> _peerEntryNetworkIds b , _peerEntryActiveSessionCount = _peerEntryActiveSessionCount a + _peerEntryActiveSessionCount b - , _peerEntrySticky = False + , _peerEntrySticky = PeerEntrySticky False } -- | Add a 'PeerInfo' to an existing 'PeerSet'. @@ -282,7 +295,7 @@ deletePeer -> PeerSet deletePeer i True s = deleteIx (_peerAddr i) s deletePeer i False s = case _peerEntrySticky <$> getOne (getEQ (_peerAddr i) s) of - Just True -> s + Just (PeerEntrySticky True) -> s _ -> deleteIx (_peerAddr i) s insertPeerEntryList :: [PeerEntry] -> PeerSet -> PeerSet @@ -369,19 +382,22 @@ peerDbDelete_ (PeerDb _ _ _ lock var) forceSticky i = withMVar lock -- 2. we haven't used since 12h, and that -- 3. have had more than 5 failed connection attempts. -- -prunePeerDb :: PeerDb -> IO () -prunePeerDb (PeerDb _ _ _ lock var) = do +prunePeerDb :: LogFunction -> PeerDb -> IO () +prunePeerDb lg (PeerDb _ _ _ lock var) = do + now <- getCurrentTime withMVar lock $ \_ -> do - now <- getCurrentTime - let cutoff = Just $ addUTCTime ((-60) * 60 * 12) now - atomically $ modifyTVar' var $ \s -> - getGT (ActiveSessionCount 0) s - ||| - getLTE (SuccessiveFailures 5) s - ||| - getGT (LastSuccess cutoff) s - ||| - fromList (filter _peerEntrySticky $ toList s) + deletes <- atomically $ do + s <- readTVar var + let cutoff = Just $ addUTCTime ((-60) * 60 * 12) now + let deletes = s + IxSet.@> SuccessiveFailures 5 + IxSet.@< LastSuccess cutoff + IxSet.@= ActiveSessionCount 0 + IxSet.@= PeerEntrySticky False + writeTVar var $! s IxSet.\\\ deletes + return deletes + unless (IxSet.null deletes) $ + lg @Text Info $ "Pruned peers: " <> sshow (_peerAddr . _peerEntryInfo <$> IxSet.toList deletes) peerDbInsertList :: [PeerEntry] -> PeerDb -> IO () peerDbInsertList _ (PeerDb True _ _ _ _) = return () diff --git a/test/lib/Chainweb/Test/Orphans/Internal.hs b/test/lib/Chainweb/Test/Orphans/Internal.hs index f7438d891d..c9a160cb71 100644 --- a/test/lib/Chainweb/Test/Orphans/Internal.hs +++ b/test/lib/Chainweb/Test/Orphans/Internal.hs @@ -250,6 +250,7 @@ deriving newtype instance Arbitrary LastSuccess deriving newtype instance Arbitrary SuccessiveFailures deriving newtype instance Arbitrary AddedTime deriving newtype instance Arbitrary ActiveSessionCount +deriving newtype instance Arbitrary PeerEntrySticky deriving via (NonEmptyList Int) instance Arbitrary NodeVersion instance Arbitrary X509KeyPem where From b4a4db043fb2a7e1c7665e30f3e31d4f56f7e3ea Mon Sep 17 00:00:00 2001 From: Edmund Noble Date: Fri, 1 Nov 2024 15:37:30 -0400 Subject: [PATCH 18/19] Remove redundant `force` calls in updatePeerDb This code is already strict enough. ixset-typed is a set and a strict list of maps. When updated, ixset-typed will force the set and maps to whnf, and insert the new element into a Set. Sets are strict in their elements and maps are strict in their keys, and the value type `PeerEntry` is itself strict in its components. --- src/P2P/Node/PeerDB.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/P2P/Node/PeerDB.hs b/src/P2P/Node/PeerDB.hs index 7ec4f193e3..8a22012f3a 100644 --- a/src/P2P/Node/PeerDB.hs +++ b/src/P2P/Node/PeerDB.hs @@ -436,7 +436,7 @@ updatePeerDb (PeerDb _ _ _ lock var) a f = withMVar lock . const . atomically . modifyTVar' var $ \s -> case getOne $ getEQ a s of Nothing -> s - Just x -> force $ updateIx a (force $ f x) s + Just x -> updateIx a (f x) s incrementActiveSessionCount :: PeerDb -> PeerInfo -> IO () incrementActiveSessionCount db i From 0865649fe6bf68e3ae213dbed522dd14b56ff39b Mon Sep 17 00:00:00 2001 From: chessai Date: Thu, 14 Nov 2024 13:08:46 -0600 Subject: [PATCH 19/19] split ea out into its own executable Change-Id: I50906fb06cdc29c74a778fba0f673797cfdb53a6 --- .github/workflows/applications.yml | 13 +- chainweb.cabal | 34 ++++- default.nix | 2 +- tools/cwtool/CwTool.hs | 5 - tools/ea/Allocations.hs | 208 ----------------------------- tools/ea/Ea.hs | 60 ++++----- 6 files changed, 62 insertions(+), 260 deletions(-) delete mode 100644 tools/ea/Allocations.hs diff --git a/.github/workflows/applications.yml b/.github/workflows/applications.yml index 964a2892a5..5877e4e6f8 100644 --- a/.github/workflows/applications.yml +++ b/.github/workflows/applications.yml @@ -359,7 +359,7 @@ jobs: - name: Build chainweb library run: cabal build --ghc-options=-j2 lib:chainweb - name: Build chainweb applications - run: | + run: | cabal build -j2 --ghc-options=-j2 \ exe:compact \ test:chainweb-tests \ @@ -380,7 +380,7 @@ jobs: fi - name: Run ea and verify consistency of genesis headers run: | - cabal run cwtool -- ea + cabal run ea if ! git diff --exit-code -- src/Chainweb/BlockHeader/Genesis/ src/Chainweb/Pact/Transactions/ ; then echo "Inconsistent genesis headers detected. Did you forget to run ea?" 1>&2 exit 1 @@ -390,14 +390,15 @@ jobs: - name: Prepare artifacts run: | mkdir -p artifacts/chainweb + cp $(cabal list-bin bench) artifacts/chainweb cp $(cabal list-bin chainweb-node) artifacts/chainweb - cp $(cabal list-bin compact) artifacts/chainweb cp $(cabal list-bin chainweb-tests) artifacts/chainweb - cp $(cabal list-bin multi-node-network-tests) artifacts/chainweb - cp $(cabal list-bin remote-tests) artifacts/chainweb + cp $(cabal list-bin compact) artifacts/chainweb cp $(cabal list-bin compaction-tests) artifacts/chainweb cp $(cabal list-bin cwtool) artifacts/chainweb - cp $(cabal list-bin bench) artifacts/chainweb + cp $(cabal list-bin ea) artifacts/chainweb + cp $(cabal list-bin multi-node-network-tests) artifacts/chainweb + cp $(cabal list-bin remote-tests) artifacts/chainweb cp README.md artifacts/chainweb cp CHANGELOG.md artifacts/chainweb cp LICENSE artifacts/chainweb diff --git a/chainweb.cabal b/chainweb.cabal index b8c91b26ed..e829d34b6a 100644 --- a/chainweb.cabal +++ b/chainweb.cabal @@ -802,6 +802,33 @@ test-suite remote-tests -- Misc Applications -- -------------------------------------------------------------------------- -- +executable ea + import: warning-flags, debugging-flags + default-language: Haskell2010 + ghc-options: + -threaded + -rtsopts + "-with-rtsopts=-N -H1G -A64M" + -Wno-x-partial -Wno-unrecognised-warning-flags + hs-source-dirs: + tools/ea + main-is: + Ea.hs + other-modules: + Ea.Genesis + build-depends: + , chainweb + , chainweb:chainweb-test-utils + + , base + , chainweb-storage + , lens + , loglevel + , pact + , temporary + , text + , vector + executable cwtool import: warning-flags, debugging-flags default-language: Haskell2010 @@ -812,7 +839,6 @@ executable cwtool -Wno-x-partial -Wno-unrecognised-warning-flags hs-source-dirs: tools/cwtool - tools/ea tools/encode-decode tools/genconf tools/run-nodes @@ -823,10 +849,7 @@ executable cwtool tools/calculate-release main-is: CwTool.hs other-modules: - Allocations CheckpointerDBChecksum - Ea - Ea.Genesis EncodeDecodeB64Util GenConf HeaderDump @@ -848,7 +871,6 @@ executable cwtool , base >= 4.12 && < 5 , bytestring >= 0.10.12 , case-insensitive >= 1.2 - , cassava >= 0.5.1 , cereal >= 0.5 , chainweb-storage >= 0.1 , configuration-tools >= 0.6 @@ -859,7 +881,6 @@ executable cwtool , direct-sqlite >= 2.3.27 , directory >= 1.3 , exceptions >= 0.8 - , file-embed >= 0.0 , http-client >= 0.5 , http-client-tls >=0.3 , lens >= 4.17 @@ -876,7 +897,6 @@ executable cwtool , servant-client >= 0.18.2 , servant-client-core >= 0.18.2 , streaming >= 0.2.2 - , temporary >= 1.3 , text >= 2.0 , time >= 1.12.2 , unordered-containers >= 0.2.16 diff --git a/default.nix b/default.nix index 00add2d417..f3a419a333 100644 --- a/default.nix +++ b/default.nix @@ -113,7 +113,7 @@ let haskellSrc = with nix-filter.lib; filter { { name = "chainweb"; - exes = ["cwtool" "compact" "pact-diff"]; + exes = ["cwtool" "compact" "ea" "pact-diff"]; } ]; diff --git a/tools/cwtool/CwTool.hs b/tools/cwtool/CwTool.hs index c7cd41e0d5..61bd4f09a0 100644 --- a/tools/cwtool/CwTool.hs +++ b/tools/cwtool/CwTool.hs @@ -14,7 +14,6 @@ import Chainweb.Pact.Backend.PactState.GrandHash.Calc (pactCalcMain) import Chainweb.Pact.Backend.PactState.GrandHash.Import (pactImportMain) import qualified CheckpointerDBChecksum -import qualified Ea import qualified EncodeDecodeB64Util import qualified GenConf import qualified HeaderDump @@ -62,10 +61,6 @@ cmdHelpLine cs = printf " %s%s" (padRight 25 $ csCmd cs) (csDescription cs) topLevelCommands :: [CommandSpec] topLevelCommands = [ CommandSpec - "ea" - "Generate Chainweb genesis blocks and their payloads" - Ea.main - , CommandSpec "run-nodes" "Run a local cluster of chainweb-node binaries" RunNodes.main diff --git a/tools/ea/Allocations.hs b/tools/ea/Allocations.hs deleted file mode 100644 index a4b98340cc..0000000000 --- a/tools/ea/Allocations.hs +++ /dev/null @@ -1,208 +0,0 @@ -{-# LANGUAGE DeriveGeneric #-} -{-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE QuasiQuotes #-} -{-# LANGUAGE RankNTypes #-} -{-# LANGUAGE TemplateHaskell #-} -{-# LANGUAGE TypeApplications #-} --- | --- Module: Allocations --- Copyright: Copyright © 2018 - 2020 Kadena LLC. --- License: MIT --- Maintainer: Colin Woodbury , Emily Pillmore --- Stability: experimental --- --- Generate allocations payloads --- -module Allocations -( -- * Allocation generation - generateAllocations - - -- * Allocation data -, AllocationEntry(..) -, AllocationTx(..) -, mkAllocationTx - -, AllocationKeys(..) -, AllocationKeyTx(..) -, mkAllocationKeyTx - -, readAllocations -, readMainnetKeys -, readTestnetKeys - -, rawAllocations -, rawMainnetKeys -, rawTestnetKeys -) where - - -import GHC.Generics - -import Data.ByteString (ByteString) -import qualified Data.ByteString.Lazy as BL -import qualified Data.Csv as CSV -import Data.FileEmbed (embedFile) -import qualified Data.Map.Strict as M -import Data.Text (Text) -import qualified Data.Text as T -import qualified Data.Text.IO as T -import Data.Vector (Vector) -import qualified Data.Vector as V - -import Chainweb.Utils - --- -------------------------------------------------------------------- -- --- Tx gen - -generateAllocations :: IO () -generateAllocations - = allocations - >> mainKeysets - >> testKeysets - where - mainKeysets = T.writeFile (prefix $ "mainnet_keysets") $ - keysToYaml "mainnet-keysets" readMainnetKeys - - testKeysets = T.writeFile (prefix $ "testnet_keysets") $ - keysToYaml "testnet-keysets" readTestnetKeys - - allocations = flip M.traverseWithKey readAllocations $ \cid txs -> do - let ys = allocToYaml cid txs - T.writeFile (prefix $ "mainnet_allocations" <> T.unpack cid) ys - - prefix t = "pact/genesis/mainnet/" <> t <> ".yaml" - -genTxs - :: forall a b - . CSV.FromRecord a - => (a -> b) - -> ByteString - -> Vector b -genTxs f bs = case CSV.decode CSV.HasHeader (BL.fromStrict bs) of - Left e -> error - $ "cannot construct genesis allocations: " - <> sshow e - Right as -> fmap f as - - -keysToYaml :: Text -> Vector AllocationKeyTx -> Text -keysToYaml nonce ks = T.concat - [ "code: |-\n" - , go f ks - , "\ndata:\n" - , go g ks - , "\nnonce: " <> nonce - , "\nkeyPairs: []\n" - ] - where - go k = V.foldl1' (<>) . V.map k - wrap tx = " " <> tx <> "\n" - f = wrap . _allocationKeyTx - g = wrap . _allocationKeyData - -allocToYaml :: Text -> Vector AllocationTx -> Text -allocToYaml cid as = T.concat - [ "code: |-\n" - , go f as - , "\nnonce: " <> "mainnet-genesis-allocations" <> cid - , "\nkeyPairs: []\n" - ] - where - go k = V.foldl1' (<>) . V.map k - wrap tx = " " <> tx <> "\n" - f = wrap . _allocationTx - - -readAllocations :: M.Map Text (Vector AllocationTx) -readAllocations = txMap - where - txMap = V.foldl' go mempty $ genTxs mkAllocationTx rawAllocations - go m a = - let - cid = _allocationTxChain a - in case M.lookup cid m of - Just v -> M.insert cid (V.cons a v) m - Nothing -> M.insert cid (V.singleton a) m - -readMainnetKeys :: Vector AllocationKeyTx -readMainnetKeys = genTxs mkAllocationKeyTx rawMainnetKeys - -readTestnetKeys :: Vector AllocationKeyTx -readTestnetKeys = genTxs mkAllocationKeyTx rawTestnetKeys - --- -------------------------------------------------------------------- -- --- Allocation/Key/Coinbase data - -data AllocationKeys = AllocationKeys - { _allocKeysName :: Text - , _allocKeysPred :: Text - , _allocKey1 :: Text - , _allocKey2 :: Text - , _allocKey3 :: Text - } deriving (Eq, Ord, Show, Generic) - -instance CSV.FromRecord AllocationKeys - -data AllocationKeyTx = AllocationKeyTx - { _allocationKeyTx :: Text - , _allocationKeyData :: Text - } deriving (Eq, Show) - -mkAllocationKeyTx :: AllocationKeys -> AllocationKeyTx -mkAllocationKeyTx (AllocationKeys n p k1 k2 k3) = AllocationKeyTx tx d - where - tx = T.concat - [ "(define-keyset " - , "\"" <> T.strip n <> "\" " - , "(read-keyset " - , "\"" <> T.strip n <> "\")" - , ")" - ] - - d = T.concat - [ "\"" <> T.strip n <> "\": { " - , "keys: " <> ks <> ", " - , "pred: \"" <> T.strip p <> "\"" - , " }" - ] - - ks = T.strip . sshow $ fmap T.strip [k1,k2,k3] - -data AllocationEntry = AllocationEntry - { _allocationName :: !Text - , _allocationTime :: !Text - , _allocationKeysetName :: !Text - , _allocationAmount :: !CsvDecimal - , _allocationChain :: !Text - } deriving (Eq, Ord, Show, Generic) - -instance CSV.FromRecord AllocationEntry - -data AllocationTx = AllocationTx - { _allocationTx :: Text - , _allocationTxChain :: Text - } deriving (Eq, Show) - -mkAllocationTx :: AllocationEntry -> AllocationTx -mkAllocationTx (AllocationEntry n t ksn a c) = AllocationTx tx c - where - tx = T.concat - [ "(coin.create-allocation-account " - , "\"" <> T.strip n <> "\" " - , "(time \"" <> T.strip t <> "\") " - , "\"" <> T.strip ksn <> "\" " - , T.pack (show a) - , ")" - ] - --- -------------------------------------------------------------------- -- --- Raw file bytes - -rawAllocations :: ByteString -rawAllocations = $(embedFile "allocations/token_payments.csv") - -rawMainnetKeys :: ByteString -rawMainnetKeys = $(embedFile "allocations/Mainnet-Keysets.csv") - -rawTestnetKeys :: ByteString -rawTestnetKeys = $(embedFile "allocations/Testnet-Keysets.csv") diff --git a/tools/ea/Ea.hs b/tools/ea/Ea.hs index 55089c4233..1271fd71fd 100644 --- a/tools/ea/Ea.hs +++ b/tools/ea/Ea.hs @@ -1,8 +1,8 @@ {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE FlexibleInstances #-} -{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE ImportQualifiedPost #-} {-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE TypeOperators #-} +{-# LANGUAGE PatternSynonyms #-} -- | -- Module: Ea @@ -24,30 +24,7 @@ -- -- Eä means "to be" in Quenya, the ancient language of Tolkien's elves. -- -module Ea - ( main ) where - -import Control.Exception -import Control.Lens - -import Data.Foldable -import Data.Functor -import Data.Text (Text) -import qualified Data.Text as T -import qualified Data.Text.Encoding as TE -import qualified Data.Text.IO as TIO -import qualified Data.Text.Lazy as TL -import qualified Data.Text.Lazy.Builder as TB -import Data.Traversable -import qualified Data.Vector as V -import GHC.Exts(the) - -import System.IO.Temp -import System.LogLevel (LogLevel(..)) - -import Text.Printf - --- internal modules +module Main (main) where import Chainweb.BlockHeaderDB import Chainweb.Logger (genericLogger) @@ -61,21 +38,38 @@ import Chainweb.Payload import Chainweb.Payload.PayloadStore.InMemory import Chainweb.Storage.Table.RocksDB import Chainweb.Time -import Chainweb.Transaction - (ChainwebTransaction, chainwebPayloadCodec, mkPayloadWithTextOld) +import Chainweb.Transaction (ChainwebTransaction, chainwebPayloadCodec, mkPayloadWithTextOld) import Chainweb.Utils import Chainweb.Version - +import Chainweb.Version.Development (pattern Development) +import Chainweb.Version.RecapDevelopment (pattern RecapDevelopment) +import Chainweb.Version.Registry (registerVersion) +import Control.Exception +import Control.Lens +import Data.Foldable +import Data.Functor +import Data.Text (Text) +import Data.Text qualified as T +import Data.Text.Encoding qualified as TE +import Data.Text.IO qualified as TIO +import Data.Text.Lazy qualified as TL +import Data.Text.Lazy.Builder qualified as TB +import Data.Traversable +import Data.Vector qualified as V import Ea.Genesis - +import GHC.Exts(the) import Pact.ApiReq import Pact.Types.ChainMeta import Pact.Types.Command hiding (Payload) - ---- +import System.IO.Temp +import System.LogLevel (LogLevel(..)) +import Text.Printf main :: IO () -main = void $ do +main = do + registerVersion RecapDevelopment + registerVersion Development + recapDevnet devnet fastnet