diff --git a/Cargo.lock b/Cargo.lock index 35a6e2fe6..79aa93d07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,18 +23,18 @@ dependencies = [ [[package]] name = "addr2line" -version = "0.22.0" +version = "0.24.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" dependencies = [ - "gimli 0.29.0", + "gimli 0.31.1", ] [[package]] -name = "adler" -version = "1.0.2" +name = "adler2" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" [[package]] name = "aead" @@ -179,9 +179,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" [[package]] name = "approx" @@ -263,7 +263,7 @@ dependencies = [ "num-bigint", "num-traits", "paste", - "rustc_version 0.4.0", + "rustc_version 0.4.1", "zeroize", ] @@ -399,9 +399,9 @@ checksum = "5d5dde061bd34119e902bbb2d9b90c5692635cf59fb91d582c2b68043f1b8293" [[package]] name = "arrayref" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" [[package]] name = "arrayvec" @@ -411,9 +411,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "arrayvec" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "asn1-rs" @@ -473,9 +473,9 @@ dependencies = [ [[package]] name = "async-io" -version = "2.3.3" +version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d6baa8f0178795da0e71bc42c9e5d13261aac7ee549853162e66a241ba17964" +checksum = "444b0228950ee6501b3568d3c93bf1176a1fdbc3b758dcd9475046d30f4dc7e8" dependencies = [ "async-lock 3.4.0", "cfg-if", @@ -484,10 +484,10 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix 0.38.34", + "rustix 0.38.37", "slab", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -518,18 +518,18 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] name = "async-trait" -version = "0.1.81" +version = "0.1.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" +checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -558,23 +558,23 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "backtrace" -version = "0.3.73" +version = "0.3.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" dependencies = [ - "addr2line 0.22.0", - "cc", + "addr2line 0.24.2", "cfg-if", "libc", "miniz_oxide", - "object 0.36.2", + "object 0.36.5", "rustc-demangle", + "windows-targets 0.52.6", ] [[package]] @@ -659,13 +659,13 @@ dependencies = [ "lazy_static", "lazycell", "peeking_take_while", - "prettyplease 0.2.20", + "prettyplease 0.2.22", "proc-macro2", "quote", "regex", "rustc-hash", "shlex", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -708,7 +708,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23285ad32269793932e830392f2fe2f83e26488fd3ec778883a93c8323735780" dependencies = [ "arrayref", - "arrayvec 0.7.4", + "arrayvec 0.7.6", "constant_time_eq", ] @@ -719,18 +719,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94230421e395b9920d23df13ea5d77a20e1725331f90fbbf6df6040b33f756ae" dependencies = [ "arrayref", - "arrayvec 0.7.4", + "arrayvec 0.7.6", "constant_time_eq", ] [[package]] name = "blake3" -version = "1.5.3" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9ec96fe9a81b5e365f9db71fe00edc4fe4ca2cc7dcb7861f0603012a7caa210" +checksum = "d82033247fd8e890df8f740e407ad4d038debb9eb1f40533fffb32e7d17dc6f7" dependencies = [ "arrayref", - "arrayvec 0.7.4", + "arrayvec 0.7.6", "cc", "cfg-if", "constant_time_eq", @@ -850,9 +850,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.16.1" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e" +checksum = "94bbb0ad554ad961ddc5da507a12a29b14e4ae5bda06b19f575a3e6079d2e2ae" [[package]] name = "byteorder" @@ -862,9 +862,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.6.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" +checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" [[package]] name = "bzip2-sys" @@ -879,9 +879,9 @@ dependencies = [ [[package]] name = "camino" -version = "1.1.7" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0ec6b951b160caa93cc0c7b209e5a3bff7aae9062213451ac99493cd844c239" +checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" dependencies = [ "serde", ] @@ -911,12 +911,13 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.7" +version = "1.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26a5c3fd7bfa1ce3897a3a3501d362b2d87b7f2583ebcb4a949ec25911025cbc" +checksum = "812acba72f0a070b003d3697490d2b55b837230ae7c6c6497f05cc2ddbb8d938" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -1362,9 +1363,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.11" +version = "4.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35723e6a11662c2afb578bcf0b88bf6ea8e21282a953428f240574fcc3a2b5b3" +checksum = "7be5744db7978a28d9df86a214130d106a89ce49644cbc4e3f0c22c3fba30615" dependencies = [ "clap_builder", "clap_derive", @@ -1372,9 +1373,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.11" +version = "4.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49eb96cbfa7cfa35017b7cd548c75b14c3118c98b423041d70562665e07fb0fa" +checksum = "a5fbc17d3ef8278f55b282b2a2e75ae6f6c7d4bb70ed3d0382375104bfafdb4b" dependencies = [ "anstream", "anstyle", @@ -1384,14 +1385,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.11" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d029b67f89d30bbb547c89fd5161293c0aec155fc691d7924b64550662db93e" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -1497,9 +1498,9 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "core-foundation" @@ -1513,9 +1514,9 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.6" +version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" [[package]] name = "core2" @@ -1537,9 +1538,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +checksum = "608697df725056feaccfa42cffdaeeec3fccc4ffc38358ecd19b243e716a78e0" dependencies = [ "libc", ] @@ -1771,7 +1772,7 @@ dependencies = [ "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", - "rustc_version 0.4.0", + "rustc_version 0.4.1", "subtle", "zeroize", ] @@ -1784,14 +1785,14 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] name = "cxx" -version = "1.0.124" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "273dcfd3acd4e1e276af13ed2a43eea7001318823e7a726a6b3ed39b4acc0b82" +checksum = "54ccead7d199d584d139148b04b4a368d1ec7556a1d9ea2548febb1b9d49f9a4" dependencies = [ "cc", "cxxbridge-flags", @@ -1801,9 +1802,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.124" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8b2766fbd92be34e9ed143898fce6c572dc009de39506ed6903e5a05b68914e" +checksum = "c77953e99f01508f89f55c494bfa867171ef3a6c8cea03d26975368f2121a5c1" dependencies = [ "cc", "codespan-reporting", @@ -1811,24 +1812,24 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] name = "cxxbridge-flags" -version = "1.0.124" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "839fcd5e43464614ffaa989eaf1c139ef1f0c51672a1ed08023307fa1b909ccd" +checksum = "65777e06cc48f0cb0152024c77d6cf9e4bdb4408e7b48bea993d42fa0f5b02b6" [[package]] name = "cxxbridge-macro" -version = "1.0.124" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b2c1c1776b986979be68bb2285da855f8d8a35851a769fca8740df7c3d07877" +checksum = "98532a60dedaebc4848cb2cba5023337cc9ea3af16a5b062633fabfd9f18fb60" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -1937,7 +1938,7 @@ checksum = "d65d7ce8132b7c0e54497a4d9a55a1c2a0912a0d786cf894472ba818fba45762" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -1948,7 +1949,7 @@ checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -2036,7 +2037,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -2051,7 +2052,7 @@ dependencies = [ "ark-serialize", "ark-std", "ark-transcript", - "arrayvec 0.7.4", + "arrayvec 0.7.6", "rand_core 0.6.4", "zeroize", ] @@ -2077,9 +2078,9 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.72", + "syn 2.0.79", "termcolor", - "toml 0.8.16", + "toml 0.8.19", "walkdir", ] @@ -2241,7 +2242,7 @@ checksum = "de0d48a183585823424a4ce1aa132d174a6a81bd540895822eb4c8373a8e49e8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -2324,10 +2325,10 @@ dependencies = [ "blake2", "file-guard", "fs-err", - "prettyplease 0.2.20", + "prettyplease 0.2.22", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -2344,9 +2345,9 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastrand" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "fdlimit" @@ -2408,14 +2409,14 @@ dependencies = [ [[package]] name = "filetime" -version = "0.2.23" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +checksum = "35c0522e981e68cbfa8c3f978441a5f34b30b96e146b33cd3359176b50fe8586" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.4.1", - "windows-sys 0.52.0", + "libredox", + "windows-sys 0.59.0", ] [[package]] @@ -2454,9 +2455,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flate2" -version = "1.0.30" +version = "1.0.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" +checksum = "a1b589b4dc103969ad3cf85c950899926ec64300a1a46d76c03a6072957036f0" dependencies = [ "crc32fast", "libz-sys", @@ -2582,7 +2583,7 @@ dependencies = [ "proc-macro-crate 1.1.3", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -2709,7 +2710,7 @@ dependencies = [ "proc-macro-warning", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -2721,7 +2722,7 @@ dependencies = [ "proc-macro-crate 1.1.3", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -2731,7 +2732,7 @@ source = "git+https://github.com/paritytech/polkadot-sdk?tag=polkadot-v1.1.0#f60 dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -2881,7 +2882,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -3024,9 +3025,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.29.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" @@ -3036,15 +3037,15 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" [[package]] name = "globset" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57da3b9b5b85bd66f31093f8c408b90a74431672542466497dcbdfdc02034be1" +checksum = "15f1ce686646e7f1e19bf7d5533fe443a45dbfb990e00629110797578b42fb19" dependencies = [ "aho-corasick", "bstr", "log", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -3070,7 +3071,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 2.2.6", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -3126,9 +3127,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.5" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" [[package]] name = "heck" @@ -3274,9 +3275,9 @@ checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" [[package]] name = "httparse" -version = "1.9.4" +version = "1.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" [[package]] name = "httpdate" @@ -3333,9 +3334,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.60" +version = "0.1.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -3465,12 +3466,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.0", ] [[package]] @@ -3550,17 +3551,17 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.9.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "is-terminal" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +checksum = "261f68e344040fbd0edea105bef17c66edf46f984ddb1115b775ce31be948f4b" dependencies = [ - "hermit-abi 0.3.9", + "hermit-abi 0.4.0", "libc", "windows-sys 0.52.0", ] @@ -3606,9 +3607,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" dependencies = [ "wasm-bindgen", ] @@ -3656,7 +3657,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b5dde66c53d6dcdc8caea1874a45632ec0fcf5b437789f1e45766a1512ce803" dependencies = [ "anyhow", - "arrayvec 0.7.4", + "arrayvec 0.7.6", "async-lock 2.8.0", "async-trait", "beef", @@ -3759,9 +3760,9 @@ dependencies = [ [[package]] name = "k256" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "956ff9b67e26e1a6a866cb758f12c6f8746208489e3e4a4b5580802f2f0a587b" +checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" dependencies = [ "cfg-if", "ecdsa", @@ -3826,9 +3827,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.155" +version = "0.2.159" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" [[package]] name = "libloading" @@ -3991,7 +3992,7 @@ version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39d5ef876a2b2323d63c258e63c2f8e36f205fe5a11f0b3095d59635650790ff" dependencies = [ - "arrayvec 0.7.4", + "arrayvec 0.7.6", "asynchronous-codec", "bytes", "either", @@ -4247,6 +4248,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.6.0", "libc", + "redox_syscall 0.5.7", ] [[package]] @@ -4314,9 +4316,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.18" +version = "1.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" +checksum = "d2d16453e800a8cf6dd2fc3eb4bc99b786a9b90c663b8559a5b1a041bf89e472" dependencies = [ "cc", "pkg-config", @@ -4404,19 +4406,18 @@ dependencies = [ [[package]] name = "lz4" -version = "1.26.0" +version = "1.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958b4caa893816eea05507c20cfe47574a43d9a697138a7872990bba8a0ece68" +checksum = "4d1febb2b4a79ddd1980eede06a8f7902197960aa0383ffcfdd62fe723036725" dependencies = [ - "libc", "lz4-sys", ] [[package]] name = "lz4-sys" -version = "1.10.0" +version = "1.11.1+lz4-1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109de74d5d2353660401699a4174a4ff23fcc649caf553df71933c7fb45ad868" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" dependencies = [ "cc", "libc", @@ -4440,7 +4441,7 @@ dependencies = [ "macro_magic_core", "macro_magic_macros", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -4454,7 +4455,7 @@ dependencies = [ "macro_magic_core_macros", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -4465,7 +4466,7 @@ checksum = "d710e1214dffbab3b5dacb21475dde7d6ed84c69ff722b3a47a782668d44fbac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -4476,7 +4477,7 @@ checksum = "b8fb85ec1620619edf2984a7693497d4ec88a9665d8b87e942856884c92dbf2a" dependencies = [ "macro_magic_core", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -4528,7 +4529,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" dependencies = [ - "rustix 0.38.34", + "rustix 0.38.37", ] [[package]] @@ -4590,18 +4591,18 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ - "adler", + "adler2", ] [[package]] name = "mio" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" dependencies = [ "hermit-abi 0.3.9", "libc", @@ -4741,7 +4742,7 @@ checksum = "254a5372af8fc138e36684761d3c0cdb758a4410e938babcff1c860ce14ddbfc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -4892,7 +4893,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" dependencies = [ - "arrayvec 0.7.4", + "arrayvec 0.7.6", "itoa", ] @@ -4955,9 +4956,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.2" +version = "0.36.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" dependencies = [ "memchr", ] @@ -4973,9 +4974,12 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "82881c4be219ab5faaf2ad5e5e5ecdff8c66bd7402ca3160975c93b24961afd1" +dependencies = [ + "portable-atomic", +] [[package]] name = "opaque-debug" @@ -5205,7 +5209,7 @@ source = "git+https://github.com/paritytech/polkadot-sdk?tag=polkadot-v1.1.0#f60 dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -5933,7 +5937,7 @@ dependencies = [ "proc-macro-crate 1.1.3", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -6123,7 +6127,7 @@ version = "3.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "306800abfa29c7f16596b5970a588435e3d5b3149683d00c12b699cc19f895ee" dependencies = [ - "arrayvec 0.7.4", + "arrayvec 0.7.6", "bitvec", "byte-slice-cast", "bytes", @@ -6138,7 +6142,7 @@ version = "3.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d830939c76d294956402033aee57a6da7b438f2294eb94864c37b0569053a42c" dependencies = [ - "proc-macro-crate 3.1.0", + "proc-macro-crate 3.2.0", "proc-macro2", "quote", "syn 1.0.109", @@ -6158,9 +6162,9 @@ checksum = "e1ad0aff30c1da14b1254fcb2af73e1fa9a28670e584a626f53a369d0e157304" [[package]] name = "parking" -version = "2.2.0" +version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" @@ -6205,7 +6209,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.3", + "redox_syscall 0.5.7", "smallvec", "windows-targets 0.52.6", ] @@ -6233,11 +6237,12 @@ dependencies = [ [[package]] name = "pbkdf2" -version = "0.11.0" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ "digest 0.10.7", + "hmac 0.12.1", ] [[package]] @@ -6263,9 +6268,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.11" +version = "2.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd53dff83f26735fdc1ca837098ccf133605d794cdae66acfc2bfac3ec809d95" +checksum = "fdbef9d1d47087a895abd220ed25eb4ad973a5e26f6a4367b038c25e28dfc2d9" dependencies = [ "memchr", "thiserror", @@ -6274,9 +6279,9 @@ dependencies = [ [[package]] name = "pest_derive" -version = "2.7.11" +version = "2.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a548d2beca6773b1c244554d36fcf8548a8a58e74156968211567250e48e49a" +checksum = "4d3a6e3394ec80feb3b6393c725571754c6188490265c61aaf260810d6b95aa0" dependencies = [ "pest", "pest_generator", @@ -6284,22 +6289,22 @@ dependencies = [ [[package]] name = "pest_generator" -version = "2.7.11" +version = "2.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c93a82e8d145725dcbaf44e5ea887c8a869efdcc28706df2d08c69e17077183" +checksum = "94429506bde1ca69d1b5601962c73f4172ab4726571a59ea95931218cb0e930e" dependencies = [ "pest", "pest_meta", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] name = "pest_meta" -version = "2.7.11" +version = "2.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a941429fea7e08bedec25e4f6785b6ffaacc6b755da98df5ef3e7dcf4a124c4f" +checksum = "ac8a071862e93690b6e34e9a5fb8e33ff3734473ac0245b27232222c4906a33f" dependencies = [ "once_cell", "pest", @@ -6313,7 +6318,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.2.6", + "indexmap 2.6.0", ] [[package]] @@ -6333,7 +6338,7 @@ checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -6366,9 +6371,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "polkadot-ckb-merkle-mountain-range" @@ -6382,17 +6387,17 @@ dependencies = [ [[package]] name = "polling" -version = "3.7.2" +version = "3.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3ed00ed3fbf728b5816498ecd316d1716eecaced9c0c8d2c5a6740ca214985b" +checksum = "cc2790cd301dec6cd3b7a025e4815cf825724a51c98dccfe6a3e55f05ffb6511" dependencies = [ "cfg-if", "concurrent-queue", "hermit-abi 0.4.0", "pin-project-lite 0.2.14", - "rustix 0.38.34", + "rustix 0.38.37", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6420,9 +6425,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.7.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" [[package]] name = "powerfmt" @@ -6432,9 +6437,12 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.17" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] [[package]] name = "predicates" @@ -6478,12 +6486,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.20" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f12335488a2f3b0a83b14edad48dca9879ce89b2edd10e80237e4e852dd645e" +checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" dependencies = [ "proc-macro2", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -6511,11 +6519,11 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "toml_edit 0.21.1", + "toml_edit 0.22.22", ] [[package]] @@ -6550,7 +6558,7 @@ checksum = "3d1eaa7fa0aa1929ffdf7eeb6eac234dde6268914a14ad44d23521ab6a9b258e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -6596,7 +6604,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -6655,9 +6663,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.21" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +checksum = "aa37f80ca58604976033fae9515a8a2989fc13797d953f7c04fb8fa36a11f205" dependencies = [ "cc", ] @@ -6721,9 +6729,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.36" +version = "1.0.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" dependencies = [ "proc-macro2", ] @@ -6863,27 +6871,18 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "redox_syscall" -version = "0.5.3" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" dependencies = [ "bitflags 2.6.0", ] [[package]] name = "redox_users" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ "getrandom 0.2.15", "libredox", @@ -6907,7 +6906,7 @@ checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -6924,14 +6923,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.5" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.7", - "regex-syntax 0.8.4", + "regex-automata 0.4.8", + "regex-syntax 0.8.5", ] [[package]] @@ -6945,13 +6944,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -6962,9 +6961,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "resolv-conf" @@ -7106,9 +7105,9 @@ dependencies = [ [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver 1.0.23", ] @@ -7138,9 +7137,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags 2.6.0", "errno", @@ -7358,7 +7357,7 @@ dependencies = [ "proc-macro-crate 1.1.3", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -8187,7 +8186,7 @@ dependencies = [ "proc-macro-crate 1.1.3", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -8267,7 +8266,7 @@ version = "2.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d35494501194174bda522a32605929eefc9ecf7e0a326c26db1fdd85881eb62" dependencies = [ - "proc-macro-crate 3.1.0", + "proc-macro-crate 3.2.0", "proc-macro2", "quote", "syn 1.0.109", @@ -8275,11 +8274,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +checksum = "e9aaafd5a2b6e3d657ff009d82fbd630b6bd54dd4eb06f21693925cdf80f9b8b" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -8318,7 +8317,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8de18f6d8ba0aad7045f5feae07ec29899c1112584a38509a84ad7b04451eaa0" dependencies = [ "arrayref", - "arrayvec 0.7.4", + "arrayvec 0.7.6", "curve25519-dalek 4.1.3", "getrandom_or_panic", "merlin 3.0.0", @@ -8406,9 +8405,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -8449,29 +8448,29 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.204" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" +checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.204" +version = "1.0.210" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" +checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] name = "serde_json" -version = "1.0.121" +version = "1.0.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ab380d7d9f22ef3f21ad3e6c1ebe8e4fc7a2000ccba2e4d71fc96f15b2cb609" +checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" dependencies = [ "itoa", "memchr", @@ -8481,9 +8480,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" dependencies = [ "serde", ] @@ -8639,7 +8638,7 @@ dependencies = [ "curve25519-dalek 4.1.3", "rand_core 0.6.4", "ring 0.17.8", - "rustc_version 0.4.0", + "rustc_version 0.4.1", "sha2 0.10.8", "subtle", ] @@ -8713,7 +8712,7 @@ dependencies = [ "proc-macro-crate 1.1.3", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -8872,7 +8871,7 @@ version = "21.0.0" source = "git+https://github.com/paritytech/polkadot-sdk?tag=polkadot-v1.1.0#f60318f68687e601c47de5ad5ca88e2c3f8139a7" dependencies = [ "array-bytes", - "arrayvec 0.7.4", + "arrayvec 0.7.6", "bandersnatch_vrfs", "bitflags 1.3.2", "blake2", @@ -8933,7 +8932,7 @@ source = "git+https://github.com/paritytech/polkadot-sdk?tag=polkadot-v1.1.0#f60 dependencies = [ "quote", "sp-core-hashing", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -8952,7 +8951,7 @@ source = "git+https://github.com/paritytech/polkadot-sdk?tag=polkadot-v1.1.0#f60 dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -9151,7 +9150,7 @@ dependencies = [ "proc-macro-crate 1.1.3", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -9343,7 +9342,7 @@ dependencies = [ "parity-scale-codec", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -9409,9 +9408,9 @@ dependencies = [ [[package]] name = "ss58-registry" -version = "1.47.0" +version = "1.50.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4743ce898933fbff7bbf414f497c459a782d496269644b3d650a398ae6a487ba" +checksum = "43fce22ed1df64d04b262351c8f9d5c6da4f76f79f25ad15529792f893fad25d" dependencies = [ "Inflector", "num-format", @@ -9506,7 +9505,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -9634,9 +9633,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.72" +version = "2.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" dependencies = [ "proc-macro2", "quote", @@ -9684,20 +9683,21 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "target-lexicon" -version = "0.12.15" +version = "0.12.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.10.1" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand", - "rustix 0.38.34", - "windows-sys 0.52.0", + "once_cell", + "rustix 0.38.37", + "windows-sys 0.59.0", ] [[package]] @@ -9717,22 +9717,22 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -9803,14 +9803,13 @@ dependencies = [ [[package]] name = "tiny-bip39" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62cc94d358b5a1e84a5cb9109f559aa3c4d634d2b1b4de3d0fa4adc7c78e2861" +checksum = "4a6e875ccbd782b2d91350816d4ab27da3c9424c381f9ba07ed3e2e1ae680d90" dependencies = [ "anyhow", - "hmac 0.12.1", "once_cell", - "pbkdf2 0.11.0", + "pbkdf2 0.12.2", "rand 0.8.5", "rustc-hash", "sha2 0.10.8", @@ -9846,9 +9845,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.39.2" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", "bytes", @@ -9870,7 +9869,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -9896,9 +9895,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" +checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" dependencies = [ "futures-core", "pin-project-lite 0.2.14", @@ -9908,9 +9907,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.11" +version = "0.7.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" dependencies = [ "bytes", "futures-core", @@ -9943,21 +9942,21 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.16" +version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81967dd0dd2c1ab0bc3468bd7caecc32b8a4aa47d0c8c695d8c2b2108168d62c" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.22.17", + "toml_edit 0.22.22", ] [[package]] name = "toml_datetime" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fb9f64314842840f1d940ac544da178732128f1c78c21772e876579e0da1db" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" dependencies = [ "serde", ] @@ -9968,7 +9967,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", @@ -9977,26 +9976,15 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" -dependencies = [ - "indexmap 2.2.6", - "toml_datetime", - "winnow 0.5.40", -] - -[[package]] -name = "toml_edit" -version = "0.22.17" +version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9f8729f5aea9562aac1cc0441f5d6de3cff1ee0c5d67293eeca5eb36ee7c16" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", - "winnow 0.6.16", + "winnow 0.6.20", ] [[package]] @@ -10030,15 +10018,15 @@ dependencies = [ [[package]] name = "tower-layer" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" [[package]] name = "tower-service" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" @@ -10060,7 +10048,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -10263,9 +10251,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "ucd-trie" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" [[package]] name = "uint" @@ -10281,36 +10269,36 @@ dependencies = [ [[package]] name = "unicode-bidi" -version = "0.3.15" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" +checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" [[package]] name = "unicode-ident" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" [[package]] name = "unicode-normalization" -version = "0.1.23" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" dependencies = [ "tinyvec", ] [[package]] name = "unicode-width" -version = "0.1.13" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-xid" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "universal-hash" @@ -10426,34 +10414,35 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" dependencies = [ "cfg-if", + "once_cell", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.42" +version = "0.4.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" +checksum = "61e9300f63a621e96ed275155c108eb6f843b6a26d053f122ab69724559dc8ed" dependencies = [ "cfg-if", "js-sys", @@ -10463,9 +10452,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -10473,22 +10462,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" [[package]] name = "wasm-instrument" @@ -10810,9 +10799,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.69" +version = "0.3.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" +checksum = "26fdeaafd9bd129f65e7c031593c24d62186301e0c72c8978fa1678be7d532c0" dependencies = [ "js-sys", "wasm-bindgen", @@ -10852,14 +10841,14 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.34", + "rustix 0.38.37", ] [[package]] name = "wide" -version = "0.7.26" +version = "0.7.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901e8597c777fa042e9e245bd56c0dc4418c5db3f845b6ff94fbac732c6a0692" +checksum = "b828f995bf1e9622031f8009f8481a85406ce1f4d4588ff746d872043e855690" dependencies = [ "bytemuck", "safe_arch", @@ -10889,11 +10878,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -10957,6 +10946,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-targets" version = "0.42.2" @@ -11146,9 +11144,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.6.16" +version = "0.6.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b480ae9340fc261e6be3e95a1ba86d54ae3f9171132a73ce8d4bbaf68339507c" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" dependencies = [ "memchr", ] @@ -11242,6 +11240,7 @@ version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ + "byteorder", "zerocopy-derive", ] @@ -11253,7 +11252,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -11273,7 +11272,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.72", + "syn 2.0.79", ] [[package]] @@ -11316,9 +11315,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.12+zstd.1.5.6" +version = "2.0.13+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" +checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" dependencies = [ "cc", "pkg-config", diff --git a/pallets/ddc-verification/src/lib.rs b/pallets/ddc-verification/src/lib.rs index 4cec4c46e..92ffacce3 100644 --- a/pallets/ddc-verification/src/lib.rs +++ b/pallets/ddc-verification/src/lib.rs @@ -43,11 +43,11 @@ use sp_runtime::{ Percent, }; use sp_std::{collections::btree_map::BTreeMap, prelude::*}; - pub mod weights; use itertools::Itertools; use rand::{prelude::*, rngs::SmallRng, SeedableRng}; use sp_staking::StakingInterface; +use sp_std::fmt::Debug; use crate::weights::WeightInfo; @@ -58,7 +58,8 @@ mod tests; #[frame_support::pallet] pub mod pallet { - use ddc_primitives::{BucketId, MergeActivityHash, KEY_TYPE}; + + use ddc_primitives::{AggregatorInfo, BucketId, MergeActivityHash, KEY_TYPE}; use frame_support::PalletId; use sp_core::crypto::AccountId32; use sp_runtime::SaturatedConversion; @@ -70,7 +71,7 @@ pub mod pallet { frame_support::traits::StorageVersion::new(0); const SUCCESS_CODE: u16 = 200; - const BUF_SIZE: usize = 128; + const _BUF_SIZE: usize = 128; const RESPONSE_TIMEOUT: u64 = 20000; #[pallet::pallet] @@ -117,7 +118,11 @@ pub mod pallet { const MAJORITY: u8; /// Block to start from. const BLOCK_TO_START: u16; - const MIN_DAC_NODES_FOR_CONSENSUS: u16; + const DAC_REDUNDANCY_FACTOR: u16; + + #[pallet::constant] + type AggregatorsQuorum: Get; + const MAX_PAYOUT_BATCH_COUNT: u16; const MAX_PAYOUT_BATCH_SIZE: u16; const MAX_MERKLE_NODE_IDENTIFIER: u16; @@ -157,34 +162,6 @@ pub mod pallet { cluster_id: ClusterId, era_id: DdcEra, }, - /// Not enough nodes for consensus. - NotEnoughNodesForConsensus { - cluster_id: ClusterId, - era_id: DdcEra, - node_id: String, - validator: T::AccountId, - }, - /// Not enough buckets for consensus. - NotEnoughBucketsForConsensus { - cluster_id: ClusterId, - era_id: DdcEra, - bucket_id: BucketId, - validator: T::AccountId, - }, - /// Not enough records for consensus. - NotEnoughRecordsForConsensus { - cluster_id: ClusterId, - era_id: DdcEra, - record_id: String, - validator: T::AccountId, - }, - /// No activity in consensus. - ActivityNotInConsensus { - cluster_id: ClusterId, - era_id: DdcEra, - id: ActivityHash, - validator: T::AccountId, - }, /// Node Usage Retrieval Error. NodeUsageRetrievalError { cluster_id: ClusterId, @@ -192,8 +169,8 @@ pub mod pallet { node_pub_key: NodePubKey, validator: T::AccountId, }, - /// Customer Usage Retrieval Error. - CustomerUsageRetrievalError { + /// Bucket aggregates Retrieval Error. + BucketAggregatesRetrievalError { cluster_id: ClusterId, era_id: DdcEra, node_pub_key: NodePubKey, @@ -294,6 +271,12 @@ pub mod pallet { ValidatorKeySet { validator: T::AccountId, }, + FailedToFetchClusterNodes { + validator: T::AccountId, + }, + FailedToFetchDacNodes { + validator: T::AccountId, + }, FailedToFetchNodeTotalUsage { cluster_id: ClusterId, node_pub_key: NodePubKey, @@ -308,77 +291,41 @@ pub mod pallet { payers_batch_merkle_root_hashes: Vec, payees_batch_merkle_root_hashes: Vec, }, - BucketAggregatesRetrievalError { + BucketAggregateRetrievalError { cluster_id: ClusterId, era_id: DdcEra, bucket_id: BucketId, node_pub_key: NodePubKey, validator: T::AccountId, }, - NotEnoughNodeAggregatesForConsensus { - cluster_id: ClusterId, - era_id: DdcEra, - bucket_id: BucketId, - node_id: String, - validator: T::AccountId, - }, - BucketAggregateActivityNotInConsensus { + ChallengeResponseRetrievalError { cluster_id: ClusterId, era_id: DdcEra, - id: ActivityHash, - node_ids: Vec, + aggregate_key: AggregateKey, + aggregator: NodePubKey, validator: T::AccountId, }, - ChallengeResponseRetrievalError { + TraverseResponseRetrievalError { cluster_id: ClusterId, era_id: DdcEra, - node_id: String, - bucket_id: BucketId, - node_pub_key: NodePubKey, + aggregate_key: AggregateKey, + aggregator: NodePubKey, validator: T::AccountId, }, + EmptyConsistentGroup, } /// Consensus Errors #[derive(Debug, Encode, Decode, Clone, TypeInfo, PartialEq)] pub enum OCWError { - /// Not enough nodes for consensus. - NotEnoughNodesForConsensus { - cluster_id: ClusterId, - era_id: DdcEra, - node_id: String, - }, - NotEnoughBucketsForConsensus { - cluster_id: ClusterId, - era_id: DdcEra, - bucket_id: BucketId, - }, - NotEnoughRecordsForConsensus { - cluster_id: ClusterId, - era_id: DdcEra, - record_id: String, - }, - /// No activity in consensus. - ActivityNotInConsensus { - cluster_id: ClusterId, - era_id: DdcEra, - id: ActivityHash, - }, - /// No Bucket Aggregate activity in consensus. - BucketAggregateActivityNotInConsensus { - cluster_id: ClusterId, - era_id: DdcEra, - id: ActivityHash, - node_ids: Vec, - }, /// Node Usage Retrieval Error. NodeUsageRetrievalError { cluster_id: ClusterId, era_id: DdcEra, node_pub_key: NodePubKey, }, - /// Customer Usage Retrieval Error. - CustomerUsageRetrievalError { + /// Bucket aggregates Retrieval Error. + BucketAggregatesRetrievalError { cluster_id: ClusterId, era_id: DdcEra, node_pub_key: NodePubKey, @@ -387,8 +334,8 @@ pub mod pallet { cluster_id: ClusterId, node_pub_key: NodePubKey, }, - /// Bucket aggregates Retrieval Error. - BucketAggregatesRetrievalError { + /// Bucket aggregate Retrieval Error. + BucketAggregateRetrievalError { cluster_id: ClusterId, era_id: DdcEra, bucket_id: BucketId, @@ -398,9 +345,15 @@ pub mod pallet { ChallengeResponseRetrievalError { cluster_id: ClusterId, era_id: DdcEra, - node_id: String, - bucket_id: BucketId, - node_pub_key: NodePubKey, + aggregate_key: AggregateKey, + aggregator: NodePubKey, + }, + /// Traverse Response Retrieval Error. + TraverseResponseRetrievalError { + cluster_id: ClusterId, + era_id: DdcEra, + aggregate_key: AggregateKey, + aggregator: NodePubKey, }, PrepareEraTransactionError { cluster_id: ClusterId, @@ -468,17 +421,13 @@ pub mod pallet { }, FailedToFetchCurrentValidator, FailedToFetchNodeProvider, + FailedToFetchClusterNodes, + FailedToFetchDacNodes, FailedToFetchNodeTotalUsage { cluster_id: ClusterId, node_pub_key: NodePubKey, }, - /// Not enough subaggregates for consensus. - NotEnoughNodeAggregatesForConsensus { - cluster_id: ClusterId, - era_id: DdcEra, - bucket_id: BucketId, - node_id: String, - }, + EmptyConsistentGroup, } #[pallet::error] @@ -605,11 +554,11 @@ pub mod pallet { pub(crate) batch_proof: MMRProof, } - /// Node activity of a node. + /// Node aggregate response from aggregator. #[derive( Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode, )] - pub(crate) struct NodeActivity { + pub(crate) struct NodeAggregateResponse { /// Node id. pub(crate) node_id: String, /// Total amount of stored bytes. @@ -622,23 +571,31 @@ pub mod pallet { pub(crate) number_of_gets: u64, } - /// Customer Activity of a bucket. + /// Bucket aggregate response from aggregator. #[derive( Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode, )] - pub(crate) struct CustomerActivity { + pub(crate) struct BucketAggregateResponse { /// Bucket id pub(crate) bucket_id: BucketId, - /// SubAggregates. - pub(crate) sub_aggregates: Vec, + /// Total amount of stored bytes. + pub(crate) stored_bytes: i64, + /// Total amount of transferred bytes. + pub(crate) transferred_bytes: u64, + /// Total number of puts. + pub(crate) number_of_puts: u64, + /// Total number of gets. + pub(crate) number_of_gets: u64, + /// Bucket sub aggregates. + pub(crate) sub_aggregates: Vec, } - /// Sub Aggregates of a bucket. + /// Sub aggregates of a bucket. #[derive( Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode, )] #[allow(non_snake_case)] - pub(crate) struct BucketSubAggregate { + pub(crate) struct BucketSubAggregateResponse { /// Node id. pub(crate) NodeID: String, /// Total amount of stored bytes. @@ -651,11 +608,11 @@ pub mod pallet { pub(crate) number_of_gets: u64, } - /// Bucket aggregate by bucket id. + /// Bucket activity per a DDC node. #[derive( - Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode, + Debug, Serialize, Deserialize, Clone, Ord, PartialOrd, PartialEq, Eq, Encode, Decode, )] - pub(crate) struct BucketNodeAggregatesActivity { + pub(crate) struct BucketSubAggregate { /// Bucket id pub(crate) bucket_id: BucketId, /// Node id. @@ -668,6 +625,26 @@ pub mod pallet { pub(crate) number_of_puts: u64, /// Total number of gets. pub(crate) number_of_gets: u64, + /// Aggregator data. + pub(crate) aggregator: AggregatorInfo, + } + + #[derive( + Debug, Serialize, Deserialize, Clone, Ord, PartialOrd, PartialEq, Eq, Encode, Decode, + )] + pub(crate) struct NodeAggregate { + /// Node id. + pub(crate) node_id: String, + /// Total amount of stored bytes. + pub(crate) stored_bytes: i64, + /// Total amount of transferred bytes. + pub(crate) transferred_bytes: u64, + /// Total number of puts. + pub(crate) number_of_puts: u64, + /// Total number of gets. + pub(crate) number_of_gets: u64, + /// Node data. + pub(crate) aggregator: AggregatorInfo, } /// Challenge Response @@ -763,49 +740,62 @@ pub mod pallet { pub value: String, } - // Define a common trait - pub trait Activity: - Clone + Ord + PartialEq + Eq + Serialize + for<'de> Deserialize<'de> - { - fn get_consensus_id(&self) -> ActivityHash; - fn hash(&self) -> ActivityHash; - - fn get_consensus_error(&self, cluster_id: ClusterId, era_id: DdcEra) -> OCWError; + #[derive( + Debug, Serialize, Deserialize, Clone, Hash, Ord, PartialOrd, PartialEq, Eq, Encode, Decode, + )] + pub(crate) struct MerkleTreeNodeResponse { + merkle_tree_node_id: u64, + hash: String, + stored_bytes: i64, + transferred_bytes: u64, + number_of_puts: u64, + number_of_gets: u64, } - impl Activity for NodeActivity { - fn get_consensus_id(&self) -> ActivityHash { - T::ActivityHasher::hash(self.node_id.as_bytes()).into() + #[derive(Debug, Clone, PartialEq)] + pub(crate) struct ConsistentGroup(pub ActivityHash, pub Vec); + impl ConsistentGroup { + pub fn hash(&self) -> ActivityHash { + self.0 } - fn hash(&self) -> ActivityHash { - T::ActivityHasher::hash(&self.encode()).into() + pub fn get(&self, idx: usize) -> Option<&A> { + self.1.get(idx) } - fn get_consensus_error(&self, cluster_id: ClusterId, era_id: DdcEra) -> OCWError { - let node_id = &self.node_id; + pub fn len(&self) -> usize { + self.1.len() + } - OCWError::NotEnoughNodesForConsensus { cluster_id, era_id, node_id: node_id.clone() } + pub fn _items(&self) -> &Vec { + &self.1 } } - impl Activity for CustomerActivity { - fn get_consensus_id(&self) -> ActivityHash { - T::ActivityHasher::hash(&self.bucket_id.encode()).into() - } - fn hash(&self) -> ActivityHash { - T::ActivityHasher::hash(&self.encode()).into() - } + #[derive(Debug, Clone, PartialEq)] + pub(crate) struct ConsistentGroups { + pub in_consensus: Vec>, + pub in_quorum: Vec>, + pub in_others: Vec>, + } - fn get_consensus_error(&self, cluster_id: ClusterId, era_id: DdcEra) -> OCWError { - let bucket_id = &self.bucket_id; + #[derive(Debug, Clone, Encode, Decode, TypeInfo, PartialEq)] + pub enum AggregateKey { + NodeAggregateKey(String), + BucketSubAggregateKey(BucketId, String), + } - OCWError::NotEnoughBucketsForConsensus { cluster_id, era_id, bucket_id: *bucket_id } - } + pub(crate) trait Aggregate: + Clone + Ord + PartialEq + Eq + Serialize + for<'de> Deserialize<'de> + Debug + { + fn hash(&self) -> ActivityHash; + fn get_key(&self) -> AggregateKey; + fn get_number_of_leaves(&self) -> u64; + fn get_aggregator(&self) -> AggregatorInfo; } - impl Activity for BucketNodeAggregatesActivity { - fn get_consensus_id(&self) -> ActivityHash { + impl Aggregate for BucketSubAggregate { + fn hash(&self) -> ActivityHash { let mut data = self.bucket_id.encode(); data.extend_from_slice(&self.node_id.encode()); data.extend_from_slice(&self.stored_bytes.encode()); @@ -815,44 +805,71 @@ pub mod pallet { T::ActivityHasher::hash(&data).into() } - fn hash(&self) -> ActivityHash { - T::ActivityHasher::hash(&self.encode()).into() + fn get_key(&self) -> AggregateKey { + AggregateKey::BucketSubAggregateKey(self.bucket_id, self.node_id.clone()) } - fn get_consensus_error(&self, cluster_id: ClusterId, era_id: DdcEra) -> OCWError { - let node_id = &self.node_id; - let bucket_id = &self.bucket_id; + fn get_number_of_leaves(&self) -> u64 { + self.number_of_gets.saturating_add(self.number_of_puts) + } - OCWError::NotEnoughNodeAggregatesForConsensus { - cluster_id, - era_id, - bucket_id: *bucket_id, - node_id: node_id.clone(), - } + fn get_aggregator(&self) -> AggregatorInfo { + self.aggregator.clone() } } - impl Activity for Leaf { - fn get_consensus_id(&self) -> ActivityHash { - T::ActivityHasher::hash(&self.record.id.encode()).into() + impl Aggregate for NodeAggregate { + fn hash(&self) -> ActivityHash { + let mut data = self.node_id.encode(); + data.extend_from_slice(&self.stored_bytes.encode()); + data.extend_from_slice(&self.transferred_bytes.encode()); + data.extend_from_slice(&self.number_of_puts.encode()); + data.extend_from_slice(&self.number_of_gets.encode()); + T::ActivityHasher::hash(&data).into() + } + + fn get_key(&self) -> AggregateKey { + AggregateKey::NodeAggregateKey(self.node_id.clone()) } - fn hash(&self) -> ActivityHash { + fn get_aggregator(&self) -> AggregatorInfo { + self.aggregator.clone() + } + + fn get_number_of_leaves(&self) -> u64 { + self.number_of_gets.saturating_add(self.number_of_puts) + } + } + pub trait NodeAggregateLeaf: + Clone + Ord + PartialEq + Eq + Serialize + for<'de> Deserialize<'de> + { + fn leaf_hash(&self) -> ActivityHash; + } + + pub trait BucketSubAggregateLeaf: + Clone + Ord + PartialEq + Eq + Serialize + for<'de> Deserialize<'de> + { + fn leaf_hash(&self) -> ActivityHash; + } + + impl NodeAggregateLeaf for Leaf { + fn leaf_hash(&self) -> ActivityHash { let mut data = self.record.id.encode(); data.extend_from_slice(&self.record.upstream.request.requestType.encode()); data.extend_from_slice(&self.stored_bytes.encode()); data.extend_from_slice(&self.transferred_bytes.encode()); T::ActivityHasher::hash(&data).into() } + } - fn get_consensus_error(&self, cluster_id: ClusterId, era_id: DdcEra) -> OCWError { - let record_id = &self.record.id; - - OCWError::NotEnoughRecordsForConsensus { - cluster_id, - era_id, - record_id: record_id.clone(), - } + impl BucketSubAggregateLeaf for Leaf { + fn leaf_hash(&self) -> ActivityHash { + let mut data = self.record.upstream.request.bucketId.encode(); + data.extend_from_slice(&self.record.encode()); + data.extend_from_slice(&self.record.upstream.request.requestType.encode()); + data.extend_from_slice(&self.stored_bytes.encode()); + data.extend_from_slice(&self.transferred_bytes.encode()); + T::ActivityHasher::hash(&data).into() } } @@ -905,25 +922,18 @@ pub mod pallet { log::info!("👋 Hello from pallet-ddc-verification."); - // todo! fetch clusters from ddc-clusters and loop the whole process for each cluster + // todo: fetch clusters from ddc-clusters and loop the whole process for each cluster let cluster_id = unwrap_or_log_error!( Self::get_cluster_to_validate(), "🏭❌ Error retrieving cluster to validate" ); - let dac_nodes = unwrap_or_log_error!( - Self::get_dac_nodes(&cluster_id), - "🏭❌ Error retrieving dac nodes to validate" - ); - - let min_nodes = T::MIN_DAC_NODES_FOR_CONSENSUS; let batch_size = T::MAX_PAYOUT_BATCH_SIZE; let mut errors: Vec = Vec::new(); - let processed_dac_data = - Self::process_dac_data(&cluster_id, None, &dac_nodes, min_nodes, batch_size.into()); + let dac_era_result = Self::process_dac_era(&cluster_id, None, batch_size.into()); - match processed_dac_data { + match dac_era_result { Ok(Some(( era_activity, payers_merkle_root_hash, @@ -1033,12 +1043,7 @@ pub mod pallet { } // todo! factor out as macro as this is repetitive - match Self::prepare_begin_charging_customers( - &cluster_id, - &dac_nodes, - min_nodes, - batch_size.into(), - ) { + match Self::prepare_begin_charging_customers(&cluster_id, batch_size.into()) { Ok(Some((era_id, max_batch_index))) => { log::info!( "🏭🎁 prepare_begin_charging_customers processed successfully for cluster_id: {:?}, era_id: {:?}", @@ -1087,12 +1092,7 @@ pub mod pallet { } // todo! factor out as macro as this is repetitive - match Self::prepare_send_charging_customers_batch( - &cluster_id, - batch_size.into(), - &dac_nodes, - min_nodes, - ) { + match Self::prepare_send_charging_customers_batch(&cluster_id, batch_size.into()) { Ok(Some((era_id, batch_payout))) => { let payers_log: Vec<(String, BucketId, CustomerUsage)> = batch_payout .payers @@ -1212,12 +1212,7 @@ pub mod pallet { } // todo! factor out as macro as this is repetitive - match Self::prepare_begin_rewarding_providers( - &cluster_id, - &dac_nodes, - min_nodes, - batch_size.into(), - ) { + match Self::prepare_begin_rewarding_providers(&cluster_id, batch_size.into()) { Ok(Some((era_id, max_batch_index, total_node_usage))) => { log::info!( "🏭📝prepare_begin_rewarding_providers processed successfully for cluster_id: {:?}, era_id: {:?}", @@ -1272,12 +1267,7 @@ pub mod pallet { } // todo! factor out as macro as this is repetitive - match Self::prepare_send_rewarding_providers_batch( - &cluster_id, - batch_size.into(), - &dac_nodes, - min_nodes, - ) { + match Self::prepare_send_rewarding_providers_batch(&cluster_id, batch_size.into()) { Ok(Some((era_id, batch_payout))) => { log::info!( "🎁 prepare_send_rewarding_providers_batch processed successfully for cluster_id: {:?}, era_id: {:?}", @@ -1455,22 +1445,20 @@ pub mod pallet { impl Pallet { #[allow(clippy::type_complexity)] - pub(crate) fn process_dac_data( + pub(crate) fn process_dac_era( cluster_id: &ClusterId, era_id_to_process: Option, - dac_nodes: &[(NodePubKey, StorageNodeParams)], - min_nodes: u16, batch_size: usize, ) -> Result< Option<(EraActivity, ActivityHash, ActivityHash, Vec, Vec)>, Vec, > { log::info!("🚀 Processing dac data for cluster_id: {:?}", cluster_id); - // todo! Need to debug follwing condition. Why it is not working on Devnet - // if dac_nodes.len().ilog2() < min_nodes.into() { - // return Err(vec![OCWError::NotEnoughDACNodes { num_nodes: min_nodes }]); - // } + let dac_nodes = Self::get_dac_nodes(cluster_id).map_err(|_| { + log::error!("🏭❌ Error retrieving dac nodes to validate cluster {:?}", cluster_id); + vec![OCWError::FailedToFetchDacNodes] + })?; let era_activity = if let Some(era_activity) = era_id_to_process { EraActivity { @@ -1479,40 +1467,38 @@ pub mod pallet { end: era_activity.end, } } else { - match Self::get_era_for_validation(cluster_id, dac_nodes) { + match Self::get_era_for_validation(cluster_id, &dac_nodes) { Ok(Some(era_activity)) => era_activity, Ok(None) => return Ok(None), Err(err) => return Err(vec![err]), } }; - let nodes_usage = - Self::fetch_nodes_usage_for_era(cluster_id, era_activity.id, dac_nodes) - .map_err(|err| vec![err])?; - let customers_usage = - Self::fetch_customers_usage_for_era(cluster_id, era_activity.id, dac_nodes) - .map_err(|err| vec![err])?; + // todo: move to cluster protocol parameters + let dac_redundancy_factor = T::DAC_REDUNDANCY_FACTOR; + let aggregators_quorum = T::AggregatorsQuorum::get(); - let (bucket_node_aggregates_in_consensus, bucket_node_aggregates_not_in_consensus) = - Self::fetch_sub_trees(cluster_id, era_activity.id, customers_usage, min_nodes)?; + let nodes_aggregates_by_aggregator = + Self::fetch_nodes_aggregates_for_era(cluster_id, era_activity.id, &dac_nodes) + .map_err(|err| vec![err])?; - let mut bucket_aggregates_passed_challenges: Vec = vec![]; + let buckets_aggregates_by_aggregator = + Self::fetch_buckets_aggregates_for_era(cluster_id, era_activity.id, &dac_nodes) + .map_err(|err| vec![err])?; - if !bucket_node_aggregates_not_in_consensus.is_empty() { - bucket_aggregates_passed_challenges = - Self::challenge_and_find_valid_sub_aggregates_not_in_consensus( - cluster_id, - era_activity.id, - dac_nodes, - bucket_node_aggregates_not_in_consensus, - )?; - } + let buckets_sub_aggregates_groups = Self::group_buckets_sub_aggregates_by_consistency( + cluster_id, + era_activity.id, + buckets_aggregates_by_aggregator, + dac_redundancy_factor, + aggregators_quorum, + ); - let mut total_bucket_aggregates = bucket_node_aggregates_in_consensus.clone(); - total_bucket_aggregates.extend(bucket_aggregates_passed_challenges); + let total_buckets_usage = + Self::get_total_usage(cluster_id, era_activity.id, buckets_sub_aggregates_groups)?; let customer_activity_hashes: Vec = - total_bucket_aggregates.clone().into_iter().map(|c| c.hash::()).collect(); + total_buckets_usage.clone().into_iter().map(|c| c.hash::()).collect(); let customer_activity_hashes_string: Vec = customer_activity_hashes.clone().into_iter().map(hex::encode).collect(); @@ -1526,7 +1512,7 @@ pub mod pallet { let customers_activity_batch_roots = Self::convert_to_batch_merkle_roots( cluster_id, era_activity.id, - Self::split_to_batches(&bucket_node_aggregates_in_consensus, batch_size), + Self::split_to_batches(&total_buckets_usage, batch_size), ) .map_err(|err| vec![err])?; @@ -1559,16 +1545,19 @@ pub mod pallet { customer_batch_roots_string, ); - let nodes_activity_in_consensus = Self::get_consensus_for_activities( + let nodes_aggregates_groups = Self::group_nodes_aggregates_by_consistency( cluster_id, era_activity.id, - &nodes_usage, - min_nodes, - Percent::from_percent(T::MAJORITY), - )?; + nodes_aggregates_by_aggregator, + dac_redundancy_factor, + aggregators_quorum, + ); + + let total_nodes_usage = + Self::get_total_usage(cluster_id, era_activity.id, nodes_aggregates_groups)?; let node_activity_hashes: Vec = - nodes_activity_in_consensus.clone().into_iter().map(|c| c.hash::()).collect(); + total_nodes_usage.clone().into_iter().map(|c| c.hash::()).collect(); let node_activity_hashes_string: Vec = node_activity_hashes.clone().into_iter().map(hex::encode).collect(); @@ -1583,7 +1572,7 @@ pub mod pallet { let nodes_activity_batch_roots = Self::convert_to_batch_merkle_roots( cluster_id, era_activity.id, - Self::split_to_batches(&nodes_activity_in_consensus, batch_size), + Self::split_to_batches(&total_nodes_usage, batch_size), ) .map_err(|err| vec![err])?; @@ -1616,10 +1605,10 @@ pub mod pallet { Self::store_validation_activities( cluster_id, era_activity.id, - &bucket_node_aggregates_in_consensus, + &total_buckets_usage, customers_activity_root, &customers_activity_batch_roots, - &nodes_activity_in_consensus, + &total_nodes_usage, nodes_activity_root, &nodes_activity_batch_roots, ); @@ -1633,196 +1622,280 @@ pub mod pallet { ))) } - pub(crate) fn challenge_and_find_valid_sub_aggregates_not_in_consensus( + pub(crate) fn get_total_usage( cluster_id: &ClusterId, era_id: DdcEra, - dac_nodes: &[(NodePubKey, StorageNodeParams)], - bucket_node_aggregates_not_in_consensus: Vec, - ) -> Result, Vec> { - let mut bucket_aggregates_passed_challenges: Vec = vec![]; - let mut bucket_aggregates_not_passed_challenges: Vec = - vec![]; - let number_of_identifiers = T::MAX_MERKLE_NODE_IDENTIFIER; + consistent_groups: ConsistentGroups, + ) -> Result, Vec> { + let mut total_usage: Vec = vec![]; - log::info!( - "🚀 Challenge process starts when bucket sub aggregates are not in consensus!" - ); + // todo: run a light challenge for unanimous consensus + let in_consensus_usage = consistent_groups + .in_consensus + .clone() + .into_iter() + .map(|g| g.get(0).ok_or(vec![OCWError::EmptyConsistentGroup]).cloned()) + .collect::, _>>()?; + total_usage.extend(in_consensus_usage); + + // todo: run a light challenge for quorum, i.e. for majority + let in_quorum_usage = consistent_groups + .in_quorum + .clone() + .into_iter() + .map(|g| g.get(0).ok_or(vec![OCWError::EmptyConsistentGroup]).cloned()) + .collect::, _>>()?; + total_usage.extend(in_quorum_usage); - for bucket_node_aggregate_activity in bucket_node_aggregates_not_in_consensus { - let merkle_node_ids = Self::find_random_merkle_node_ids( - number_of_identifiers.into(), - bucket_node_aggregate_activity.clone(), - ); - let bucket_id = bucket_node_aggregate_activity.clone().bucket_id; + let verified_usage = + Self::challenge_others(cluster_id, era_id, consistent_groups.in_others)?; - log::info!("🚀 Merkle Node Identifiers for bucket_node_aggregate_activity: node id: {:?} bucket_id:{:?} identifiers: {:?}", - bucket_node_aggregate_activity.clone().node_id, bucket_id.clone(), merkle_node_ids); + if !verified_usage.is_empty() { + total_usage.extend(verified_usage); + } - let challenge_responses = Self::fetch_challenge_responses( - cluster_id, - bucket_node_aggregate_activity.clone().node_id, - era_id, - bucket_id, - merkle_node_ids, - dac_nodes, - ) - .map_err(|err| vec![err])?; - - log::info!("🚀 Fetched challenge response node id: {:?} bucket_id:{:?} challenge_response: {:?}", - bucket_node_aggregate_activity.clone().node_id, bucket_id.clone(), challenge_responses); - - let resulting_hash_from_leafs_and_paths = - Self::find_resulting_hash_from_leafs_and_paths( - challenge_responses, - cluster_id, - era_id, - bucket_id, - bucket_node_aggregate_activity.clone().node_id, - )?; + Ok(total_usage) + } - let root_challenge_responses = Self::fetch_challenge_responses( - cluster_id, - bucket_node_aggregate_activity.clone().node_id, - era_id, - bucket_id, - vec![1], - dac_nodes, - ) - .map_err(|err| vec![err])?; + pub(crate) fn challenge_others( + _cluster_id: &ClusterId, + _era_id: DdcEra, + others: Vec>, + ) -> Result, Vec> { + let redundancy_factor = T::DAC_REDUNDANCY_FACTOR; + let mut verified_usage: Vec = vec![]; - log::info!("🚀 Fetched Root challenge response node id: {:?} bucket_id:{:?} challenge_response: {:?}", - bucket_node_aggregate_activity.clone().node_id, bucket_id.clone(), root_challenge_responses); + for group in others { + if group.len() > redundancy_factor.into() { + log::info!( + "⚠️ Number of consistent aggregates exceeds the redundancy factor {:?}", + group.hash() + ); - let merkle_root_hash = Self::find_resulting_hash_from_leafs_and_paths( - root_challenge_responses, - cluster_id, - era_id, - bucket_id, - bucket_node_aggregate_activity.clone().node_id, - )?; + let excessive_aggregate = + group.get(0).ok_or(vec![OCWError::EmptyConsistentGroup]).cloned()?; - if resulting_hash_from_leafs_and_paths == merkle_root_hash { log::info!( - "🚀👍 The node id: {:?} with bucket_id:{:?} has passed the challenge. The activity detail is {:?}", - bucket_node_aggregate_activity.clone().node_id, - bucket_id, - bucket_node_aggregate_activity + "🔎‍ Challenging excessive aggregate {:?}", + excessive_aggregate.hash::() ); - bucket_aggregates_passed_challenges.push(bucket_node_aggregate_activity); + // todo: run a challenge dedicated to the excessive number of aggregates. + // we assume it won't happen at the moment, so we just take the aggregate to + // payouts stage + verified_usage.push(excessive_aggregate); } else { + let defective_aggregate = + group.get(0).ok_or(vec![OCWError::EmptyConsistentGroup]).cloned()?; + log::info!( - "🚀👎 The node id: {:?} with bucket_id:{:?} has not passed the challenge. The activity detail is {:?}", - bucket_node_aggregate_activity.clone().node_id, - bucket_id, - bucket_node_aggregate_activity + "🔎‍ Challenging defective aggregate {:?}", + defective_aggregate.hash::() ); - bucket_aggregates_not_passed_challenges.push(bucket_node_aggregate_activity); + let is_passed = true; + // todo: run an intensive challenge for deviating aggregate + // let is_passed = Self::_challenge_aggregate(_cluster_id, _era_id, + // &defective_aggregate)?; + if is_passed { + // we assume all aggregates are valid at the moment, so we just take the + // aggregate to payouts stage + verified_usage.push(defective_aggregate); + } } } - let mut data_grouped = Vec::new(); - for (key, chunk) in - &bucket_aggregates_passed_challenges.into_iter().chunk_by(|elt| elt.bucket_id) - { - data_grouped.push((key, chunk.collect())); - } - - Ok(Self::fetch_valid_aggregates_passed_challenges(data_grouped)) + Ok(verified_usage) } - pub(crate) fn fetch_valid_aggregates_passed_challenges( - bucket_aggregates_passed_challenges: Vec<(BucketId, Vec)>, - ) -> Vec { - let mut valid_aggregates_passed_challenges: Vec = vec![]; + pub(crate) fn _challenge_aggregate( + cluster_id: &ClusterId, + era_id: DdcEra, + aggregate: &A, + ) -> Result> { + let number_of_identifiers = T::MAX_MERKLE_NODE_IDENTIFIER; - for (bucket_id, bucket_aggregates_passed_challenge_activities) in - bucket_aggregates_passed_challenges - { - let valid_activities = - bucket_aggregates_passed_challenge_activities.iter().cloned().max_by_key( - |activity| activity.transferred_bytes as i64 + activity.stored_bytes, + log::info!( + "🚀 Challenge process starts when bucket sub aggregates are not in consensus!" + ); + + let aggregate_key = aggregate.get_key(); + let merkle_node_ids = Self::_find_random_merkle_node_ids( + number_of_identifiers.into(), + aggregate.get_number_of_leaves(), + aggregate_key.clone(), + ); + + log::info!( + "🚀 Merkle Node Identifiers for aggregate key: {:?} identifiers: {:?}", + aggregate_key, + merkle_node_ids + ); + + let aggregator = aggregate.get_aggregator(); + + let challenge_response = Self::_fetch_challenge_responses( + cluster_id, + era_id, + aggregate_key.clone(), + merkle_node_ids, + aggregator.clone(), + ) + .map_err(|err| vec![err])?; + + log::info!( + "🚀 Fetched challenge response for aggregate key: {:?}, challenge_response: {:?}", + aggregate_key, + challenge_response + ); + + let calculated_merkle_root = Self::_get_hash_from_merkle_path( + challenge_response, + cluster_id, + era_id, + aggregate_key.clone(), + )?; + + log::info!( + "🚀 Calculated merkle root for aggregate key: {:?}, calculated_merkle_root: {:?}", + aggregate_key, + calculated_merkle_root + ); + + let traverse_response = Self::_fetch_traverse_response( + era_id, + aggregate_key.clone(), + vec![1], + 1, + &aggregator.node_params, + ) + .map_err(|_| { + vec![OCWError::TraverseResponseRetrievalError { + cluster_id: *cluster_id, + era_id, + aggregate_key: aggregate_key.clone(), + aggregator: aggregator.node_pub_key, + }] + })?; + + if let Some(root_merkle_node) = traverse_response.first() { + let mut merkle_root_buf = [0u8; _BUF_SIZE]; + let bytes = + Base64::decode(root_merkle_node.hash.clone(), &mut merkle_root_buf).unwrap(); // todo! remove unwrap + let traversed_merkle_root = ActivityHash::from(sp_core::H256::from_slice(bytes)); + + log::info!( + "🚀 Fetched merkle root for aggregate key: {:?} traversed_merkle_root: {:?}", + aggregate_key, + traversed_merkle_root + ); + + let is_matched = if calculated_merkle_root == traversed_merkle_root { + log::info!( + "🚀👍 The aggregate with hash {:?} and key {:?} has passed the challenge.", + aggregate.hash::(), + aggregate_key, ); - if let Some(activity) = valid_activities { + true + } else { log::info!( - "🚀⛳️ The activity bucket_id:{:?} with maximum usage, which has passed the challenge. The activity detail is {:?}", - bucket_id, - activity + "🚀👎 The aggregate with hash {:?} and key {:?} has not passed the challenge.", + aggregate.hash::(), + aggregate_key, ); - valid_aggregates_passed_challenges.push(activity); - } + + false + }; + + Ok(is_matched) + } else { + Ok(false) } - valid_aggregates_passed_challenges } - pub(crate) fn find_resulting_hash_from_leafs_and_paths( - challenge_responses: Vec, + pub(crate) fn _get_hash_from_merkle_path( + challenge_response: ChallengeAggregateResponse, cluster_id: &ClusterId, era_id: DdcEra, - bucket_id: BucketId, - node_id: String, + aggregate_key: AggregateKey, ) -> Result> { - let mut resulting_hash_from_leafs_and_paths = ActivityHash::default(); + log::info!("Getting hash from merkle tree path for aggregate key: {:?}", aggregate_key); - for challenge_response in challenge_responses { - for proof in challenge_response.proofs { - let leaf_record_hashes: Vec = - proof.leafs.into_iter().map(|p| p.hash::()).collect(); + let mut resulting_hash = ActivityHash::default(); - let leaf_record_hashes_string: Vec = - leaf_record_hashes.clone().into_iter().map(hex::encode).collect(); + for proof in challenge_response.proofs { + let leaf_record_hashes: Vec = match aggregate_key { + AggregateKey::BucketSubAggregateKey(_, _) => proof + .leafs + .into_iter() + .map(|p| NodeAggregateLeaf::leaf_hash::(&p)) + .collect(), + AggregateKey::NodeAggregateKey(_) => proof + .leafs + .into_iter() + .map(|p| BucketSubAggregateLeaf::leaf_hash::(&p)) + .collect(), + }; - log::info!("🚀 Fetched leaf record hashes node id: {:?} bucket_id:{:?} leaf_record_hashes: {:?}", - node_id, bucket_id.clone(), leaf_record_hashes_string); + let leaf_record_hashes_string: Vec = + leaf_record_hashes.clone().into_iter().map(hex::encode).collect(); - let leaf_node_root = - Self::create_merkle_root(cluster_id, era_id, &leaf_record_hashes) - .map_err(|err| vec![err])?; + log::info!( + "🚀 Fetched leaf record hashes aggregate key: {:?} leaf_record_hashes: {:?}", + aggregate_key, + leaf_record_hashes_string + ); - log::info!("🚀 Fetched leaf record root node id: {:?} bucket_id:{:?} leaf_record_root_hash: {:?}", - node_id, bucket_id.clone(), hex::encode(leaf_node_root)); + let leaf_node_root = + Self::create_merkle_root(cluster_id, era_id, &leaf_record_hashes) + .map_err(|err| vec![err])?; - let paths = proof.path.iter().rev(); + log::info!( + "🚀 Fetched leaf record root aggregate key: {:?} leaf_record_root_hash: {:?}", + aggregate_key, + hex::encode(leaf_node_root) + ); - resulting_hash_from_leafs_and_paths = leaf_node_root; - for path in paths { - let mut dec_buf = [0u8; BUF_SIZE]; - let bytes = Base64::decode(path, &mut dec_buf).unwrap(); // todo! remove unwrap - let path_hash: ActivityHash = - ActivityHash::from(sp_core::H256::from_slice(bytes)); + let paths = proof.path.iter().rev(); - let node_root = Self::create_merkle_root( - cluster_id, - era_id, - &[resulting_hash_from_leafs_and_paths, path_hash], - ) - .map_err(|err| vec![err])?; + resulting_hash = leaf_node_root; + for path in paths { + let mut dec_buf = [0u8; _BUF_SIZE]; + let bytes = Base64::decode(path, &mut dec_buf).unwrap(); // todo! remove unwrap + let path_hash: ActivityHash = + ActivityHash::from(sp_core::H256::from_slice(bytes)); + + let node_root = + Self::create_merkle_root(cluster_id, era_id, &[resulting_hash, path_hash]) + .map_err(|err| vec![err])?; - log::info!("🚀 Fetched leaf node root node id: {:?} bucket_id:{:?} for path:{:?} leaf_node_hash: {:?}", - node_id, bucket_id, path, hex::encode(node_root)); + log::info!("🚀 Fetched leaf node root aggregate_key: {:?} for path:{:?} leaf_node_hash: {:?}", + aggregate_key, path, hex::encode(node_root)); - resulting_hash_from_leafs_and_paths = node_root; - } + resulting_hash = node_root; } } - Ok(resulting_hash_from_leafs_and_paths) + Ok(resulting_hash) } - pub(crate) fn find_random_merkle_node_ids( + + pub(crate) fn _find_random_merkle_node_ids( number_of_identifiers: usize, - bucket_node_aggregates_activity: BucketNodeAggregatesActivity, + number_of_leaves: u64, + aggregate_key: AggregateKey, ) -> Vec { - let total_activity = bucket_node_aggregates_activity.number_of_puts + - bucket_node_aggregates_activity.number_of_gets; - let total_levels = total_activity.ilog2() + 1; + let nonce_key = match aggregate_key { + AggregateKey::NodeAggregateKey(node_id) => node_id, + AggregateKey::BucketSubAggregateKey(.., node_id) => node_id, + }; - let int_list: Vec = (0..total_levels as u64).collect(); + let nonce = Self::_store_and_fetch_nonce(nonce_key); + let mut small_rng = SmallRng::seed_from_u64(nonce); - let nonce = Self::store_and_fetch_nonce(bucket_node_aggregates_activity.node_id); + let total_levels = number_of_leaves.ilog2() + 1; + let int_list: Vec = (0..total_levels as u64).collect(); - let mut small_rng = SmallRng::seed_from_u64(nonce); let ids: Vec = int_list .choose_multiple(&mut small_rng, number_of_identifiers) .cloned() @@ -1830,47 +1903,77 @@ pub mod pallet { ids } - pub(crate) fn fetch_sub_trees( + + /// Computes the consensus for a set of partial activities across multiple buckets within a + /// given cluster and era. + /// + /// This function collects activities from various buckets, groups them by their consensus + /// ID, and then determines if a consensus is reached for each group based on the minimum + /// number of nodes and a given threshold. If the consensus is reached, the activity is + /// included in the result. Otherwise, appropriate errors are returned. + /// + /// # Input Parameters + /// - `cluster_id: &ClusterId`: The ID of the cluster for which consensus is being computed. + /// - `era_id: DdcEra`: The era ID within the cluster. + /// - `buckets_aggregates_by_aggregator: &[(NodePubKey, Vec)]`: A list of tuples, where + /// each tuple contains a node's public key and a vector of activities reported for that + /// bucket. + /// - `redundancy_factor: u16`: The number of aggregators that should report total activity + /// for a node or a bucket + /// - `quorum: Percent`: The threshold percentage that determines if an activity is in + /// consensus. + /// + /// # Output + /// - `Result, Vec>`: + /// - `Ok(Vec)`: A vector of activities that have reached consensus. + /// - `Err(Vec)`: A vector of errors indicating why consensus was not reached + /// for some activities. + pub(crate) fn group_buckets_sub_aggregates_by_consistency( cluster_id: &ClusterId, era_id: DdcEra, - customer_activities: Vec, - quorum: u16, - ) -> Result< - (Vec, Vec), - Vec, - > { - let mut bucket_node_aggregates_activities: Vec = - Vec::new(); + buckets_aggregates_by_aggregator: Vec<(AggregatorInfo, Vec)>, + redundancy_factor: u16, + quorum: Percent, + ) -> ConsistentGroups { + let mut buckets_sub_aggregates: Vec = Vec::new(); log::info!( - "🏠⏳ Starting fetching bucket node aggregates for cluster_id: {:?} for era_id: {:?}", + "🏠⏳ Starting fetching bucket sub-aggregates for cluster_id: {:?} for era_id: {:?}", cluster_id, era_id ); - for customer_activity in customer_activities.clone() { - for bucket_sub_aggregate in customer_activity.sub_aggregates.clone() { - let bucket_node_aggregates_activity = BucketNodeAggregatesActivity { - bucket_id: customer_activity.bucket_id, - node_id: bucket_sub_aggregate.NodeID, - stored_bytes: bucket_sub_aggregate.stored_bytes, - transferred_bytes: bucket_sub_aggregate.transferred_bytes, - number_of_puts: bucket_sub_aggregate.number_of_puts, - number_of_gets: bucket_sub_aggregate.number_of_gets, - }; + for (aggregator_info, buckets_aggregates_resp) in + buckets_aggregates_by_aggregator.clone() + { + for bucket_aggregate_resp in buckets_aggregates_resp { + for bucket_sub_aggregate_resp in bucket_aggregate_resp.sub_aggregates.clone() { + let bucket_sub_aggregate = BucketSubAggregate { + bucket_id: bucket_aggregate_resp.bucket_id, + node_id: bucket_sub_aggregate_resp.NodeID, + stored_bytes: bucket_sub_aggregate_resp.stored_bytes, + transferred_bytes: bucket_sub_aggregate_resp.transferred_bytes, + number_of_puts: bucket_sub_aggregate_resp.number_of_puts, + number_of_gets: bucket_sub_aggregate_resp.number_of_gets, + aggregator: aggregator_info.clone(), + }; + + buckets_sub_aggregates.push(bucket_sub_aggregate); + } - bucket_node_aggregates_activities.push(bucket_node_aggregates_activity); + log::info!("🏠🚀 Fetched Bucket sub-aggregates for cluster_id: {:?} for era_id: {:?} for bucket_id {:?}::: Bucket Sub-Aggregates are {:?}", cluster_id, era_id, bucket_aggregate_resp.bucket_id, bucket_aggregate_resp.sub_aggregates); } - log::info!("🏠🚀 Fetched Bucket node-aggregates for cluster_id: {:?} for era_id: {:?} for bucket_id {:?}::: Bucket Sub-Aggregates are {:?}", cluster_id, era_id, customer_activity.bucket_id, customer_activity.sub_aggregates); } - Self::get_consensus_for_bucket_node_aggregates( - cluster_id, - era_id, - bucket_node_aggregates_activities, - quorum, - Percent::from_percent(T::MAJORITY), - ) + let buckets_sub_aggregates_groups = + Self::group_by_consistency(buckets_sub_aggregates, redundancy_factor, quorum); + + log::info!("🏠🌕 Bucket Sub-Aggregates, which are in consensus for cluster_id: {:?} for era_id: {:?}::: {:?}", cluster_id, era_id, buckets_sub_aggregates_groups.in_consensus); + log::info!("🏠🌗 Bucket Sub-Aggregates, which are in quorum for cluster_id: {:?} for era_id: {:?}::: {:?}", cluster_id, era_id, buckets_sub_aggregates_groups.in_quorum); + log::info!("🏠🌘 Bucket Sub-Aggregates, which are neither in consensus nor in quorum for cluster_id: {:?} for era_id: {:?}::: {:?}", cluster_id, era_id, buckets_sub_aggregates_groups.in_others); + + buckets_sub_aggregates_groups } + #[allow(dead_code)] pub(crate) fn prepare_begin_billing_report( cluster_id: &ClusterId, @@ -1881,8 +1984,6 @@ pub mod pallet { pub(crate) fn prepare_begin_charging_customers( cluster_id: &ClusterId, - dac_nodes: &[(NodePubKey, StorageNodeParams)], - min_nodes: u16, batch_size: usize, ) -> Result, Vec> { if let Some((era_id, start, end)) = @@ -1892,11 +1993,9 @@ pub mod pallet { PayoutState::Initialized { if let Some((_, _, customers_activity_batch_roots, _, _, _)) = - Self::fetch_validation_activities::< - BucketNodeAggregatesActivity, - NodeActivity, - >(cluster_id, era_id) - { + Self::fetch_validation_activities::( + cluster_id, era_id, + ) { Self::fetch_customer_activity( cluster_id, era_id, @@ -1905,20 +2004,12 @@ pub mod pallet { } else { let era_activity = EraActivity { id: era_id, start, end }; - let _ = Self::process_dac_data( - cluster_id, - Some(era_activity), - dac_nodes, - min_nodes, - batch_size, - )?; + let _ = Self::process_dac_era(cluster_id, Some(era_activity), batch_size)?; if let Some((_, _, customers_activity_batch_roots, _, _, _)) = - Self::fetch_validation_activities::< - BucketNodeAggregatesActivity, - NodeActivity, - >(cluster_id, era_id) - { + Self::fetch_validation_activities::( + cluster_id, era_id, + ) { Self::fetch_customer_activity( cluster_id, era_id, @@ -1956,8 +2047,6 @@ pub mod pallet { pub(crate) fn prepare_send_charging_customers_batch( cluster_id: &ClusterId, batch_size: usize, - dac_nodes: &[(NodePubKey, StorageNodeParams)], - min_nodes: u16, ) -> Result)>, Vec> { if let Some((era_id, start, end)) = Self::get_era_for_payout(cluster_id, EraValidationStatus::PayoutInProgress) @@ -1972,11 +2061,9 @@ pub mod pallet { _, _, _, - )) = Self::fetch_validation_activities::< - BucketNodeAggregatesActivity, - NodeActivity, - >(cluster_id, era_id) - { + )) = Self::fetch_validation_activities::( + cluster_id, era_id, + ) { Self::fetch_charging_activities( cluster_id, batch_size, @@ -1987,13 +2074,7 @@ pub mod pallet { } else { let era_activity = EraActivity { id: era_id, start, end }; - let _ = Self::process_dac_data( - cluster_id, - Some(era_activity), - dac_nodes, - min_nodes, - batch_size, - )?; + let _ = Self::process_dac_era(cluster_id, Some(era_activity), batch_size)?; if let Some(( bucket_nodes_activity_in_consensus, @@ -2002,11 +2083,9 @@ pub mod pallet { _, _, _, - )) = Self::fetch_validation_activities::< - BucketNodeAggregatesActivity, - NodeActivity, - >(cluster_id, era_id) - { + )) = Self::fetch_validation_activities::( + cluster_id, era_id, + ) { Self::fetch_charging_activities( cluster_id, batch_size, @@ -2030,7 +2109,7 @@ pub mod pallet { cluster_id: &ClusterId, batch_size: usize, era_id: DdcEra, - bucket_nodes_activity_in_consensus: Vec, + bucket_nodes_activity_in_consensus: Vec, customers_activity_batch_roots: Vec, ) -> Result)>, Vec> { let batch_index = T::PayoutVisitor::get_next_customer_batch_for_payment( @@ -2124,14 +2203,12 @@ pub mod pallet { pub(crate) fn prepare_begin_rewarding_providers( cluster_id: &ClusterId, - dac_nodes: &[(NodePubKey, StorageNodeParams)], - min_nodes: u16, batch_size: usize, ) -> Result, Vec> { if let Some((era_id, start, end)) = Self::get_era_for_payout(cluster_id, EraValidationStatus::PayoutInProgress) { - let nodes_total_usages = Self::get_nodes_total_usage(cluster_id, dac_nodes)?; + let nodes_total_usages = Self::get_nodes_total_usage(cluster_id)?; let nodes_total_usage: i64 = nodes_total_usages .iter() @@ -2148,11 +2225,9 @@ pub mod pallet { nodes_activity_in_consensus, _, nodes_activity_batch_roots, - )) = Self::fetch_validation_activities::< - BucketNodeAggregatesActivity, - NodeActivity, - >(cluster_id, era_id) - { + )) = Self::fetch_validation_activities::( + cluster_id, era_id, + ) { Self::fetch_reward_activities( cluster_id, era_id, @@ -2163,13 +2238,7 @@ pub mod pallet { } else { let era_activity = EraActivity { id: era_id, start, end }; - let _ = Self::process_dac_data( - cluster_id, - Some(era_activity), - dac_nodes, - min_nodes, - batch_size, - )?; + let _ = Self::process_dac_era(cluster_id, Some(era_activity), batch_size)?; if let Some(( _, @@ -2178,11 +2247,9 @@ pub mod pallet { nodes_activity_in_consensus, _, nodes_activity_batch_roots, - )) = Self::fetch_validation_activities::< - BucketNodeAggregatesActivity, - NodeActivity, - >(cluster_id, era_id) - { + )) = Self::fetch_validation_activities::( + cluster_id, era_id, + ) { Self::fetch_reward_activities( cluster_id, era_id, @@ -2205,7 +2272,7 @@ pub mod pallet { pub(crate) fn fetch_reward_activities( cluster_id: &ClusterId, era_id: DdcEra, - nodes_activity_in_consensus: Vec, + nodes_activity_in_consensus: Vec, nodes_activity_batch_roots: Vec, current_nodes_total_usage: i64, ) -> Result, Vec> { @@ -2239,8 +2306,6 @@ pub mod pallet { pub(crate) fn prepare_send_rewarding_providers_batch( cluster_id: &ClusterId, batch_size: usize, - dac_nodes: &[(NodePubKey, StorageNodeParams)], - min_nodes: u16, ) -> Result)>, Vec> { if let Some((era_id, start, end)) = Self::get_era_for_payout(cluster_id, EraValidationStatus::PayoutInProgress) @@ -2255,11 +2320,9 @@ pub mod pallet { nodes_activity_in_consensus, _, nodes_activity_batch_roots, - )) = Self::fetch_validation_activities::< - BucketNodeAggregatesActivity, - NodeActivity, - >(cluster_id, era_id) - { + )) = Self::fetch_validation_activities::( + cluster_id, era_id, + ) { Self::fetch_reward_provider_batch( cluster_id, batch_size, @@ -2270,13 +2333,7 @@ pub mod pallet { } else { let era_activity = EraActivity { id: era_id, start, end }; - let _ = Self::process_dac_data( - cluster_id, - Some(era_activity), - dac_nodes, - min_nodes, - batch_size, - )?; + let _ = Self::process_dac_era(cluster_id, Some(era_activity), batch_size)?; if let Some(( _, @@ -2285,11 +2342,9 @@ pub mod pallet { nodes_activity_in_consensus, _, nodes_activity_batch_roots, - )) = Self::fetch_validation_activities::< - BucketNodeAggregatesActivity, - NodeActivity, - >(cluster_id, era_id) - { + )) = Self::fetch_validation_activities::( + cluster_id, era_id, + ) { Self::fetch_reward_provider_batch( cluster_id, batch_size, @@ -2313,7 +2368,7 @@ pub mod pallet { cluster_id: &ClusterId, batch_size: usize, era_id: DdcEra, - nodes_activity_in_consensus: Vec, + nodes_activity_in_consensus: Vec, nodes_activity_batch_roots: Vec, ) -> Result)>, Vec> { let batch_index = T::PayoutVisitor::get_next_provider_batch_for_payment( @@ -2471,12 +2526,19 @@ pub mod pallet { pub(crate) fn get_nodes_total_usage( cluster_id: &ClusterId, - dac_nodes: &[(NodePubKey, StorageNodeParams)], ) -> Result>, Vec> { let mut results: Vec> = Vec::new(); let mut errors: Vec = Vec::new(); - for (node_pub_key, _params) in dac_nodes.iter() { + let nodes = match T::ClusterManager::get_nodes(cluster_id) { + Ok(nodes_pub_keys) => nodes_pub_keys, + Err(_) => { + errors.push(OCWError::FailedToFetchClusterNodes); + return Err(errors); + }, + }; + + for node_pub_key in nodes.iter() { match T::NodeVisitor::get_total_usage(node_pub_key) { Ok(usage) => results.push(usage), Err(_err) => { @@ -2548,7 +2610,7 @@ pub mod pallet { } } - pub(crate) fn store_and_fetch_nonce(node_id: String) -> u64 { + pub(crate) fn _store_and_fetch_nonce(node_id: String) -> u64 { let key = format!("offchain::activities::nonce::{:?}", node_id).into_bytes(); let encoded_nonce = sp_io::offchain::local_storage_get(StorageKind::PERSISTENT, &key) .unwrap_or_else(|| 0.encode()); @@ -2609,7 +2671,7 @@ pub mod pallet { /// /// # Output /// - `Vec`: A vector of Merkle roots, one for each batch of activities. - pub(crate) fn convert_to_batch_merkle_roots( + pub(crate) fn convert_to_batch_merkle_roots( cluster_id: &ClusterId, era_id: DdcEra, activities: Vec>, @@ -2637,7 +2699,7 @@ pub mod pallet { /// /// # Output /// - `Vec>`: A vector of vectors, where each inner vector is a batch of activities. - pub(crate) fn split_to_batches( + pub(crate) fn split_to_batches( activities: &[A], batch_size: usize, ) -> Vec> { @@ -2834,38 +2896,6 @@ pub mod pallet { Ok(all_node_eras.iter().cloned().min_by_key(|n| n.id)) } - /// Determines if a consensus is reached for a set of activities based on a specified - /// threshold. - /// - /// This function counts the occurrences of each activity in the provided list and checks if - /// any activity's count meets or exceeds the given threshold. If such an activity is found, - /// it is returned. - /// - /// # Input Parameters - /// - `activities: &[A]`: A slice of activities to be analyzed for consensus. - /// - `threshold: usize`: The minimum number of occurrences required for an activity to be - /// considered in consensus. - /// - /// # Output - /// - `Option`: - /// - `Some(A)`: An activity that has met or exceeded the threshold. - /// - `None`: No activity met the threshold. - pub(crate) fn reach_consensus( - activities: &[A], - threshold: usize, - ) -> Option { - let mut count_map: BTreeMap = BTreeMap::new(); - - for activity in activities { - *count_map.entry(activity.clone()).or_default() += 1; - } - - count_map - .into_iter() - .find(|&(_, count)| count >= threshold) - .map(|(activity, _)| activity) - } - /// Computes the consensus for a set of activities across multiple nodes within a given /// cluster and era. /// @@ -2877,11 +2907,12 @@ pub mod pallet { /// # Input Parameters /// - `cluster_id: &ClusterId`: The ID of the cluster for which consensus is being computed. /// - `era_id: DdcEra`: The era ID within the cluster. - /// - `activities: &[(NodePubKey, Vec)]`: A list of tuples, where each tuple contains a - /// node's public key and a vector of activities reported by that node. - /// - `min_nodes: u16`: The minimum number of nodes that must report an activity for it to - /// be considered for consensus. - /// - `threshold: Percent`: The threshold percentage that determines if an activity is in + /// - `nodes_aggregates_by_aggregator: &[(NodePubKey, Vec)]`: A list of tuples, where + /// each tuple contains a node's public key and a vector of activities reported by that + /// node. + /// - `redundancy_factor: u16`: The number of aggregators that should report total activity + /// for a node or a bucket + /// - `quorum: Percent`: The threshold percentage that determines if an activity is in /// consensus. /// /// # Output @@ -2889,114 +2920,82 @@ pub mod pallet { /// - `Ok(Vec)`: A vector of activities that have reached consensus. /// - `Err(Vec)`: A vector of errors indicating why consensus was not reached /// for some activities. - pub(crate) fn get_consensus_for_activities( + pub(crate) fn group_nodes_aggregates_by_consistency( cluster_id: &ClusterId, era_id: DdcEra, - activities: &[(NodePubKey, Vec)], - min_nodes: u16, - threshold: Percent, - ) -> Result, Vec> { - let mut customer_buckets: BTreeMap> = BTreeMap::new(); - - // Flatten and collect all customer activities - for (_node_id, activities) in activities.iter() { - for activity in activities.iter() { - customer_buckets - .entry(activity.get_consensus_id::()) - .or_default() - .push(activity.clone()); - } - } + nodes_aggregates_by_aggregator: Vec<(AggregatorInfo, Vec)>, + redundancy_factor: u16, + quorum: Percent, + ) -> ConsistentGroups { + let mut nodes_aggregates: Vec = Vec::new(); - let mut consensus_activities = Vec::new(); - let mut errors = Vec::new(); - let min_threshold = threshold * min_nodes; - - // Check if each customer/bucket appears in at least `min_nodes` nodes - for (id, activities) in customer_buckets { - if activities.len() < min_nodes.into() { - let errs: Vec = activities - .into_iter() - .map(|a| a.get_consensus_error(*cluster_id, era_id)) - .collect(); + log::info!( + "🏠⏳ Starting fetching node aggregates for cluster_id: {:?} for era_id: {:?}", + cluster_id, + era_id + ); - errors.extend(errs); - } else if let Some(activity) = - Self::reach_consensus(&activities, min_threshold.into()) - { - consensus_activities.push(activity); - } else { - errors.push(OCWError::ActivityNotInConsensus { - cluster_id: (*cluster_id), - era_id, - id, - }); + for (aggregator_info, nodes_aggregates_resp) in nodes_aggregates_by_aggregator.clone() { + for node_aggregate_resp in nodes_aggregates_resp.clone() { + let node_aggregate = NodeAggregate { + node_id: node_aggregate_resp.node_id, + stored_bytes: node_aggregate_resp.stored_bytes, + transferred_bytes: node_aggregate_resp.transferred_bytes, + number_of_puts: node_aggregate_resp.number_of_puts, + number_of_gets: node_aggregate_resp.number_of_gets, + aggregator: aggregator_info.clone(), + }; + nodes_aggregates.push(node_aggregate); } - } - if errors.is_empty() { - Ok(consensus_activities) - } else { - Err(errors) + log::info!("🏠🚀 Fetched Node-aggregates for cluster_id: {:?} for era_id: {:?} :::Node Aggregates are {:?}", cluster_id, era_id, nodes_aggregates); } + + let nodes_aggregates_groups = + Self::group_by_consistency(nodes_aggregates, redundancy_factor, quorum); + + log::info!("🏠🌕 Node Aggregates, which are in consensus for cluster_id: {:?} for era_id: {:?}::: {:?}", cluster_id, era_id, nodes_aggregates_groups.in_consensus); + log::info!("🏠🌗 Node Aggregates, which are in quorum for cluster_id: {:?} for era_id: {:?}::: {:?}", cluster_id, era_id, nodes_aggregates_groups.in_quorum); + log::info!("🏠🌘 Node Aggregates, which are neither in consensus nor in quorum for cluster_id: {:?} for era_id: {:?}::: {:?}", cluster_id, era_id, nodes_aggregates_groups.in_others); + + nodes_aggregates_groups } - pub(crate) fn get_consensus_for_bucket_node_aggregates( - cluster_id: &ClusterId, - era_id: DdcEra, - activities: Vec, - min_nodes: u16, - threshold: Percent, - ) -> Result< - (Vec, Vec), - Vec, - > { - let mut bucket_node_aggregates: BTreeMap< - ActivityHash, - Vec, - > = BTreeMap::new(); - - // Flatten and collect all customer activities - for activity in activities.iter() { - bucket_node_aggregates - .entry(activity.get_consensus_id::()) + pub(crate) fn group_by_consistency( + aggregates: Vec, + redundancy_factor: u16, + quorum: Percent, + ) -> ConsistentGroups + where + A: Aggregate + Clone, + { + let mut consistent_aggregates: BTreeMap> = BTreeMap::new(); + + for aggregate in aggregates.iter() { + consistent_aggregates + .entry(aggregate.hash::()) .or_default() - .push(activity.clone()); + .push(aggregate.clone()); } - let mut consensus_activities = Vec::new(); - let mut not_consensus_activities = Vec::new(); - let mut errors = Vec::new(); - let min_threshold = threshold * min_nodes; - - // Check if each customer/bucket appears in at least `min_nodes` nodes - for (id, activities) in bucket_node_aggregates { - if activities.len() < min_nodes.into() { - not_consensus_activities.extend(activities); - } else if let Some(activity) = - Self::reach_consensus(&activities, min_threshold.into()) - { - consensus_activities.push(activity); + let mut in_consensus = Vec::new(); + let mut in_quorum = Vec::new(); + let mut in_others = Vec::new(); + + let max_aggregates_count = redundancy_factor; + let quorum_threshold = quorum * max_aggregates_count; + + for (hash, group) in consistent_aggregates { + if group.len() == usize::from(max_aggregates_count) { + in_consensus.push(ConsistentGroup(hash, group)); + } else if group.len() >= quorum_threshold.into() { + in_quorum.push(ConsistentGroup(hash, group)); } else { - let node_ids = - activities.into_iter().map(|a| a.node_id).collect::>(); - errors.push(OCWError::BucketAggregateActivityNotInConsensus { - cluster_id: (*cluster_id), - era_id, - id, - node_ids, - }); + in_others.push(ConsistentGroup(hash, group)); } } - // todo! Reduce log size and put small message - log::info!("🏠👍 Sub-Trees, which are in consensus for cluster_id: {:?} for era_id: {:?}::: {:?}", cluster_id, era_id, consensus_activities); - log::info!("🏠👎 Sub-Trees, which are not in consensus for cluster_id: {:?} for era_id: {:?}::: {:?}", cluster_id, era_id, not_consensus_activities); - if errors.is_empty() { - Ok((consensus_activities, not_consensus_activities)) - } else { - Err(errors) - } + ConsistentGroups { in_consensus, in_quorum, in_others } } /// Fetch cluster to validate. @@ -3005,71 +3004,112 @@ pub mod pallet { Self::cluster_to_validate().ok_or(Error::ClusterToValidateRetrievalError) } - /// Fetch customer usage for an era. + /// Fetch Challenge node aggregate or bucket sub-aggregate. + pub(crate) fn _fetch_challenge_responses( + cluster_id: &ClusterId, + era_id: DdcEra, + aggregate_key: AggregateKey, + merkle_node_identifiers: Vec, + aggregator: AggregatorInfo, + ) -> Result { + let response = Self::_fetch_challenge_response( + era_id, + aggregate_key.clone(), + merkle_node_identifiers.clone(), + &aggregator.node_params, + ) + .map_err(|_| OCWError::ChallengeResponseRetrievalError { + cluster_id: *cluster_id, + era_id, + aggregate_key, + aggregator: aggregator.node_pub_key, + })?; + + Ok(response) + } + + /// Fetch challenge response. /// /// Parameters: - /// - `cluster_id`: cluster id of a cluster /// - `era_id`: era id - /// - `node_params`: DAC node parameters - pub(crate) fn fetch_challenge_responses( - cluster_id: &ClusterId, - node_id: String, + /// - `aggregate_key`: key of the aggregate to challenge + /// - `merkle_node_identifiers`: set of merkle node identifiers to challenge + /// - `node_params`: aggregator node parameters + pub(crate) fn _fetch_challenge_response( era_id: DdcEra, - bucket_id: BucketId, + aggregate_key: AggregateKey, merkle_node_identifiers: Vec, - dac_nodes: &[(NodePubKey, StorageNodeParams)], - ) -> Result, OCWError> { - let mut challenge_responses = Vec::new(); + node_params: &StorageNodeParams, + ) -> Result { + let scheme = "http"; + let host = str::from_utf8(&node_params.host).map_err(|_| http::Error::Unknown)?; - for (node_pub_key, node_params) in dac_nodes { - // todo! probably shouldn't stop when some DAC is not responding as we can still - // work with others - let response = Self::fetch_challenge_response( - node_id.clone(), - era_id, - bucket_id, - merkle_node_identifiers.clone(), - node_params, - ) - .map_err(|_| OCWError::ChallengeResponseRetrievalError { - cluster_id: *cluster_id, - era_id, - node_id: node_id.clone(), - bucket_id, - node_pub_key: node_pub_key.clone(), - })?; + let ids = merkle_node_identifiers + .iter() + .map(|x| format!("{}", x.clone())) + .collect::>() + .join(","); + + let url = match aggregate_key { + AggregateKey::NodeAggregateKey(node_id) => format!( + "{}://{}:{}/activity/nodes/{}/challenge?eraId={}&merkleTreeNodeId={}", + scheme, host, node_params.http_port, node_id, era_id, ids + ), + AggregateKey::BucketSubAggregateKey(bucket_id, node_id) => format!( + "{}://{}:{}/activity/buckets/{}/challenge?eraId={}&nodeId={}&merkleTreeNodeId={}", + scheme, host, node_params.http_port, bucket_id, era_id, node_id, ids + ), + }; - challenge_responses.push(response); + let request = http::Request::get(&url); + let timeout = sp_io::offchain::timestamp() + .add(sp_runtime::offchain::Duration::from_millis(RESPONSE_TIMEOUT)); + let pending = request.deadline(timeout).send().map_err(|_| http::Error::IoError)?; + + let response = + pending.try_wait(timeout).map_err(|_| http::Error::DeadlineReached)??; + if response.code != SUCCESS_CODE { + return Err(http::Error::Unknown); } - Ok(challenge_responses) + let body = response.body().collect::>(); + serde_json::from_slice(&body).map_err(|_| http::Error::Unknown) } - /// Fetch challenge response. + + /// Fetch traverse response. /// /// Parameters: - /// - `cluster_id`: cluster id of a cluster /// - `era_id`: era id - /// - `node_params`: DAC node parameters - pub(crate) fn fetch_challenge_response( - node_id: String, + /// - `aggregate_key`: key of the aggregate to challenge + /// - `merkle_node_identifiers`: set of merkle node identifiers to challenge + /// - `levels`: a number of levels to raverse + /// - `node_params`: aggregator node parameters + pub(crate) fn _fetch_traverse_response( era_id: DdcEra, - bucket_id: BucketId, + aggregate_key: AggregateKey, merkle_node_identifiers: Vec, + levels: u16, node_params: &StorageNodeParams, - ) -> Result { + ) -> Result, http::Error> { let scheme = "http"; let host = str::from_utf8(&node_params.host).map_err(|_| http::Error::Unknown)?; - let result = merkle_node_identifiers + let ids = merkle_node_identifiers .iter() .map(|x| format!("{}", x.clone())) .collect::>() .join(","); - let url = format!( - "{}://{}:{}/activity/buckets/{}/challenge?eraId={}&nodeId={}&merkleTreeNodeId={}", - scheme, host, node_params.http_port, bucket_id, era_id, node_id, result - ); + let url = match aggregate_key { + AggregateKey::NodeAggregateKey(node_id) => format!( + "{}://{}:{}/activity/nodes/{}/traverse?eraId={}&merkleTreeNodeId={}&levels={}", + scheme, host, node_params.http_port, node_id, era_id, ids, levels + ), + AggregateKey::BucketSubAggregateKey(bucket_id, node_id) => format!( + "{}://{}:{}/activity/buckets/{}/traverse?eraId={}&nodeId={}&merkleTreeNodeId={}&levels={}", + scheme, host, node_params.http_port, bucket_id, era_id, node_id, ids, levels + ), + }; let request = http::Request::get(&url); let timeout = sp_io::offchain::timestamp() @@ -3085,6 +3125,7 @@ pub mod pallet { let body = response.body().collect::>(); serde_json::from_slice(&body).map_err(|_| http::Error::Unknown) } + /// Fetch processed era. /// /// Parameters: @@ -3119,11 +3160,11 @@ pub mod pallet { /// - `cluster_id`: cluster id of a cluster /// - `era_id`: era id /// - `node_params`: DAC node parameters - pub(crate) fn fetch_customers_usage( + pub(crate) fn fetch_bucket_aggregates( _cluster_id: &ClusterId, era_id: DdcEra, node_params: &StorageNodeParams, - ) -> Result, http::Error> { + ) -> Result, http::Error> { let scheme = "http"; let host = str::from_utf8(&node_params.host).map_err(|_| http::Error::Unknown)?; let url = format!( @@ -3152,11 +3193,11 @@ pub mod pallet { /// - `cluster_id`: cluster id of a cluster /// - `era_id`: era id /// - `node_params`: DAC node parameters - pub(crate) fn fetch_node_usage( + pub(crate) fn fetch_node_aggregates( _cluster_id: &ClusterId, era_id: DdcEra, node_params: &StorageNodeParams, - ) -> Result, http::Error> { + ) -> Result, http::Error> { let scheme = "http"; let host = str::from_utf8(&node_params.host).map_err(|_| http::Error::Unknown)?; let url = format!( @@ -3224,34 +3265,38 @@ pub mod pallet { /// - `cluster_id`: cluster id of a cluster /// - `era_id`: era id /// - `node_params`: DAC node parameters - fn fetch_nodes_usage_for_era( + pub(crate) fn fetch_nodes_aggregates_for_era( cluster_id: &ClusterId, era_id: DdcEra, dac_nodes: &[(NodePubKey, StorageNodeParams)], - ) -> Result)>, OCWError> { - let mut node_usages = Vec::new(); + ) -> Result)>, OCWError> { + let mut nodes_aggregates = Vec::new(); for (node_pub_key, node_params) in dac_nodes { // todo! probably shouldn't stop when some DAC is not responding as we can still // work with others - let usage = - Self::fetch_node_usage(cluster_id, era_id, node_params).map_err(|_| { - OCWError::NodeUsageRetrievalError { - cluster_id: *cluster_id, - era_id, - node_pub_key: node_pub_key.clone(), - } + let aggregates = Self::fetch_node_aggregates(cluster_id, era_id, node_params) + .map_err(|_| OCWError::NodeUsageRetrievalError { + cluster_id: *cluster_id, + era_id, + node_pub_key: node_pub_key.clone(), })?; - for node_activity in usage.clone() { + + for aggregate in aggregates.clone() { let provider_id = Self::get_node_provider_id(node_pub_key).unwrap(); - Self::store_provider_id(node_activity.node_id, provider_id); // todo! this is not good - needs to be - // moved payout pallet + Self::store_provider_id(aggregate.node_id, provider_id); // todo! this is not good - needs to be + // moved payout pallet } - node_usages.push((node_pub_key.clone(), usage)); + let aggregator_info = AggregatorInfo { + node_pub_key: node_pub_key.clone(), + node_params: node_params.clone(), + }; + + nodes_aggregates.push((aggregator_info, aggregates)); } - Ok(node_usages) + Ok(nodes_aggregates) } /// Fetch customer usage for an era. @@ -3260,29 +3305,33 @@ pub mod pallet { /// - `cluster_id`: cluster id of a cluster /// - `era_id`: era id /// - `node_params`: DAC node parameters - pub(crate) fn fetch_customers_usage_for_era( + pub(crate) fn fetch_buckets_aggregates_for_era( cluster_id: &ClusterId, era_id: DdcEra, dac_nodes: &[(NodePubKey, StorageNodeParams)], - ) -> Result, OCWError> { - let mut customers_usages = Vec::new(); + ) -> Result)>, OCWError> { + let mut bucket_aggregates: Vec<(AggregatorInfo, Vec)> = + Vec::new(); for (node_pub_key, node_params) in dac_nodes { // todo! probably shouldn't stop when some DAC is not responding as we can still // work with others - let usage = - Self::fetch_customers_usage(cluster_id, era_id, node_params).map_err(|_| { - OCWError::CustomerUsageRetrievalError { - cluster_id: *cluster_id, - era_id, - node_pub_key: node_pub_key.clone(), - } + let aggregates = Self::fetch_bucket_aggregates(cluster_id, era_id, node_params) + .map_err(|_| OCWError::BucketAggregatesRetrievalError { + cluster_id: *cluster_id, + era_id, + node_pub_key: node_pub_key.clone(), })?; - customers_usages.extend(usage); + let aggregator_info = AggregatorInfo { + node_pub_key: node_pub_key.clone(), + node_params: node_params.clone(), + }; + + bucket_aggregates.push((aggregator_info, aggregates)); } - Ok(customers_usages) + Ok(bucket_aggregates) } /// Fetch processed era for across all nodes. @@ -3443,30 +3492,6 @@ pub mod pallet { for error in errors { match error { - OCWError::NotEnoughNodesForConsensus { cluster_id, era_id, node_id } => { - Self::deposit_event(Event::NotEnoughNodesForConsensus { - cluster_id, - era_id, - node_id, - validator: caller.clone(), - }); - }, - OCWError::NotEnoughBucketsForConsensus { cluster_id, era_id, bucket_id } => { - Self::deposit_event(Event::NotEnoughBucketsForConsensus { - cluster_id, - era_id, - bucket_id, - validator: caller.clone(), - }); - }, - OCWError::ActivityNotInConsensus { cluster_id, era_id, id } => { - Self::deposit_event(Event::ActivityNotInConsensus { - cluster_id, - era_id, - id, - validator: caller.clone(), - }); - }, OCWError::NodeUsageRetrievalError { cluster_id, era_id, node_pub_key } => { Self::deposit_event(Event::NodeUsageRetrievalError { cluster_id, @@ -3475,8 +3500,12 @@ pub mod pallet { validator: caller.clone(), }); }, - OCWError::CustomerUsageRetrievalError { cluster_id, era_id, node_pub_key } => { - Self::deposit_event(Event::CustomerUsageRetrievalError { + OCWError::BucketAggregatesRetrievalError { + cluster_id, + era_id, + node_pub_key, + } => { + Self::deposit_event(Event::BucketAggregatesRetrievalError { cluster_id, era_id, node_pub_key, @@ -3631,13 +3660,13 @@ pub mod pallet { validator: caller.clone(), }); }, - OCWError::BucketAggregatesRetrievalError { + OCWError::BucketAggregateRetrievalError { cluster_id, era_id, bucket_id, node_pub_key, } => { - Self::deposit_event(Event::BucketAggregatesRetrievalError { + Self::deposit_event(Event::BucketAggregateRetrievalError { cluster_id, era_id, bucket_id, @@ -3645,58 +3674,47 @@ pub mod pallet { validator: caller.clone(), }); }, - OCWError::NotEnoughNodeAggregatesForConsensus { + OCWError::ChallengeResponseRetrievalError { cluster_id, era_id, - bucket_id, - node_id, + aggregate_key, + aggregator, } => { - Self::deposit_event(Event::NotEnoughNodeAggregatesForConsensus { + Self::deposit_event(Event::ChallengeResponseRetrievalError { cluster_id, era_id, - bucket_id, - node_id, + aggregate_key, + aggregator, validator: caller.clone(), }); }, - OCWError::BucketAggregateActivityNotInConsensus { + OCWError::TraverseResponseRetrievalError { cluster_id, era_id, - id, - node_ids, + aggregate_key, + aggregator, } => { - Self::deposit_event(Event::BucketAggregateActivityNotInConsensus { + Self::deposit_event(Event::TraverseResponseRetrievalError { cluster_id, era_id, - id, - node_ids, + aggregate_key, + aggregator, validator: caller.clone(), }); }, - OCWError::ChallengeResponseRetrievalError { - cluster_id, - era_id, - node_id, - bucket_id, - node_pub_key, - } => { - Self::deposit_event(Event::ChallengeResponseRetrievalError { - cluster_id, - era_id, - node_id, - bucket_id, - node_pub_key, + OCWError::FailedToFetchClusterNodes => { + Self::deposit_event(Event::FailedToFetchClusterNodes { validator: caller.clone(), }); }, - OCWError::NotEnoughRecordsForConsensus { cluster_id, era_id, record_id } => { - Self::deposit_event(Event::NotEnoughRecordsForConsensus { - cluster_id, - era_id, - record_id, + OCWError::FailedToFetchDacNodes => { + Self::deposit_event(Event::FailedToFetchDacNodes { validator: caller.clone(), }); }, + OCWError::EmptyConsistentGroup => { + Self::deposit_event(Event::EmptyConsistentGroup); + }, } } diff --git a/pallets/ddc-verification/src/mock.rs b/pallets/ddc-verification/src/mock.rs index 6b9fc3c69..b5d8edfab 100644 --- a/pallets/ddc-verification/src/mock.rs +++ b/pallets/ddc-verification/src/mock.rs @@ -22,7 +22,7 @@ use sp_runtime::{ curve::PiecewiseLinear, testing::{TestXt, UintAuthorityId}, traits::{BlakeTwo256, Extrinsic as ExtrinsicT, IdentifyAccount, IdentityLookup, Verify, Zero}, - BuildStorage, MultiSignature, Perbill, + BuildStorage, MultiSignature, Perbill, Percent, }; use sp_staking::{EraIndex, SessionIndex}; @@ -211,6 +211,7 @@ impl pallet_timestamp::Config for Test { } parameter_types! { pub const VerificationPalletId: PalletId = PalletId(*b"verifypa"); + pub const MajorityOfAggregators: Percent = Percent::from_percent(67); } impl crate::Config for Test { @@ -226,7 +227,8 @@ impl crate::Config for Test { type ActivityHasher = sp_runtime::traits::BlakeTwo256; const MAJORITY: u8 = 67; const BLOCK_TO_START: u16 = 100; - const MIN_DAC_NODES_FOR_CONSENSUS: u16 = 3; + const DAC_REDUNDANCY_FACTOR: u16 = 3; + type AggregatorsQuorum = MajorityOfAggregators; const MAX_PAYOUT_BATCH_SIZE: u16 = MAX_PAYOUT_BATCH_SIZE; const MAX_PAYOUT_BATCH_COUNT: u16 = MAX_PAYOUT_BATCH_COUNT; type ActivityHash = H256; diff --git a/pallets/ddc-verification/src/tests.rs b/pallets/ddc-verification/src/tests.rs index 65e787516..0396a6dee 100644 --- a/pallets/ddc-verification/src/tests.rs +++ b/pallets/ddc-verification/src/tests.rs @@ -1,5 +1,6 @@ use ddc_primitives::{ - ClusterId, MergeActivityHash, StorageNodeMode, StorageNodeParams, StorageNodePubKey, KEY_TYPE, + AggregatorInfo, ClusterId, MergeActivityHash, StorageNodeMode, StorageNodeParams, + StorageNodePubKey, KEY_TYPE, }; use frame_support::{assert_noop, assert_ok}; use sp_core::{ @@ -13,7 +14,7 @@ use sp_io::TestExternalities; use sp_keystore::{testing::MemoryKeystore, Keystore, KeystoreExt}; use sp_runtime::AccountId32; -use crate::{mock::*, Error, NodeActivity, OCWError, *}; +use crate::{mock::*, Error, NodeAggregateResponse, *}; #[allow(dead_code)] fn register_validators(validators: Vec) { @@ -40,47 +41,65 @@ fn get_validators() -> Vec { vec![validator1, validator2, validator3, validator4, validator5] } -fn get_node_activities() -> Vec { - let node1 = NodeActivity { +fn get_node_activities() -> Vec { + let aggregator = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "178.251.228.236".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example.com".to_vec(), + }, + }; + + let node1 = NodeAggregate { node_id: "0".to_string(), stored_bytes: -100, transferred_bytes: 50, number_of_puts: 10, number_of_gets: 20, + aggregator: aggregator.clone(), }; - let node2 = NodeActivity { + let node2 = NodeAggregate { node_id: "1".to_string(), stored_bytes: -101, transferred_bytes: 51, number_of_puts: 11, number_of_gets: 21, + aggregator: aggregator.clone(), }; - let node3 = NodeActivity { + let node3 = NodeAggregate { node_id: "2".to_string(), stored_bytes: 102, transferred_bytes: 52, number_of_puts: 12, number_of_gets: 22, + aggregator: aggregator.clone(), }; - let node4 = NodeActivity { + let node4 = NodeAggregate { node_id: "3".to_string(), stored_bytes: 103, transferred_bytes: 53, number_of_puts: 13, number_of_gets: 23, + aggregator: aggregator.clone(), }; - let node5 = NodeActivity { + let node5 = NodeAggregate { node_id: "4".to_string(), stored_bytes: 104, transferred_bytes: 54, number_of_puts: 14, number_of_gets: 24, + aggregator: aggregator.clone(), }; vec![node1, node2, node3, node4, node5] } #[test] -fn fetch_node_usage_works() { +fn fetch_node_aggregates_works() { let mut ext = TestExternalities::default(); let (offchain, offchain_state) = TestOffchainExt::new(); let (pool, _) = TestTransactionPoolExt::new(); @@ -96,15 +115,15 @@ fn fetch_node_usage_works() { let port = 80; let era_id = 1; - // Create a sample NodeActivity instance - let node_activity1 = NodeActivity { + // Create a sample NodeAggregateResponse instance + let node_activity1 = NodeAggregateResponse { node_id: "1".to_string(), stored_bytes: 100, transferred_bytes: 50, number_of_puts: 10, number_of_gets: 20, }; - let node_activity2 = NodeActivity { + let node_activity2 = NodeAggregateResponse { node_id: "2".to_string(), stored_bytes: 110, transferred_bytes: 510, @@ -137,7 +156,7 @@ fn fetch_node_usage_works() { domain: b"example2.com".to_vec(), }; - let result = Pallet::::fetch_node_usage(&cluster_id, era_id, &node_params); + let result = Pallet::::fetch_node_aggregates(&cluster_id, era_id, &node_params); assert!(result.is_ok()); let activities = result.unwrap(); assert_eq!(activities[0].number_of_gets, node_activity1.number_of_gets); @@ -153,7 +172,7 @@ fn fetch_node_usage_works() { } #[test] -fn fetch_customers_usage_works() { +fn fetch_bucket_aggregates_works() { let mut ext = TestExternalities::default(); let (offchain, offchain_state) = TestOffchainExt::new(); let (pool, _) = TestTransactionPoolExt::new(); @@ -169,10 +188,14 @@ fn fetch_customers_usage_works() { let port = 80; let era_id = 1; - // Create a sample NodeActivity instance - let customer_activity1 = CustomerActivity { + // Create a sample NodeAggregateResponse instance + let bucket_aggregate1 = BucketAggregateResponse { + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, bucket_id: 111, - sub_aggregates: vec![BucketSubAggregate { + sub_aggregates: vec![BucketSubAggregateResponse { NodeID: "1".to_string(), stored_bytes: 100, transferred_bytes: 50, @@ -180,9 +203,13 @@ fn fetch_customers_usage_works() { number_of_gets: 20, }], }; - let customer_activity2 = CustomerActivity { + let bucket_aggregate2 = BucketAggregateResponse { + stored_bytes: 1000, + transferred_bytes: 500, + number_of_puts: 100, + number_of_gets: 200, bucket_id: 222, - sub_aggregates: vec![BucketSubAggregate { + sub_aggregates: vec![BucketSubAggregateResponse { NodeID: "1".to_string(), stored_bytes: 1000, transferred_bytes: 500, @@ -191,7 +218,7 @@ fn fetch_customers_usage_works() { }], }; let customers_activity_json = - serde_json::to_string(&vec![customer_activity1.clone(), customer_activity2.clone()]) + serde_json::to_string(&vec![bucket_aggregate1.clone(), bucket_aggregate2.clone()]) .unwrap(); // Mock HTTP request and response @@ -217,930 +244,1640 @@ fn fetch_customers_usage_works() { domain: b"example2.com".to_vec(), }; - let result = Pallet::::fetch_customers_usage(&cluster_id, era_id, &node_params); + let result = Pallet::::fetch_bucket_aggregates(&cluster_id, era_id, &node_params); assert!(result.is_ok()); let activities = result.unwrap(); assert_eq!( activities[0].sub_aggregates[0].number_of_gets, - customer_activity1.sub_aggregates[0].number_of_gets + bucket_aggregate1.sub_aggregates[0].number_of_gets ); assert_eq!( activities[0].sub_aggregates[0].number_of_puts, - customer_activity1.sub_aggregates[0].number_of_puts + bucket_aggregate1.sub_aggregates[0].number_of_puts ); assert_eq!( activities[0].sub_aggregates[0].transferred_bytes, - customer_activity1.sub_aggregates[0].transferred_bytes + bucket_aggregate1.sub_aggregates[0].transferred_bytes ); assert_eq!( activities[0].sub_aggregates[0].stored_bytes, - customer_activity1.sub_aggregates[0].stored_bytes + bucket_aggregate1.sub_aggregates[0].stored_bytes ); assert_eq!( activities[1].sub_aggregates[0].number_of_gets, - customer_activity2.sub_aggregates[0].number_of_gets + bucket_aggregate2.sub_aggregates[0].number_of_gets ); assert_eq!( activities[1].sub_aggregates[0].number_of_puts, - customer_activity2.sub_aggregates[0].number_of_puts + bucket_aggregate2.sub_aggregates[0].number_of_puts ); assert_eq!( activities[1].sub_aggregates[0].transferred_bytes, - customer_activity2.sub_aggregates[0].transferred_bytes + bucket_aggregate2.sub_aggregates[0].transferred_bytes ); assert_eq!( activities[1].sub_aggregates[0].stored_bytes, - customer_activity2.sub_aggregates[0].stored_bytes + bucket_aggregate2.sub_aggregates[0].stored_bytes ); }); } #[test] -fn test_reach_consensus_empty() { - let activities: Vec = Vec::new(); - let result = DdcVerification::reach_consensus(&activities, 3); - assert!(result.is_none()); -} +fn buckets_sub_aggregates_in_consensus_merged() { + let redundancy_factor = 3; + let quorum = Percent::from_percent(67); + let cluster_id = ClusterId::from([1; 20]); + let era_id = 476817; -#[test] -fn test_reach_consensus_success() { - let activities = vec![ - CustomerActivity { + let aggregator1 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "178.251.228.236".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example1.com".to_vec(), + }, + }; + + let aggregator2 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([2; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "95.217.8.119".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example2.com".to_vec(), + }, + }; + + let aggregator3 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([3; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "178.251.228.42".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example3.com".to_vec(), + }, + }; + + let resp1 = ( + aggregator1, + vec![BucketAggregateResponse { + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { + sub_aggregates: vec![BucketSubAggregateResponse { NodeID: "1".to_string(), stored_bytes: 100, transferred_bytes: 50, number_of_puts: 10, number_of_gets: 20, }], - }, - CustomerActivity { + }], + ); + + let resp2 = ( + aggregator2, + vec![BucketAggregateResponse { + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { + sub_aggregates: vec![BucketSubAggregateResponse { NodeID: "1".to_string(), stored_bytes: 100, transferred_bytes: 50, number_of_puts: 10, number_of_gets: 20, }], - }, - CustomerActivity { + }], + ); + + let resp3 = ( + aggregator3, + vec![BucketAggregateResponse { + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { + sub_aggregates: vec![BucketSubAggregateResponse { NodeID: "1".to_string(), stored_bytes: 100, transferred_bytes: 50, number_of_puts: 10, number_of_gets: 20, }], - }, - ]; - let result = DdcVerification::reach_consensus(&activities, 3); - assert!(result.is_some()); - assert_eq!(result.unwrap().sub_aggregates[0].stored_bytes, 100); + }], + ); + + let groups = DdcVerification::group_buckets_sub_aggregates_by_consistency( + &cluster_id, + era_id, + vec![resp1, resp2, resp3], + redundancy_factor, + quorum, + ); + assert_eq!(groups.in_consensus.len(), 1); + assert_eq!(groups.in_quorum.len(), 0); + assert_eq!(groups.in_others.len(), 0); + + let result = DdcVerification::get_total_usage(&cluster_id, era_id, groups); + + assert!(result.is_ok()); + let usages = result.unwrap(); + let usage = usages.first().unwrap(); + assert_eq!(usage.stored_bytes, 100); + assert_eq!(usage.transferred_bytes, 50); + assert_eq!(usage.number_of_puts, 10); + assert_eq!(usage.number_of_gets, 20); } #[test] -fn test_reach_consensus_failure() { - let activities = vec![ - CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }, - CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 200, - transferred_bytes: 100, - number_of_puts: 20, - number_of_gets: 40, - }], +fn buckets_sub_aggregates_in_quorum_merged() { + let redundancy_factor = 3; + let quorum = Percent::from_percent(67); + let cluster_id = ClusterId::from([1; 20]); + let era_id = 476817; + + let aggregator1 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "178.251.228.236".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example1.com".to_vec(), }, - CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 300, - transferred_bytes: 150, - number_of_puts: 30, - number_of_gets: 60, - }], + }; + + let aggregator2 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([2; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "95.217.8.119".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example2.com".to_vec(), }, - ]; - let result = DdcVerification::reach_consensus(&activities, 3); - assert!(result.is_none()); -} + }; -#[test] -fn test_reach_consensus_threshold() { - let activities = vec![ - CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], + let aggregator3 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([3; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "178.251.228.42".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example3.com".to_vec(), }, - CustomerActivity { + }; + + let resp1 = ( + aggregator1, + vec![BucketAggregateResponse { + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { + sub_aggregates: vec![BucketSubAggregateResponse { NodeID: "1".to_string(), stored_bytes: 100, transferred_bytes: 50, number_of_puts: 10, number_of_gets: 20, }], - }, - CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 200, - transferred_bytes: 100, - number_of_puts: 20, - number_of_gets: 40, - }], - }, - ]; - - let mut result = DdcVerification::reach_consensus(&activities, 2); - assert!(result.is_some()); - assert_eq!(result.unwrap().sub_aggregates[0].stored_bytes, 100); - result = DdcVerification::reach_consensus(&activities, 3); - assert!(result.is_none()); -} + }], + ); -#[test] -fn test_reach_consensus_exact_threshold() { - let activities = vec![ - CustomerActivity { + let resp2 = ( + aggregator2, + vec![BucketAggregateResponse { + stored_bytes: 200, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { + sub_aggregates: vec![BucketSubAggregateResponse { NodeID: "1".to_string(), - stored_bytes: 100, + stored_bytes: 200, transferred_bytes: 50, number_of_puts: 10, number_of_gets: 20, }], - }, - CustomerActivity { + }], + ); + + let resp3 = ( + aggregator3, + vec![BucketAggregateResponse { + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { + sub_aggregates: vec![BucketSubAggregateResponse { NodeID: "1".to_string(), stored_bytes: 100, transferred_bytes: 50, number_of_puts: 10, number_of_gets: 20, }], - }, - ]; - let result = DdcVerification::reach_consensus(&activities, 3); - assert!(result.is_none()); -} - -#[test] -fn test_get_consensus_customers_activity_success() { - let cluster_id = ClusterId::from([1; 20]); - let era_id = 1; - let min_nodes = 3; - let threshold = Percent::from_percent(67); - - let node_pubkey_0 = NodePubKey::StoragePubKey(AccountId32::new([0; 32])); - let node_pubkey_1 = NodePubKey::StoragePubKey(AccountId32::new([1; 32])); - let node_pubkey_2 = NodePubKey::StoragePubKey(AccountId32::new([2; 32])); - - let customers_activity = vec![ - ( - node_pubkey_0, - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ( - node_pubkey_1, - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ( - node_pubkey_2, - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ]; + }], + ); - let result = DdcVerification::get_consensus_for_activities( + let groups = DdcVerification::group_buckets_sub_aggregates_by_consistency( &cluster_id, era_id, - &customers_activity, - min_nodes, - threshold, + vec![resp1, resp2, resp3], + redundancy_factor, + quorum, ); - assert!(result.is_ok()); - let consensus_activities = result.unwrap(); - assert_eq!(consensus_activities.len(), 1); - assert_eq!(consensus_activities[0].sub_aggregates[0].stored_bytes, 100); -} + assert_eq!(groups.in_consensus.len(), 0); + assert_eq!(groups.in_quorum.len(), 1); + assert_eq!(groups.in_others.len(), 1); -#[test] -fn test_get_consensus_customers_activity_success2() { - let cluster_id = ClusterId::from([1; 20]); - let era_id = 1; - let min_nodes = 3; - let threshold = Percent::from_percent(67); - - let node_pubkey_0 = NodePubKey::StoragePubKey(AccountId32::new([0; 32])); - let node_pubkey_1 = NodePubKey::StoragePubKey(AccountId32::new([1; 32])); - let node_pubkey_2 = NodePubKey::StoragePubKey(AccountId32::new([2; 32])); - - let customers_activity = vec![ - ( - node_pubkey_0.clone(), - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ( - node_pubkey_1.clone(), - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ( - node_pubkey_2.clone(), - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 110, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ( - node_pubkey_0, - vec![CustomerActivity { - bucket_id: 2, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 110, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ( - node_pubkey_1, - vec![CustomerActivity { - bucket_id: 2, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 110, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ( - node_pubkey_2, - vec![CustomerActivity { - bucket_id: 2, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 110, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ]; + let result = DdcVerification::get_total_usage(&cluster_id, era_id, groups); - let result = DdcVerification::get_consensus_for_activities( - &cluster_id, - era_id, - &customers_activity, - min_nodes, - threshold, - ); assert!(result.is_ok()); - let consensus_activities = result.unwrap(); - assert_eq!(consensus_activities.len(), 2); - assert_eq!(consensus_activities[1].sub_aggregates[0].stored_bytes, 110); - assert_eq!(consensus_activities[1].bucket_id, 2); - assert_eq!(consensus_activities[0].sub_aggregates[0].stored_bytes, 100); - assert_eq!(consensus_activities[0].bucket_id, 1); + let usages = result.unwrap(); + let usage = usages.first().unwrap(); + assert_eq!(usage.stored_bytes, 100); + assert_eq!(usage.transferred_bytes, 50); + assert_eq!(usage.number_of_puts, 10); + assert_eq!(usage.number_of_gets, 20); } #[test] -fn test_get_consensus_nodes_activity_success() { +fn buckets_sub_aggregates_in_others_merged() { + let redundancy_factor = 3; + let quorum = Percent::from_percent(100); let cluster_id = ClusterId::from([1; 20]); - let era_id = 1; - let min_nodes = 3; - let threshold = Percent::from_percent(67); + let era_id = 476817; + + let aggregator1 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "178.251.228.236".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example1.com".to_vec(), + }, + }; - let node_pubkey_0 = NodePubKey::StoragePubKey(AccountId32::new([0; 32])); - let node_pubkey_1 = NodePubKey::StoragePubKey(AccountId32::new([1; 32])); - let node_pubkey_2 = NodePubKey::StoragePubKey(AccountId32::new([2; 32])); + let aggregator2 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([2; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "95.217.8.119".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example2.com".to_vec(), + }, + }; - let customers_activity = vec![ - ( - node_pubkey_0, - vec![NodeActivity { - node_id: "0".to_string(), + let aggregator3 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([3; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "178.251.228.42".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example3.com".to_vec(), + }, + }; + + let resp1 = ( + aggregator1, + vec![BucketAggregateResponse { + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + bucket_id: 1, + sub_aggregates: vec![BucketSubAggregateResponse { + NodeID: "1".to_string(), stored_bytes: 100, transferred_bytes: 50, number_of_puts: 10, number_of_gets: 20, }], - ), - ( - node_pubkey_1, - vec![NodeActivity { - node_id: "0".to_string(), - stored_bytes: 100, + }], + ); + + let resp2 = ( + aggregator2, + vec![BucketAggregateResponse { + stored_bytes: 200, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + bucket_id: 1, + sub_aggregates: vec![BucketSubAggregateResponse { + NodeID: "1".to_string(), + stored_bytes: 200, transferred_bytes: 50, number_of_puts: 10, number_of_gets: 20, }], - ), - ( - node_pubkey_2, - vec![NodeActivity { - node_id: "0".to_string(), + }], + ); + + let resp3 = ( + aggregator3, + vec![BucketAggregateResponse { + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + bucket_id: 1, + sub_aggregates: vec![BucketSubAggregateResponse { + NodeID: "1".to_string(), stored_bytes: 100, transferred_bytes: 50, number_of_puts: 10, number_of_gets: 20, }], - ), - ]; + }], + ); - let result = DdcVerification::get_consensus_for_activities( + let groups = DdcVerification::group_buckets_sub_aggregates_by_consistency( &cluster_id, era_id, - &customers_activity, - min_nodes, - threshold, + vec![resp1, resp2, resp3], + redundancy_factor, + quorum, ); - assert!(result.is_ok()); - let consensus_activities = result.unwrap(); - assert_eq!(consensus_activities.len(), 1); - assert_eq!(consensus_activities[0].stored_bytes, 100); -} -#[test] -fn test_get_consensus_customers_activity_empty() { - let cluster_id = ClusterId::from([1; 20]); - let era_id = 1; - let min_nodes = 3; - let threshold = Percent::from_percent(67); - let node_pubkey_0 = NodePubKey::StoragePubKey(AccountId32::new([0; 32])); - let node_pubkey_1 = NodePubKey::StoragePubKey(AccountId32::new([1; 32])); - let node_pubkey_2 = NodePubKey::StoragePubKey(AccountId32::new([2; 32])); + assert_eq!(groups.in_consensus.len(), 0); + assert_eq!(groups.in_quorum.len(), 0); + assert_eq!(groups.in_others.len(), 2); - let customers_activity = vec![ - (node_pubkey_0.clone(), Vec::::new()), - (node_pubkey_1.clone(), Vec::::new()), - (node_pubkey_2.clone(), Vec::::new()), - ]; + let result = DdcVerification::get_total_usage(&cluster_id, era_id, groups); - let result = DdcVerification::get_consensus_for_activities( - &cluster_id, - era_id, - &customers_activity, - min_nodes, - threshold, - ); assert!(result.is_ok()); - let consensus_activities = result.unwrap(); - assert_eq!(consensus_activities.len(), 0); + let usages = result.unwrap(); + + let usage1 = usages.first().unwrap(); + assert_eq!(usage1.stored_bytes, 100); + assert_eq!(usage1.transferred_bytes, 50); + assert_eq!(usage1.number_of_puts, 10); + assert_eq!(usage1.number_of_gets, 20); + + let usage2 = usages.get(1).unwrap(); + assert_eq!(usage2.stored_bytes, 200); + assert_eq!(usage2.transferred_bytes, 50); + assert_eq!(usage2.number_of_puts, 10); + assert_eq!(usage2.number_of_gets, 20); } #[test] -fn test_get_consensus_customers_activity_not_enough_nodes() { - let cluster_id1 = ClusterId::from([1; 20]); - let era_id1 = 1; - let min_nodes = 3; - let threshold = Percent::from_percent(67); - let node_pubkey_0 = NodePubKey::StoragePubKey(AccountId32::new([0; 32])); - let node_pubkey_1 = NodePubKey::StoragePubKey(AccountId32::new([1; 32])); - - let customers_activity = vec![ - ( - node_pubkey_0, - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ( - node_pubkey_1, - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ]; +fn nodes_aggregates_in_consensus_merged() { + let redundancy_factor = 3; + let quorum = Percent::from_percent(67); + let cluster_id = ClusterId::from([1; 20]); + let era_id = 476817; - let result = DdcVerification::get_consensus_for_activities( - &cluster_id1, - era_id1, - &customers_activity, - min_nodes, - threshold, - ); - assert!(result.is_err()); - let errors = result.err().unwrap(); - assert_eq!(errors.len(), 2); - match &errors[0] { - OCWError::NotEnoughBucketsForConsensus { cluster_id, era_id, bucket_id } => { - assert_eq!(*bucket_id, 1); - assert_eq!(*cluster_id, cluster_id1); - assert_eq!(*era_id, era_id1); + let aggregator1 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "178.251.228.236".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example1.com".to_vec(), }, - _ => panic!("Expected NotEnoughNodes error"), - } -} - -#[test] -fn test_get_consensus_nodes_activity_not_enough_nodes() { - let cluster_id1 = ClusterId::from([1; 20]); - let era_id1 = 1; - let min_nodes = 3; - let threshold = Percent::from_percent(67); - let node_pubkey_0 = NodePubKey::StoragePubKey(AccountId32::new([0; 32])); - let node_pubkey_1 = NodePubKey::StoragePubKey(AccountId32::new([1; 32])); - - let nodes_activity = vec![ - ( - node_pubkey_0, - vec![NodeActivity { - node_id: "0".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - ), - ( - node_pubkey_1, - vec![NodeActivity { - node_id: "0".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - ), - ]; + }; - let result = DdcVerification::get_consensus_for_activities( - &cluster_id1, - era_id1, - &nodes_activity, - min_nodes, - threshold, + let aggregator2 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([2; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "95.217.8.119".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example2.com".to_vec(), + }, + }; + + let aggregator3 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([3; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "178.251.228.42".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example3.com".to_vec(), + }, + }; + + let resp1 = ( + aggregator1, + vec![NodeAggregateResponse { + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + }], ); - assert!(result.is_err()); - let errors = result.err().unwrap(); - assert_eq!(errors.len(), 2); - match &errors[0] { - OCWError::NotEnoughNodesForConsensus { cluster_id, era_id, node_id } => { - assert_eq!(*node_id, "0".to_string()); - assert_eq!(*cluster_id, cluster_id1); - assert_eq!(*era_id, era_id1); + + let resp2 = ( + aggregator2, + vec![NodeAggregateResponse { + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + }], + ); + + let resp3 = ( + aggregator3, + vec![NodeAggregateResponse { + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + }], + ); + + let groups = DdcVerification::group_nodes_aggregates_by_consistency( + &cluster_id, + era_id, + vec![resp1, resp2, resp3], + redundancy_factor, + quorum, + ); + assert_eq!(groups.in_consensus.len(), 1); + assert_eq!(groups.in_quorum.len(), 0); + assert_eq!(groups.in_others.len(), 0); + + let result = DdcVerification::get_total_usage(&cluster_id, era_id, groups); + + assert!(result.is_ok()); + let usages = result.unwrap(); + let usage = usages.first().unwrap(); + assert_eq!(usage.stored_bytes, 100); + assert_eq!(usage.transferred_bytes, 50); + assert_eq!(usage.number_of_puts, 10); + assert_eq!(usage.number_of_gets, 20); +} + +#[test] +fn nodes_aggregates_in_quorum_merged() { + let redundancy_factor = 3; + let quorum = Percent::from_percent(67); + let cluster_id = ClusterId::from([1; 20]); + let era_id = 476817; + + let aggregator1 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "178.251.228.236".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example1.com".to_vec(), }, - _ => panic!("Expected NotEnoughNodes error"), - } + }; + + let aggregator2 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([2; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "95.217.8.119".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example2.com".to_vec(), + }, + }; + + let aggregator3 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([3; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "178.251.228.42".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example3.com".to_vec(), + }, + }; + + let resp1 = ( + aggregator1, + vec![NodeAggregateResponse { + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + }], + ); + + let resp2 = ( + aggregator2, + vec![NodeAggregateResponse { + node_id: "1".to_string(), + stored_bytes: 200, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + }], + ); + + let resp3 = ( + aggregator3, + vec![NodeAggregateResponse { + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + }], + ); + + let groups = DdcVerification::group_nodes_aggregates_by_consistency( + &cluster_id, + era_id, + vec![resp1, resp2, resp3], + redundancy_factor, + quorum, + ); + assert_eq!(groups.in_consensus.len(), 0); + assert_eq!(groups.in_quorum.len(), 1); + assert_eq!(groups.in_others.len(), 1); + + let result = DdcVerification::get_total_usage(&cluster_id, era_id, groups); + + assert!(result.is_ok()); + let usages = result.unwrap(); + let usage = usages.first().unwrap(); + assert_eq!(usage.stored_bytes, 100); + assert_eq!(usage.transferred_bytes, 50); + assert_eq!(usage.number_of_puts, 10); + assert_eq!(usage.number_of_gets, 20); +} + +#[test] +fn nodes_aggregates_in_others_merged() { + let redundancy_factor = 3; + let quorum = Percent::from_percent(100); + let cluster_id = ClusterId::from([1; 20]); + let era_id = 476817; + + let aggregator1 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "178.251.228.236".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example1.com".to_vec(), + }, + }; + + let aggregator2 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([2; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "95.217.8.119".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example2.com".to_vec(), + }, + }; + + let aggregator3 = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([3; 32])), + node_params: StorageNodeParams { + ssl: false, + host: "178.251.228.42".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example3.com".to_vec(), + }, + }; + + let resp1 = ( + aggregator1, + vec![NodeAggregateResponse { + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + }], + ); + + let resp2 = ( + aggregator2, + vec![NodeAggregateResponse { + node_id: "1".to_string(), + stored_bytes: 200, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + }], + ); + + let resp3 = ( + aggregator3, + vec![NodeAggregateResponse { + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + }], + ); + + let groups = DdcVerification::group_nodes_aggregates_by_consistency( + &cluster_id, + era_id, + vec![resp1, resp2, resp3], + redundancy_factor, + quorum, + ); + + assert_eq!(groups.in_consensus.len(), 0); + assert_eq!(groups.in_quorum.len(), 0); + assert_eq!(groups.in_others.len(), 2); + + let result = DdcVerification::get_total_usage(&cluster_id, era_id, groups); + + assert!(result.is_ok()); + let usages = result.unwrap(); + + let usage1 = usages.first().unwrap(); + assert_eq!(usage1.stored_bytes, 200); + assert_eq!(usage1.transferred_bytes, 50); + assert_eq!(usage1.number_of_puts, 10); + assert_eq!(usage1.number_of_gets, 20); + + let usage2 = usages.get(1).unwrap(); + assert_eq!(usage2.stored_bytes, 100); + assert_eq!(usage2.transferred_bytes, 50); + assert_eq!(usage2.number_of_puts, 10); + assert_eq!(usage2.number_of_gets, 20); +} + +#[test] +fn buckets_sub_aggregates_grouped_by_consistency() { + let redundancy_factor = 3; + let quorum = Percent::from_percent(67); + let host = "example1.com"; + let port = 80; + let node_params = StorageNodeParams { + ssl: false, + host: host.as_bytes().to_vec(), + http_port: port, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example2.com".to_vec(), + }; + let aggregator = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([0; 32])), + node_params: node_params.clone(), + }; + + let buckets_sub_aggregates = vec![ + BucketSubAggregate { + bucket_id: 1, + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + BucketSubAggregate { + bucket_id: 1, + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + BucketSubAggregate { + bucket_id: 1, + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + ]; + + let groups = + DdcVerification::group_by_consistency(buckets_sub_aggregates, redundancy_factor, quorum); + + assert_eq!(groups.in_consensus.len(), 1); + assert_eq!(groups.in_quorum.len(), 0); + assert_eq!(groups.in_others.len(), 0); + + let agg1 = groups.in_consensus[0].get(0).unwrap(); + let agg2 = groups.in_consensus[0].get(1).unwrap(); + let agg3 = groups.in_consensus[0].get(2).unwrap(); + + assert_eq!(agg1.stored_bytes, 100); + assert_eq!(agg1.transferred_bytes, 50); + assert_eq!(agg1.number_of_puts, 10); + assert_eq!(agg1.number_of_gets, 20); + + assert_eq!(agg2.stored_bytes, 100); + assert_eq!(agg2.transferred_bytes, 50); + assert_eq!(agg2.number_of_puts, 10); + assert_eq!(agg2.number_of_gets, 20); + + assert_eq!(agg3.stored_bytes, 100); + assert_eq!(agg3.transferred_bytes, 50); + assert_eq!(agg3.number_of_puts, 10); + assert_eq!(agg3.number_of_gets, 20); +} + +#[test] +fn buckets_sub_aggregates_grouped_by_consistency_2() { + let redundancy_factor = 3; + let quorum = Percent::from_percent(67); + + let host = "example1.com"; + let port = 80; + let node_params = StorageNodeParams { + ssl: false, + host: host.as_bytes().to_vec(), + http_port: port, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example2.com".to_vec(), + }; + let aggregator = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([0; 32])), + node_params: node_params.clone(), + }; + + let buckets_sub_aggregates = vec![ + BucketSubAggregate { + bucket_id: 1, + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + BucketSubAggregate { + bucket_id: 1, + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + BucketSubAggregate { + bucket_id: 1, + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + BucketSubAggregate { + bucket_id: 2, + node_id: "2".to_string(), + stored_bytes: 110, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + BucketSubAggregate { + bucket_id: 2, + node_id: "2".to_string(), + stored_bytes: 110, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + BucketSubAggregate { + bucket_id: 2, + node_id: "2".to_string(), + stored_bytes: 110, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + ]; + + let groups = + DdcVerification::group_by_consistency(buckets_sub_aggregates, redundancy_factor, quorum); + + assert_eq!(groups.in_consensus.len(), 2); + assert_eq!(groups.in_quorum.len(), 0); + assert_eq!(groups.in_others.len(), 0); + + let g1_agg1 = groups.in_consensus[0].get(0).unwrap(); + let g1_agg2 = groups.in_consensus[0].get(1).unwrap(); + let g1_agg3 = groups.in_consensus[0].get(2).unwrap(); + + assert_eq!(g1_agg1.bucket_id, 1); + assert_eq!(g1_agg1.stored_bytes, 100); + assert_eq!(g1_agg1.transferred_bytes, 50); + assert_eq!(g1_agg1.number_of_puts, 10); + assert_eq!(g1_agg1.number_of_gets, 20); + + assert_eq!(g1_agg2.bucket_id, 1); + assert_eq!(g1_agg2.stored_bytes, 100); + assert_eq!(g1_agg2.transferred_bytes, 50); + assert_eq!(g1_agg2.number_of_puts, 10); + assert_eq!(g1_agg2.number_of_gets, 20); + + assert_eq!(g1_agg3.bucket_id, 1); + assert_eq!(g1_agg3.stored_bytes, 100); + assert_eq!(g1_agg3.transferred_bytes, 50); + assert_eq!(g1_agg3.number_of_puts, 10); + assert_eq!(g1_agg3.number_of_gets, 20); + + let g2_agg1 = groups.in_consensus[1].get(0).unwrap(); + let g2_agg2 = groups.in_consensus[1].get(1).unwrap(); + let g2_agg3 = groups.in_consensus[1].get(2).unwrap(); + + assert_eq!(g2_agg1.bucket_id, 2); + assert_eq!(g2_agg1.stored_bytes, 110); + assert_eq!(g2_agg1.transferred_bytes, 50); + assert_eq!(g2_agg1.number_of_puts, 10); + assert_eq!(g2_agg1.number_of_gets, 20); + + assert_eq!(g2_agg2.bucket_id, 2); + assert_eq!(g2_agg2.stored_bytes, 110); + assert_eq!(g2_agg2.transferred_bytes, 50); + assert_eq!(g2_agg2.number_of_puts, 10); + assert_eq!(g2_agg2.number_of_gets, 20); + + assert_eq!(g2_agg3.bucket_id, 2); + assert_eq!(g2_agg3.stored_bytes, 110); + assert_eq!(g2_agg3.transferred_bytes, 50); + assert_eq!(g2_agg3.number_of_puts, 10); + assert_eq!(g2_agg3.number_of_gets, 20); +} + +#[test] +fn nodes_aggregates_grouped_by_consistency() { + let redundancy_factor = 3; + let quorum = Percent::from_percent(67); + let host = "example1.com"; + let port = 80; + let node_params = StorageNodeParams { + ssl: false, + host: host.as_bytes().to_vec(), + http_port: port, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example2.com".to_vec(), + }; + + let aggregator = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([0; 32])), + node_params: node_params.clone(), + }; + + let nodes_aggregates = vec![ + NodeAggregate { + node_id: "0".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + NodeAggregate { + node_id: "0".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + NodeAggregate { + node_id: "0".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + ]; + + let groups = DdcVerification::group_by_consistency(nodes_aggregates, redundancy_factor, quorum); + + assert_eq!(groups.in_consensus.len(), 1); + assert_eq!(groups.in_quorum.len(), 0); + assert_eq!(groups.in_others.len(), 0); + + let agg1 = groups.in_consensus[0].get(0).unwrap(); + let agg2 = groups.in_consensus[0].get(1).unwrap(); + let agg3 = groups.in_consensus[0].get(2).unwrap(); + + assert_eq!(agg1.stored_bytes, 100); + assert_eq!(agg1.transferred_bytes, 50); + assert_eq!(agg1.number_of_puts, 10); + assert_eq!(agg1.number_of_gets, 20); + + assert_eq!(agg2.stored_bytes, 100); + assert_eq!(agg2.transferred_bytes, 50); + assert_eq!(agg2.number_of_puts, 10); + assert_eq!(agg2.number_of_gets, 20); + + assert_eq!(agg3.stored_bytes, 100); + assert_eq!(agg3.transferred_bytes, 50); + assert_eq!(agg3.number_of_puts, 10); + assert_eq!(agg3.number_of_gets, 20); +} + +#[test] +fn nodes_aggregates_grouped_by_consistency_2() { + let redundancy_factor = 3; + let quorum = Percent::from_percent(67); + + let host = "example1.com"; + let port = 80; + let node_params = StorageNodeParams { + ssl: false, + host: host.as_bytes().to_vec(), + http_port: port, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example2.com".to_vec(), + }; + let aggregator = AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([0; 32])), + node_params: node_params.clone(), + }; + + let nodes_aggregates = vec![ + NodeAggregate { + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + NodeAggregate { + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + NodeAggregate { + node_id: "1".to_string(), + stored_bytes: 100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + NodeAggregate { + node_id: "2".to_string(), + stored_bytes: 110, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + NodeAggregate { + node_id: "2".to_string(), + stored_bytes: 110, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + NodeAggregate { + node_id: "2".to_string(), + stored_bytes: 110, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: aggregator.clone(), + }, + ]; + + let groups = DdcVerification::group_by_consistency(nodes_aggregates, redundancy_factor, quorum); + + assert_eq!(groups.in_consensus.len(), 2); + assert_eq!(groups.in_quorum.len(), 0); + assert_eq!(groups.in_others.len(), 0); + + let g1_agg1 = groups.in_consensus[0].get(0).unwrap(); + let g1_agg2 = groups.in_consensus[0].get(1).unwrap(); + let g1_agg3 = groups.in_consensus[0].get(2).unwrap(); + + assert_eq!(g1_agg1.node_id, "2".to_string()); + assert_eq!(g1_agg1.stored_bytes, 110); + assert_eq!(g1_agg1.transferred_bytes, 50); + assert_eq!(g1_agg1.number_of_puts, 10); + assert_eq!(g1_agg1.number_of_gets, 20); + + assert_eq!(g1_agg2.node_id, "2".to_string()); + assert_eq!(g1_agg2.stored_bytes, 110); + assert_eq!(g1_agg2.transferred_bytes, 50); + assert_eq!(g1_agg2.number_of_puts, 10); + assert_eq!(g1_agg2.number_of_gets, 20); + + assert_eq!(g1_agg3.node_id, "2".to_string()); + assert_eq!(g1_agg3.stored_bytes, 110); + assert_eq!(g1_agg3.transferred_bytes, 50); + assert_eq!(g1_agg3.number_of_puts, 10); + assert_eq!(g1_agg3.number_of_gets, 20); + + let g2_agg1 = groups.in_consensus[1].get(0).unwrap(); + let g2_agg2 = groups.in_consensus[1].get(1).unwrap(); + let g2_agg3 = groups.in_consensus[1].get(2).unwrap(); + + assert_eq!(g2_agg1.node_id, "1".to_string()); + assert_eq!(g2_agg1.stored_bytes, 100); + assert_eq!(g2_agg1.transferred_bytes, 50); + assert_eq!(g2_agg1.number_of_puts, 10); + assert_eq!(g2_agg1.number_of_gets, 20); + + assert_eq!(g2_agg2.node_id, "1".to_string()); + assert_eq!(g2_agg2.stored_bytes, 100); + assert_eq!(g2_agg2.transferred_bytes, 50); + assert_eq!(g2_agg2.number_of_puts, 10); + assert_eq!(g2_agg2.number_of_gets, 20); + + assert_eq!(g2_agg3.node_id, "1".to_string()); + assert_eq!(g2_agg3.stored_bytes, 100); + assert_eq!(g2_agg3.transferred_bytes, 50); + assert_eq!(g2_agg3.number_of_puts, 10); + assert_eq!(g2_agg3.number_of_gets, 20); +} + +#[test] +fn empty_bucket_sub_aggregates() { + let redundancy_factor = 3; + let quorum = Percent::from_percent(67); + + let empty = Vec::::new(); + let groups = DdcVerification::group_by_consistency(empty, redundancy_factor, quorum); + + assert_eq!(groups.in_consensus.len(), 0); + assert_eq!(groups.in_quorum.len(), 0); + assert_eq!(groups.in_others.len(), 0); +} + +#[test] +fn bucket_sub_aggregates_are_fetched_and_grouped() { + let mut ext = new_test_ext(); + let (offchain, offchain_state) = TestOffchainExt::new(); + let (pool, _pool_state) = TestTransactionPoolExt::new(); + + let (pair, _seed) = sp_core::sr25519::Pair::from_phrase( + "spider sell nice animal border success square soda stem charge caution echo", + None, + ) + .unwrap(); + let keystore = MemoryKeystore::new(); + keystore + .insert( + KEY_TYPE, + "0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa318", + pair.public().as_ref(), + ) + .unwrap(); + + ext.register_extension(OffchainWorkerExt::new(offchain.clone())); + ext.register_extension(OffchainDbExt::new(offchain)); + ext.register_extension(TransactionPoolExt::new(pool)); + ext.register_extension(KeystoreExt::new(keystore)); + + ext.execute_with(|| { + let mut offchain_state = offchain_state.write(); + let key = format!("offchain::validator::{:?}", KEY_TYPE).into_bytes(); + offchain_state.persistent_storage.set( + b"", + &key, + b"9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a".as_ref(), + ); + offchain_state.timestamp = Timestamp::from_unix_millis(0); + let host1 = "178.251.228.236"; + let host2 = "95.217.8.119"; + let host3 = "178.251.228.42"; + let host4 = "37.27.30.47"; + let host5 = "178.251.228.49"; + + let port = 8080; + + let pending_request1 = PendingRequest { + method: "GET".to_string(), + uri: format!("http://{}:{}/activity/buckets?eraId=476817", host1, port), + response: Some(br#"[{"bucket_id":90235,"stored_bytes":0,"transferred_bytes":38,"number_of_puts":0,"number_of_gets":1,"sub_aggregates":[{"NodeID":"0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa318","stored_bytes":578,"transferred_bytes":578,"number_of_puts":2,"number_of_gets":0}]},{"bucket_id":90235,"stored_bytes":0,"transferred_bytes":38,"number_of_puts":0,"number_of_gets":1,"sub_aggregates":[{"NodeID":"0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa319","stored_bytes":0,"transferred_bytes":505,"number_of_puts":0,"number_of_gets":1}]}]"#.to_vec()), + sent: true, + ..Default::default() + }; + + let pending_request2 = PendingRequest { + method: "GET".to_string(), + uri: format!("http://{}:{}/activity/buckets?eraId=476817", host2, port), + response: Some(br#"[{"bucket_id":90235,"stored_bytes":0,"transferred_bytes":38,"number_of_puts":0,"number_of_gets":1,"sub_aggregates":[{"NodeID":"0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa318","stored_bytes":578,"transferred_bytes":578,"number_of_puts":2,"number_of_gets":0}]},{"bucket_id":90235,"stored_bytes":0,"transferred_bytes":38,"number_of_puts":0,"number_of_gets":1,"sub_aggregates":[{"NodeID":"0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa319","stored_bytes":0,"transferred_bytes":506,"number_of_puts":0,"number_of_gets":1}]}]"#.to_vec()), + sent: true, + ..Default::default() + }; + + let pending_request3 = PendingRequest { + method: "GET".to_string(), + uri: format!("http://{}:{}/activity/buckets?eraId=476817", host3, port), + response: Some(br#"[{"bucket_id":90235,"stored_bytes":0,"transferred_bytes":38,"number_of_puts":0,"number_of_gets":1,"sub_aggregates":[{"NodeID":"0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa318","stored_bytes":578,"transferred_bytes":578,"number_of_puts":2,"number_of_gets":0}]},{"bucket_id":90235,"stored_bytes":0,"transferred_bytes":38,"number_of_puts":0,"number_of_gets":1,"sub_aggregates":[{"NodeID":"0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa319","stored_bytes":0,"transferred_bytes":505,"number_of_puts":0,"number_of_gets":1}]}]"#.to_vec()), + sent: true, + ..Default::default() + }; + + let pending_request4 = PendingRequest { + method: "GET".to_string(), + uri: format!("http://{}:{}/activity/buckets?eraId=476817", host4, port), + response: Some(br#"[{"bucket_id":90235,"stored_bytes":0,"transferred_bytes":38,"number_of_puts":0,"number_of_gets":1,"sub_aggregates":[]}]"#.to_vec()), + sent: true, + ..Default::default() + }; + + let pending_request5 = PendingRequest { + method: "GET".to_string(), + uri: format!("http://{}:{}/activity/buckets?eraId=476817", host5, port), + response: Some(br#"[{"bucket_id":90235,"stored_bytes":0,"transferred_bytes":38,"number_of_puts":0,"number_of_gets":1,"sub_aggregates":[{"NodeID":"0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa320","stored_bytes":578,"transferred_bytes":578,"number_of_puts":2,"number_of_gets":0}]}]"#.to_vec()), + sent: true, + ..Default::default() + }; + + offchain_state.expect_request(pending_request1); + offchain_state.expect_request(pending_request2); + offchain_state.expect_request(pending_request3); + offchain_state.expect_request(pending_request4); + offchain_state.expect_request(pending_request5); + + drop(offchain_state); + + let cluster_id = ClusterId::from([1; 20]); + let era_id = 476817; + let redundancy_factor = 3; + let aggregators_quorum = Percent::from_percent(67); + + let node_params1 = StorageNodeParams { + ssl: false, + host: host1.as_bytes().to_vec(), + http_port: port, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example2.com".to_vec(), + }; + + let node_params2 = StorageNodeParams { + ssl: false, + host: host2.as_bytes().to_vec(), + http_port: port, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example3.com".to_vec(), + }; + + let node_params3 = StorageNodeParams { + ssl: false, + host: host3.as_bytes().to_vec(), + http_port: port, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example4.com".to_vec(), + }; + + let node_params4 = StorageNodeParams { + ssl: false, + host: host4.as_bytes().to_vec(), + http_port: port, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example5.com".to_vec(), + }; + + let node_params5 = StorageNodeParams { + ssl: false, + host: host5.as_bytes().to_vec(), + http_port: port, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example6.com".to_vec(), + }; + + let dac_nodes: Vec<(NodePubKey, StorageNodeParams)> = vec![ + (NodePubKey::StoragePubKey(StorageNodePubKey::new([1; 32])), node_params1.clone()), + (NodePubKey::StoragePubKey(StorageNodePubKey::new([2; 32])), node_params2.clone()), + (NodePubKey::StoragePubKey(StorageNodePubKey::new([3; 32])), node_params3.clone()), + (NodePubKey::StoragePubKey(StorageNodePubKey::new([4; 32])), node_params4.clone()), + (NodePubKey::StoragePubKey(StorageNodePubKey::new([5; 32])), node_params5.clone()), + ]; + + let bucket_aggregates_by_aggregator = + DdcVerification::fetch_buckets_aggregates_for_era(&cluster_id, era_id, &dac_nodes) + .unwrap(); + + let groups = + DdcVerification::group_buckets_sub_aggregates_by_consistency(&cluster_id, era_id, bucket_aggregates_by_aggregator, redundancy_factor, aggregators_quorum); + + + // Sub aggregates which are in consensus + let bucket_sub_aggregate_in_consensus = BucketSubAggregate { + bucket_id: 90235, + node_id: "0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa318" + .to_string(), + stored_bytes: 578, + transferred_bytes: 578, + number_of_puts: 2, + number_of_gets: 0, + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: node_params1.clone(), + }, + }; + + assert_eq!( + groups.in_consensus, + vec![ + ConsistentGroup(bucket_sub_aggregate_in_consensus.hash::(), vec![ + bucket_sub_aggregate_in_consensus.clone(), + BucketSubAggregate { + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([2; 32])), + node_params: node_params2.clone(), + }, + ..bucket_sub_aggregate_in_consensus.clone() + }, + BucketSubAggregate { + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([3; 32])), + node_params: node_params3.clone(), + }, + ..bucket_sub_aggregate_in_consensus.clone() + }, + ]) + ] + ); + + // Sub aggregates which are in quorum + let bucket_sub_aggregate_in_quorum = BucketSubAggregate { + bucket_id: 90235, + node_id: "0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa319" + .to_string(), + stored_bytes: 0, + transferred_bytes: 505, + number_of_puts: 0, + number_of_gets: 1, + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: node_params1.clone(), + }, + }; + + assert_eq!( + groups.in_quorum, + vec![ + ConsistentGroup(bucket_sub_aggregate_in_quorum.hash::(), vec![bucket_sub_aggregate_in_quorum.clone(), BucketSubAggregate {aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([3; 32])), + node_params: node_params3.clone(), + }, + ..bucket_sub_aggregate_in_quorum.clone() + }, + ]), + ] + ); + + // Others sub aggregates + let bucket_sub_aggregate1_in_others = BucketSubAggregate { + bucket_id: 90235, + node_id: "0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa319" + .to_string(), + stored_bytes: 0, + transferred_bytes: 506, + number_of_puts: 0, + number_of_gets: 1, + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([2; 32])), + node_params: node_params2.clone(), + }, + }; + + let bucket_sub_aggregate2_in_others = BucketSubAggregate { + bucket_id: 90235, + node_id: "0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa320" + .to_string(), + stored_bytes: 578, + transferred_bytes: 578, + number_of_puts: 2, + number_of_gets: 0, + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([5; 32])), + node_params: node_params5.clone(), + }, + }; + + assert_eq!( + groups.in_others, + vec![ + ConsistentGroup(bucket_sub_aggregate2_in_others.hash::(), vec![bucket_sub_aggregate2_in_others]), + ConsistentGroup(bucket_sub_aggregate1_in_others.hash::(), vec![bucket_sub_aggregate1_in_others]), + ] + ); + }); } #[test] -fn test_get_consensus_customers_activity_not_in_consensus() { - let cluster_id1 = ClusterId::from([1; 20]); - let era_id1 = 1; - let min_nodes = 3; - let threshold = Percent::from_percent(67); - - let node_pubkey_0 = NodePubKey::StoragePubKey(AccountId32::new([0; 32])); - let node_pubkey_1 = NodePubKey::StoragePubKey(AccountId32::new([1; 32])); - let node_pubkey_2 = NodePubKey::StoragePubKey(AccountId32::new([2; 32])); - - let customers_activity = vec![ - ( - node_pubkey_0, - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ( - node_pubkey_1, - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 200, - transferred_bytes: 100, - number_of_puts: 20, - number_of_gets: 40, - }], - }], - ), - ( - node_pubkey_2, - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 300, - transferred_bytes: 150, - number_of_puts: 30, - number_of_gets: 60, - }], - }], - ), - ]; +fn node_aggregates_are_fetched_and_grouped() { + let mut ext = new_test_ext(); + let (offchain, offchain_state) = TestOffchainExt::new(); + let (pool, _pool_state) = TestTransactionPoolExt::new(); - let result = DdcVerification::get_consensus_for_activities( - &cluster_id1, - era_id1, - &customers_activity, - min_nodes, - threshold, - ); - assert!(result.is_err()); - let errors = result.err().unwrap(); - assert_eq!(errors.len(), 1); - match &errors[0] { - OCWError::ActivityNotInConsensus { cluster_id, era_id, id } => { - assert_eq!(*id, customers_activity[0].1[0].get_consensus_id::()); - assert_eq!(*cluster_id, cluster_id1); - assert_eq!(*era_id, era_id1); - }, - _ => panic!("Expected CustomerActivityNotInConsensus error"), - } -} + let (pair, _seed) = sp_core::sr25519::Pair::from_phrase( + "spider sell nice animal border success square soda stem charge caution echo", + None, + ) + .unwrap(); + let keystore = MemoryKeystore::new(); + keystore + .insert( + KEY_TYPE, + "0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa318", + pair.public().as_ref(), + ) + .unwrap(); -#[test] -fn test_get_consensus_customers_activity_not_in_consensus_2() { - let cluster_id1 = ClusterId::from([1; 20]); - let era_id1 = 1; - let min_nodes = 3; - let threshold = Percent::from_percent(67); - - let node_pubkey_0 = NodePubKey::StoragePubKey(AccountId32::new([0; 32])); - let node_pubkey_1 = NodePubKey::StoragePubKey(AccountId32::new([1; 32])); - let node_pubkey_2 = NodePubKey::StoragePubKey(AccountId32::new([2; 32])); - - let customers_activity = vec![ - ( - node_pubkey_0.clone(), - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ( - node_pubkey_1.clone(), - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 200, - transferred_bytes: 100, - number_of_puts: 20, - number_of_gets: 40, - }], - }], - ), - ( - node_pubkey_2.clone(), - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 300, - transferred_bytes: 150, - number_of_puts: 30, - number_of_gets: 60, - }], - }], - ), - ( - node_pubkey_0, - vec![CustomerActivity { - bucket_id: 2, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ( - node_pubkey_1, - vec![CustomerActivity { - bucket_id: 2, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 200, - transferred_bytes: 100, - number_of_puts: 20, - number_of_gets: 40, - }], - }], - ), - ( - node_pubkey_2, - vec![CustomerActivity { - bucket_id: 2, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 300, - transferred_bytes: 150, - number_of_puts: 30, - number_of_gets: 60, - }], - }], - ), - ]; + ext.register_extension(OffchainWorkerExt::new(offchain.clone())); + ext.register_extension(OffchainDbExt::new(offchain)); + ext.register_extension(TransactionPoolExt::new(pool)); + ext.register_extension(KeystoreExt::new(keystore)); - let result = DdcVerification::get_consensus_for_activities( - &cluster_id1, - era_id1, - &customers_activity, - min_nodes, - threshold, - ); - assert!(result.is_err()); - let errors = result.err().unwrap(); - assert_eq!(errors.len(), 2); - match &errors[1] { - OCWError::ActivityNotInConsensus { cluster_id, era_id, id } => { - assert_eq!(*id, customers_activity[3].1[0].get_consensus_id::()); - assert_eq!(*cluster_id, cluster_id1); - assert_eq!(*era_id, era_id1); - }, - _ => panic!("Expected CustomerActivityNotInConsensus error"), - } - match &errors[0] { - OCWError::ActivityNotInConsensus { cluster_id, era_id, id } => { - assert_eq!(*id, customers_activity[0].1[0].get_consensus_id::()); - assert_eq!(*cluster_id, cluster_id1); - assert_eq!(*era_id, era_id1); - }, - _ => panic!("Expected CustomerActivityNotInConsensus error"), - } -} + ext.execute_with(|| { + let mut offchain_state = offchain_state.write(); + let key = format!("offchain::validator::{:?}", KEY_TYPE).into_bytes(); + offchain_state.persistent_storage.set( + b"", + &key, + b"9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a".as_ref(), + ); + offchain_state.timestamp = Timestamp::from_unix_millis(0); + let host1 = "178.251.228.236"; + let host2 = "95.217.8.119"; + let host3 = "178.251.228.42"; + let host4 = "37.27.30.47"; + let host5 = "178.251.228.49"; -#[test] -fn test_get_consensus_customers_activity_diff_errors() { - let cluster_id1 = ClusterId::from([1; 20]); - let era_id1 = 1; - let min_nodes = 3; - let threshold = Percent::from_percent(67); - - let node_pubkey_0 = NodePubKey::StoragePubKey(AccountId32::new([0; 32])); - let node_pubkey_1 = NodePubKey::StoragePubKey(AccountId32::new([1; 32])); - let node_pubkey_2 = NodePubKey::StoragePubKey(AccountId32::new([2; 32])); - - let customers_activity = vec![ - ( - node_pubkey_0.clone(), - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ( - node_pubkey_1.clone(), - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 200, - transferred_bytes: 100, - number_of_puts: 20, - number_of_gets: 40, - }], - }], - ), - ( - node_pubkey_2.clone(), - vec![CustomerActivity { - bucket_id: 1, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 300, - transferred_bytes: 150, - number_of_puts: 30, - number_of_gets: 60, - }], - }], - ), - ( - node_pubkey_0, - vec![CustomerActivity { - bucket_id: 2, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - }], - ), - ( - node_pubkey_1, - vec![CustomerActivity { - bucket_id: 2, - sub_aggregates: vec![BucketSubAggregate { - NodeID: "1".to_string(), - stored_bytes: 200, - transferred_bytes: 100, - number_of_puts: 20, - number_of_gets: 40, - }], - }], - ), - ]; + let port = 8080; - let result = DdcVerification::get_consensus_for_activities( - &cluster_id1, - era_id1, - &customers_activity, - min_nodes, - threshold, - ); - assert!(result.is_err()); - let errors = result.err().unwrap(); - assert_eq!(errors.len(), 3); - match &errors[0] { - OCWError::ActivityNotInConsensus { cluster_id, era_id, id } => { - assert_eq!(*id, customers_activity[0].1[0].get_consensus_id::()); - assert_eq!(*cluster_id, cluster_id1); - assert_eq!(*era_id, era_id1); - }, - _ => panic!("Expected CustomerActivityNotInConsensus error"), - } - match &errors[1] { - OCWError::NotEnoughBucketsForConsensus { cluster_id, era_id, bucket_id } => { - assert_eq!(*bucket_id, 2); - assert_eq!(*cluster_id, cluster_id1); - assert_eq!(*era_id, era_id1); - }, - _ => panic!("Expected CustomerActivityNotInConsensus error"), - } -} + let pending_request1 = PendingRequest { + method: "GET".to_string(), + uri: format!("http://{}:{}/activity/nodes?eraId=476817", host1, port), + response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), + sent: true, + ..Default::default() + }; -#[test] -fn test_get_consensus_nodes_activity_not_in_consensus() { - let cluster_id1 = ClusterId::from([1; 20]); - let era_id1 = 1; - let min_nodes = 3; - let threshold = Percent::from_percent(67); - - let node_pubkey_0 = NodePubKey::StoragePubKey(AccountId32::new([0; 32])); - let node_pubkey_1 = NodePubKey::StoragePubKey(AccountId32::new([1; 32])); - let node_pubkey_2 = NodePubKey::StoragePubKey(AccountId32::new([2; 32])); - - let nodes_activity = vec![ - ( - node_pubkey_0, - vec![NodeActivity { - node_id: "0".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - ), - ( - node_pubkey_1, - vec![NodeActivity { - node_id: "0".to_string(), - stored_bytes: 200, - transferred_bytes: 100, - number_of_puts: 20, - number_of_gets: 40, - }], - ), - ( - node_pubkey_2, - vec![NodeActivity { - node_id: "0".to_string(), - stored_bytes: 300, - transferred_bytes: 150, - number_of_puts: 30, - number_of_gets: 60, - }], - ), - ]; + let pending_request2 = PendingRequest { + method: "GET".to_string(), + uri: format!("http://{}:{}/activity/nodes?eraId=476817", host2, port), + response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","stored_bytes": 0, "transferred_bytes": 48,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), + sent: true, + ..Default::default() + }; - let result = DdcVerification::get_consensus_for_activities( - &cluster_id1, - era_id1, - &nodes_activity, - min_nodes, - threshold, - ); - assert!(result.is_err()); - let errors = result.err().unwrap(); - assert_eq!(errors.len(), 1); - match &errors[0] { - OCWError::ActivityNotInConsensus { cluster_id, era_id, id } => { - assert_eq!(*id, nodes_activity[0].1[0].get_consensus_id::()); - assert_eq!(*cluster_id, cluster_id1); - assert_eq!(*era_id, era_id1); - }, - _ => panic!("Expected CustomerActivityNotInConsensus error"), - } + let pending_request3 = PendingRequest { + method: "GET".to_string(), + uri: format!("http://{}:{}/activity/nodes?eraId=476817", host3, port), + response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97}]"#.to_vec()), + sent: true, + ..Default::default() + }; + + let pending_request4 = PendingRequest { + method: "GET".to_string(), + uri: format!("http://{}:{}/activity/nodes?eraId=476817", host4, port), + response: Some(br#"[{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), + sent: true, + ..Default::default() + }; + + let pending_request5 = PendingRequest { + method: "GET".to_string(), + uri: format!("http://{}:{}/activity/nodes?eraId=476817", host5, port), + response: Some(br#"[{"node_id": "0xfc28d5f5bb10212077a8654f62c4f8f0b5ab985fc322a51f5a3c75943b29194b","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97}]"#.to_vec()), + sent: true, + ..Default::default() + }; + + offchain_state.expect_request(pending_request1); + offchain_state.expect_request(pending_request2); + offchain_state.expect_request(pending_request3); + offchain_state.expect_request(pending_request4); + offchain_state.expect_request(pending_request5); + + drop(offchain_state); + + let cluster_id = ClusterId::from([1; 20]); + let era_id = 476817; + let redundancy_factor = 3; + let aggregators_quorum = Percent::from_percent(67); + + let node_params1 = StorageNodeParams { + ssl: false, + host: host1.as_bytes().to_vec(), + http_port: port, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example2.com".to_vec(), + }; + + let node_params2 = StorageNodeParams { + ssl: false, + host: host2.as_bytes().to_vec(), + http_port: port, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example3.com".to_vec(), + }; + + let node_params3 = StorageNodeParams { + ssl: false, + host: host3.as_bytes().to_vec(), + http_port: port, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example4.com".to_vec(), + }; + + let node_params4 = StorageNodeParams { + ssl: false, + host: host4.as_bytes().to_vec(), + http_port: port, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example5.com".to_vec(), + }; + + let node_params5 = StorageNodeParams { + ssl: false, + host: host5.as_bytes().to_vec(), + http_port: port, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example6.com".to_vec(), + }; + + let dac_nodes: Vec<(NodePubKey, StorageNodeParams)> = vec![ + (NodePubKey::StoragePubKey(StorageNodePubKey::new([1; 32])), node_params1.clone()), + (NodePubKey::StoragePubKey(StorageNodePubKey::new([2; 32])), node_params2.clone()), + (NodePubKey::StoragePubKey(StorageNodePubKey::new([3; 32])), node_params3.clone()), + (NodePubKey::StoragePubKey(StorageNodePubKey::new([4; 32])), node_params4.clone()), + (NodePubKey::StoragePubKey(StorageNodePubKey::new([5; 32])), node_params5.clone()), + ]; + + let aggregates_by_aggregator = + DdcVerification::fetch_nodes_aggregates_for_era(&cluster_id, era_id, &dac_nodes) + .unwrap(); + + let groups = + DdcVerification::group_nodes_aggregates_by_consistency(&cluster_id, era_id, aggregates_by_aggregator, redundancy_factor, aggregators_quorum); + // Node aggregates which are in consensus + let node_aggregate_in_consensus = NodeAggregate { + node_id: "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e" + .to_string(), + stored_bytes: 675613289, + transferred_bytes: 1097091579, + number_of_puts: 889, + number_of_gets: 97, + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: node_params1.clone(), + }, + }; + + assert_eq!( + groups.in_consensus, + vec![ConsistentGroup(node_aggregate_in_consensus.hash::(), vec![node_aggregate_in_consensus.clone(), NodeAggregate { aggregator: AggregatorInfo { node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([2; 32])), node_params: node_params2.clone(), }, ..node_aggregate_in_consensus.clone() }, NodeAggregate { aggregator: AggregatorInfo { node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([3; 32])), node_params: node_params3.clone(), }, ..node_aggregate_in_consensus.clone() } ])] + ); + + // Node aggregates which are in quorum + let node_aggregate_in_quorum = NodeAggregate { + node_id: "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a" + .to_string(), + stored_bytes: 0, + transferred_bytes: 38, + number_of_puts: 0, + number_of_gets: 1, + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: node_params1.clone(), + }, + }; + + assert_eq!( + groups.in_quorum, vec![ConsistentGroup(node_aggregate_in_quorum.hash::(), vec![node_aggregate_in_quorum.clone(), NodeAggregate {aggregator: AggregatorInfo { node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([4; 32])), node_params: node_params4.clone(), }, ..node_aggregate_in_quorum.clone() }])] + ); + + // Others nodes aggregates + let node_aggregate1_in_others = NodeAggregate { + node_id: "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a" + .to_string(), + stored_bytes: 0, + transferred_bytes: 48, + number_of_puts: 0, + number_of_gets: 1, + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([2; 32])), + node_params: node_params2.clone(), + }, + }; + + let node_aggregate2_in_others = NodeAggregate { + node_id: "0xfc28d5f5bb10212077a8654f62c4f8f0b5ab985fc322a51f5a3c75943b29194b" + .to_string(), + stored_bytes: 675613289, + transferred_bytes: 1097091579, + number_of_puts: 889, + number_of_gets: 97, + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([5; 32])), + node_params: node_params5.clone(), + }, + }; + + assert_eq!( + groups.in_others, vec![ConsistentGroup(node_aggregate2_in_others.hash::(), vec![node_aggregate2_in_others]), ConsistentGroup(node_aggregate1_in_others.hash::(), vec![node_aggregate1_in_others])] + ); + }); } #[test] @@ -1182,7 +1919,7 @@ fn test_convert_to_batch_merkle_roots_empty() { let result_roots = DdcVerification::convert_to_batch_merkle_roots( &cluster_id, era_id_1, - Vec::>::new(), + Vec::>::new(), ) .unwrap(); let expected_roots: Vec = Vec::::new(); @@ -1192,9 +1929,9 @@ fn test_convert_to_batch_merkle_roots_empty() { #[test] fn test_split_to_batches_empty_activities() { - let activities: Vec = vec![]; + let activities: Vec = vec![]; let result = DdcVerification::split_to_batches(&activities, 3); - assert_eq!(result, Vec::>::new()); + assert_eq!(result, Vec::>::new()); } #[test] @@ -1244,7 +1981,7 @@ fn test_split_to_batches_non_exact_batches() { ]; sorted_activities.sort(); let result = DdcVerification::split_to_batches(&activities, 2); - let mut expected: Vec> = Vec::new(); + let mut expected: Vec> = Vec::new(); expected.push(vec![sorted_activities[0].clone(), sorted_activities[1].clone()]); expected.push(vec![sorted_activities[2].clone(), sorted_activities[3].clone()]); expected.push(vec![sorted_activities[4].clone()]); @@ -1252,200 +1989,6 @@ fn test_split_to_batches_non_exact_batches() { assert_eq!(result, expected); } -#[test] -fn test_get_consensus_nodes_activity_not_in_consensus2() { - let cluster_id1 = ClusterId::from([1; 20]); - let era_id1 = 1; - let min_nodes = 3; - let threshold = Percent::from_percent(67); - - let node_pubkey_0 = NodePubKey::StoragePubKey(AccountId32::new([0; 32])); - let node_pubkey_1 = NodePubKey::StoragePubKey(AccountId32::new([1; 32])); - let node_pubkey_2 = NodePubKey::StoragePubKey(AccountId32::new([2; 32])); - - let nodes_activity = vec![ - ( - node_pubkey_0.clone(), - vec![NodeActivity { - node_id: "0".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - ), - ( - node_pubkey_1.clone(), - vec![NodeActivity { - node_id: "0".to_string(), - stored_bytes: 200, - transferred_bytes: 100, - number_of_puts: 20, - number_of_gets: 40, - }], - ), - ( - node_pubkey_2.clone(), - vec![NodeActivity { - node_id: "0".to_string(), - stored_bytes: 300, - transferred_bytes: 150, - number_of_puts: 30, - number_of_gets: 60, - }], - ), - ( - node_pubkey_0, - vec![NodeActivity { - node_id: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - ), - ( - node_pubkey_1, - vec![NodeActivity { - node_id: "1".to_string(), - stored_bytes: 200, - transferred_bytes: 100, - number_of_puts: 20, - number_of_gets: 40, - }], - ), - ( - node_pubkey_2, - vec![NodeActivity { - node_id: "1".to_string(), - stored_bytes: 300, - transferred_bytes: 150, - number_of_puts: 30, - number_of_gets: 60, - }], - ), - ]; - - let result = DdcVerification::get_consensus_for_activities( - &cluster_id1, - era_id1, - &nodes_activity, - min_nodes, - threshold, - ); - assert!(result.is_err()); - let errors = result.err().unwrap(); - assert_eq!(errors.len(), 2); - match &errors[0] { - OCWError::ActivityNotInConsensus { cluster_id, era_id, id } => { - assert_eq!(*id, nodes_activity[0].1[0].get_consensus_id::()); - assert_eq!(*cluster_id, cluster_id1); - assert_eq!(*era_id, era_id1); - }, - _ => panic!("Expected CustomerActivityNotInConsensus error"), - } - match &errors[1] { - OCWError::ActivityNotInConsensus { cluster_id, era_id, id } => { - assert_eq!(*id, nodes_activity[3].1[0].get_consensus_id::()); - assert_eq!(*cluster_id, cluster_id1); - assert_eq!(*era_id, era_id1); - }, - _ => panic!("Expected CustomerActivityNotInConsensus error"), - } -} - -#[test] -fn test_get_consensus_nodes_activity_diff_errors() { - let cluster_id1 = ClusterId::from([1; 20]); - let era_id1 = 1; - let min_nodes = 3; - let threshold = Percent::from_percent(67); - - let node_pubkey_0 = NodePubKey::StoragePubKey(AccountId32::new([0; 32])); - let node_pubkey_1 = NodePubKey::StoragePubKey(AccountId32::new([1; 32])); - let node_pubkey_2 = NodePubKey::StoragePubKey(AccountId32::new([2; 32])); - - let nodes_activity = vec![ - ( - node_pubkey_0.clone(), - vec![NodeActivity { - node_id: "0".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - ), - ( - node_pubkey_1.clone(), - vec![NodeActivity { - node_id: "0".to_string(), - stored_bytes: 200, - transferred_bytes: 100, - number_of_puts: 20, - number_of_gets: 40, - }], - ), - ( - node_pubkey_2.clone(), - vec![NodeActivity { - node_id: "0".to_string(), - stored_bytes: 300, - transferred_bytes: 150, - number_of_puts: 30, - number_of_gets: 60, - }], - ), - ( - node_pubkey_0, - vec![NodeActivity { - node_id: "1".to_string(), - stored_bytes: 100, - transferred_bytes: 50, - number_of_puts: 10, - number_of_gets: 20, - }], - ), - ( - node_pubkey_1, - vec![NodeActivity { - node_id: "1".to_string(), - stored_bytes: 200, - transferred_bytes: 100, - number_of_puts: 20, - number_of_gets: 40, - }], - ), - ]; - - let result = DdcVerification::get_consensus_for_activities( - &cluster_id1, - era_id1, - &nodes_activity, - min_nodes, - threshold, - ); - assert!(result.is_err()); - let errors = result.err().unwrap(); - assert_eq!(errors.len(), 3); - match &errors[0] { - OCWError::ActivityNotInConsensus { cluster_id, era_id, id } => { - assert_eq!(*id, nodes_activity[0].1[0].get_consensus_id::()); - assert_eq!(*cluster_id, cluster_id1); - assert_eq!(*era_id, era_id1); - }, - _ => panic!("Expected CustomerActivityNotInConsensus error"), - } - match &errors[1] { - OCWError::NotEnoughNodesForConsensus { cluster_id, era_id, node_id } => { - assert_eq!(*node_id, "1".to_string()); - assert_eq!(*cluster_id, cluster_id1); - assert_eq!(*era_id, era_id1); - }, - _ => panic!("Expected CustomerActivityNotInConsensus error"), - } -} - #[test] fn fetch_processed_era_works() { let mut ext = TestExternalities::default(); @@ -1957,7 +2500,7 @@ fn test_single_ocw_pallet_integration() { let node_pending_request1 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/nodes?eraId=476814", host1, port), - response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","provider_id": "0xf6a3e4c537ccee3dbac555ef6df371b7e48594f1fd4f05135914c42b03e63b61","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","provider_id": "0x8d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a9ef98ad9c3626ba725e7","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), + response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), sent: true, ..Default::default() }; @@ -1965,7 +2508,7 @@ fn test_single_ocw_pallet_integration() { let node_pending_request2 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/nodes?eraId=476814", host2, port), - response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","provider_id": "0xf6a3e4c537ccee3dbac555ef6df371b7e48594f1fd4f05135914c42b03e63b61","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","provider_id": "0x8d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a9ef98ad9c3626ba725e7","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), + response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), sent: true, ..Default::default() }; @@ -1973,7 +2516,7 @@ fn test_single_ocw_pallet_integration() { let node_pending_request3 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/nodes?eraId=476814", host3, port), - response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","provider_id": "0xf6a3e4c537ccee3dbac555ef6df371b7e48594f1fd4f05135914c42b03e63b61","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","provider_id": "0x8d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a9ef98ad9c3626ba725e7","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), + response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), sent: true, ..Default::default() }; @@ -1981,7 +2524,7 @@ fn test_single_ocw_pallet_integration() { let node_pending_request4 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/nodes?eraId=476814", host4, port), - response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","provider_id": "0xf6a3e4c537ccee3dbac555ef6df371b7e48594f1fd4f05135914c42b03e63b61","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","provider_id": "0x8d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a9ef98ad9c3626ba725e7","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), + response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), sent: true, ..Default::default() }; @@ -1989,7 +2532,7 @@ fn test_single_ocw_pallet_integration() { let node_pending_request5 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/nodes?eraId=476814", host5, port), - response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","provider_id": "0xf6a3e4c537ccee3dbac555ef6df371b7e48594f1fd4f05135914c42b03e63b61","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","provider_id": "0x8d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a9ef98ad9c3626ba725e7","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), + response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), sent: true, ..Default::default() }; @@ -1997,7 +2540,7 @@ fn test_single_ocw_pallet_integration() { let node_pending_request6 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/nodes?eraId=476814", host6, port), - response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","provider_id": "0xf6a3e4c537ccee3dbac555ef6df371b7e48594f1fd4f05135914c42b03e63b61","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","provider_id": "0x8d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a9ef98ad9c3626ba725e7","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), + response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), sent: true, ..Default::default() }; @@ -2005,7 +2548,7 @@ fn test_single_ocw_pallet_integration() { let node_pending_request7 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/nodes?eraId=476814", host7, port), - response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","provider_id": "0xf6a3e4c537ccee3dbac555ef6df371b7e48594f1fd4f05135914c42b03e63b61","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","provider_id": "0x8d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a9ef98ad9c3626ba725e7","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), + response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), sent: true, ..Default::default() }; @@ -2013,7 +2556,7 @@ fn test_single_ocw_pallet_integration() { let node_pending_request8 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/nodes?eraId=476814", host8, port), - response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","provider_id": "0xf6a3e4c537ccee3dbac555ef6df371b7e48594f1fd4f05135914c42b03e63b61","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","provider_id": "0x8d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a9ef98ad9c3626ba725e7","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), + response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), sent: true, ..Default::default() }; @@ -2021,7 +2564,7 @@ fn test_single_ocw_pallet_integration() { let node_pending_request9 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/nodes?eraId=476814", host9, port), - response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","provider_id": "0xf6a3e4c537ccee3dbac555ef6df371b7e48594f1fd4f05135914c42b03e63b61","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","provider_id": "0x8d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a9ef98ad9c3626ba725e7","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), + response: Some(br#"[{"node_id": "0x48594f1fd4f05135914c42b03e63b61f6a3e4c537ccee3dbac555ef6df371b7e","stored_bytes": 675613289,"transferred_bytes": 1097091579,"number_of_puts": 889,"number_of_gets": 97},{"node_id": "0x9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a","stored_bytes": 0, "transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]"#.to_vec()), sent: true, ..Default::default() }; @@ -2029,7 +2572,7 @@ fn test_single_ocw_pallet_integration() { let bucket_pending_request1 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/buckets?eraId=476814", host1, port), - response: Some(br#"[{"bucket_id": 90235,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), + response: Some(br#"[{"bucket_id": 90235,"stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), sent: true, ..Default::default() }; @@ -2037,7 +2580,7 @@ fn test_single_ocw_pallet_integration() { let bucket_pending_request2 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/buckets?eraId=476814", host2, port), - response: Some(br#"[{"bucket_id": 90235,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), + response: Some(br#"[{"stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1,"bucket_id": 90235,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), sent: true, ..Default::default() }; @@ -2045,7 +2588,7 @@ fn test_single_ocw_pallet_integration() { let bucket_pending_request3 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/buckets?eraId=476814", host3, port), - response: Some(br#"[{"bucket_id": 90235,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), + response: Some(br#"[{"stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1,"bucket_id": 90235,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), sent: true, ..Default::default() }; @@ -2053,7 +2596,7 @@ fn test_single_ocw_pallet_integration() { let bucket_pending_request4 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/buckets?eraId=476814", host4, port), - response: Some(br#"[{"bucket_id": 90235,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), + response: Some(br#"[{"stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1,"bucket_id": 90235,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), sent: true, ..Default::default() }; @@ -2061,7 +2604,7 @@ fn test_single_ocw_pallet_integration() { let bucket_pending_request5 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/buckets?eraId=476814", host5, port), - response: Some(br#"[{"bucket_id": 90235,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), + response: Some(br#"[{"stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1,"bucket_id": 90235,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), sent: true, ..Default::default() }; @@ -2069,7 +2612,7 @@ fn test_single_ocw_pallet_integration() { let bucket_pending_request6 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/buckets?eraId=476814", host6, port), - response: Some(br#"[{"bucket_id": 90235,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), + response: Some(br#"[{"stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1,"bucket_id": 90235,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), sent: true, ..Default::default() }; @@ -2077,7 +2620,7 @@ fn test_single_ocw_pallet_integration() { let bucket_pending_request7 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/buckets?eraId=476814", host7, port), - response: Some(br#"[{"bucket_id": 90235,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), + response: Some(br#"[{"stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1,"bucket_id": 90235,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), sent: true, ..Default::default() }; @@ -2085,7 +2628,7 @@ fn test_single_ocw_pallet_integration() { let bucket_pending_request8 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/buckets?eraId=476814", host8, port), - response: Some(br#"[{"bucket_id": 90235,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), + response: Some(br#"[{"bucket_id": 90235,"stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), sent: true, ..Default::default() }; @@ -2093,7 +2636,7 @@ fn test_single_ocw_pallet_integration() { let bucket_pending_request9 = PendingRequest { method: "GET".to_string(), uri: format!("http://{}:{}/activity/buckets?eraId=476814", host9, port), - response: Some(br#"[{"bucket_id": 90235,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), + response: Some(br#"[{"bucket_id": 90235,"stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1,"sub_aggregates": [{"NodeID": "0xbe26b2458fb0c9df4ec26ec5ba083051402b2a3b9d4a7fe6106fe9f8b5efde2c","stored_bytes": 0,"transferred_bytes": 38,"number_of_puts": 0,"number_of_gets": 1}]}]"#.to_vec()), sent: true, ..Default::default() }; @@ -2150,10 +2693,76 @@ fn fetch_reward_activities_works() { let era_id = 1; let total_usage: i64 = 56; + let node_params = StorageNodeParams { + ssl: false, + host: "178.251.228.236".as_bytes().to_vec(), + http_port: 8080, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example.com".to_vec(), + }; + let result = DdcVerification::fetch_reward_activities( &cluster_id, era_id, - get_node_activities(), + vec![ + NodeAggregate { + node_id: "0".to_string(), + stored_bytes: -100, + transferred_bytes: 50, + number_of_puts: 10, + number_of_gets: 20, + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: node_params.clone(), + }, + }, + NodeAggregate { + node_id: "1".to_string(), + stored_bytes: -101, + transferred_bytes: 51, + number_of_puts: 11, + number_of_gets: 21, + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: node_params.clone(), + }, + }, + NodeAggregate { + node_id: "2".to_string(), + stored_bytes: 102, + transferred_bytes: 52, + number_of_puts: 12, + number_of_gets: 22, + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: node_params.clone(), + }, + }, + NodeAggregate { + node_id: "3".to_string(), + stored_bytes: 103, + transferred_bytes: 53, + number_of_puts: 13, + number_of_gets: 23, + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: node_params.clone(), + }, + }, + NodeAggregate { + node_id: "4".to_string(), + stored_bytes: 104, + transferred_bytes: 54, + number_of_puts: 14, + number_of_gets: 24, + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([1; 32])), + node_params: node_params.clone(), + }, + }, + ], leaves.to_vec(), total_usage, ); @@ -2179,193 +2788,6 @@ fn fetch_reward_activities_works() { assert_eq!(result.unwrap(), Some((era_id, (leaves.len() - 1) as u16, ex_result))); } -#[test] -fn test_bucket_node_aggregates() { - let mut ext = new_test_ext(); - let (offchain, offchain_state) = TestOffchainExt::new(); - let (pool, _pool_state) = TestTransactionPoolExt::new(); - - let (pair, _seed) = sp_core::sr25519::Pair::from_phrase( - "spider sell nice animal border success square soda stem charge caution echo", - None, - ) - .unwrap(); - let keystore = MemoryKeystore::new(); - keystore - .insert( - KEY_TYPE, - "0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa318", - pair.public().as_ref(), - ) - .unwrap(); - - ext.register_extension(OffchainWorkerExt::new(offchain.clone())); - ext.register_extension(OffchainDbExt::new(offchain)); - ext.register_extension(TransactionPoolExt::new(pool)); - ext.register_extension(KeystoreExt::new(keystore)); - - ext.execute_with(|| { - let mut offchain_state = offchain_state.write(); - let key = format!("offchain::validator::{:?}", KEY_TYPE).into_bytes(); - offchain_state.persistent_storage.set( - b"", - &key, - b"9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a".as_ref(), - ); - offchain_state.timestamp = Timestamp::from_unix_millis(0); - let host1 = "178.251.228.236"; - let host2 = "95.217.8.119"; - let host3 = "178.251.228.42"; - let host4 = "37.27.30.47"; - - let port = 8080; - - let pending_request1 = PendingRequest { - method: "GET".to_string(), - uri: format!("http://{}:{}/activity/buckets?eraId=476817", host1, port), - response: Some(br#"[{"bucket_id":90235,"stored_bytes":0,"transferred_bytes":38,"number_of_puts":0,"number_of_gets":1,"sub_aggregates":[{"NodeID":"0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa318","stored_bytes":578,"transferred_bytes":578,"number_of_puts":2,"number_of_gets":0}]},{"bucket_id":90235,"stored_bytes":0,"transferred_bytes":38,"number_of_puts":0,"number_of_gets":1,"sub_aggregates":[{"NodeID":"0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa319","stored_bytes":0,"transferred_bytes":505,"number_of_puts":0,"number_of_gets":1}]}]"#.to_vec()), - sent: true, - ..Default::default() - }; - - let pending_request2 = PendingRequest { - method: "GET".to_string(), - uri: format!("http://{}:{}/activity/buckets?eraId=476817", host2, port), - response: Some(br#"[{"bucket_id":90235,"stored_bytes":0,"transferred_bytes":38,"number_of_puts":0,"number_of_gets":1,"sub_aggregates":[{"NodeID":"0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa318","stored_bytes":578,"transferred_bytes":578,"number_of_puts":2,"number_of_gets":0}]},{"bucket_id":90235,"stored_bytes":0,"transferred_bytes":38,"number_of_puts":0,"number_of_gets":1,"sub_aggregates":[{"NodeID":"0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa319","stored_bytes":0,"transferred_bytes":506,"number_of_puts":0,"number_of_gets":1}]}]"#.to_vec()), - sent: true, - ..Default::default() - }; - - let pending_request3 = PendingRequest { - method: "GET".to_string(), - uri: format!("http://{}:{}/activity/buckets?eraId=476817", host3, port), - response: Some(br#"[{"bucket_id":90235,"stored_bytes":0,"transferred_bytes":38,"number_of_puts":0,"number_of_gets":1,"sub_aggregates":[{"NodeID":"0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa318","stored_bytes":578,"transferred_bytes":578,"number_of_puts":2,"number_of_gets":0}]},{"bucket_id":90235,"stored_bytes":0,"transferred_bytes":38,"number_of_puts":0,"number_of_gets":1,"sub_aggregates":[{"NodeID":"0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa319","stored_bytes":0,"transferred_bytes":505,"number_of_puts":0,"number_of_gets":1}]}]"#.to_vec()), - sent: true, - ..Default::default() - }; - - let pending_request4 = PendingRequest { - method: "GET".to_string(), - uri: format!("http://{}:{}/activity/buckets?eraId=476817", host4, port), - response: Some(br#"[{"bucket_id":90235,"stored_bytes":0,"transferred_bytes":38,"number_of_puts":0,"number_of_gets":1,"sub_aggregates":[]}]"#.to_vec()), - sent: true, - ..Default::default() - }; - - offchain_state.expect_request(pending_request1); - offchain_state.expect_request(pending_request2); - offchain_state.expect_request(pending_request3); - offchain_state.expect_request(pending_request4); - drop(offchain_state); - - let cluster_id = ClusterId::from([1; 20]); - let era_id = 476817; - let min_nodes = 3; - - let node_params1 = StorageNodeParams { - ssl: false, - host: host1.as_bytes().to_vec(), - http_port: port, - mode: StorageNodeMode::DAC, - p2p_port: 5555, - grpc_port: 4444, - domain: b"example2.com".to_vec(), - }; - - let node_params2 = StorageNodeParams { - ssl: false, - host: host2.as_bytes().to_vec(), - http_port: port, - mode: StorageNodeMode::DAC, - p2p_port: 5555, - grpc_port: 4444, - domain: b"example3.com".to_vec(), - }; - - let node_params3 = StorageNodeParams { - ssl: false, - host: host3.as_bytes().to_vec(), - http_port: port, - mode: StorageNodeMode::DAC, - p2p_port: 5555, - grpc_port: 4444, - domain: b"example4.com".to_vec(), - }; - - let node_params4 = StorageNodeParams { - ssl: false, - host: host4.as_bytes().to_vec(), - http_port: port, - mode: StorageNodeMode::DAC, - p2p_port: 5555, - grpc_port: 4444, - domain: b"example5.com".to_vec(), - }; - - let dac_nodes: Vec<(NodePubKey, StorageNodeParams)> = vec![ - (NodePubKey::StoragePubKey(StorageNodePubKey::new([1; 32])), node_params1), - (NodePubKey::StoragePubKey(StorageNodePubKey::new([2; 32])), node_params2), - (NodePubKey::StoragePubKey(StorageNodePubKey::new([3; 32])), node_params3), - (NodePubKey::StoragePubKey(StorageNodePubKey::new([4; 32])), node_params4), - ]; - - let customers_usage = - DdcVerification::fetch_customers_usage_for_era(&cluster_id, era_id, &dac_nodes) - .unwrap(); - - let result = - DdcVerification::fetch_sub_trees(&cluster_id, era_id, customers_usage, min_nodes); - - assert!(result.is_ok()); - // Sub_aggregates which are in consensus - assert_eq!( - result.clone().unwrap().0, - [BucketNodeAggregatesActivity { - bucket_id: 90235, - node_id: "0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa318" - .to_string(), - stored_bytes: 578, - transferred_bytes: 578, - number_of_puts: 2, - number_of_gets: 0 - }] - ); - // Sub_aggregates which are not in consensus - assert_eq!( - result.unwrap().1, - [ - BucketNodeAggregatesActivity { - bucket_id: 90235, - node_id: "0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa319" - .to_string(), - stored_bytes: 0, - transferred_bytes: 505, - number_of_puts: 0, - number_of_gets: 1 - }, - BucketNodeAggregatesActivity { - bucket_id: 90235, - node_id: "0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa319" - .to_string(), - stored_bytes: 0, - transferred_bytes: 505, - number_of_puts: 0, - number_of_gets: 1 - }, - BucketNodeAggregatesActivity { - bucket_id: 90235, - node_id: "0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa319" - .to_string(), - stored_bytes: 0, - transferred_bytes: 506, - number_of_puts: 0, - number_of_gets: 1 - } - ] - ); - }); -} - #[test] fn test_find_random_merkle_node_ids() { let mut ext = TestExternalities::default(); @@ -2375,34 +2797,50 @@ fn test_find_random_merkle_node_ids() { ext.register_extension(OffchainWorkerExt::new(offchain.clone())); ext.register_extension(OffchainDbExt::new(Box::new(offchain))); ext.register_extension(TransactionPoolExt::new(pool)); + let host1 = "178.251.228.236"; + + let port = 8080; + let node_params1 = StorageNodeParams { + ssl: false, + host: host1.as_bytes().to_vec(), + http_port: port, + mode: StorageNodeMode::DAC, + p2p_port: 5555, + grpc_port: 4444, + domain: b"example2.com".to_vec(), + }; ext.execute_with(|| { - let bucket_node_aggregates_not_in_consensus: BucketNodeAggregatesActivity = - BucketNodeAggregatesActivity { - bucket_id: 90235, - node_id: "0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa319" - .to_string(), - stored_bytes: 0, - transferred_bytes: 505, - number_of_puts: 12, - number_of_gets: 13, - }; - - let total_activities = bucket_node_aggregates_not_in_consensus.number_of_gets + - bucket_node_aggregates_not_in_consensus.number_of_puts; - - let ids = DdcVerification::find_random_merkle_node_ids( + let deffective_bucket_sub_aggregate = BucketSubAggregate { + bucket_id: 90235, + node_id: "0xb6186f80dce7190294665ab53860de2841383bb202c562bb8b81a624351fa319" + .to_string(), + stored_bytes: 0, + transferred_bytes: 505, + number_of_puts: 12, + number_of_gets: 13, + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([0; 32])), + node_params: node_params1.clone(), + }, + }; + + let number_of_leaves = deffective_bucket_sub_aggregate.get_number_of_leaves(); + + let ids = DdcVerification::_find_random_merkle_node_ids( 3, - bucket_node_aggregates_not_in_consensus.clone(), + number_of_leaves, + deffective_bucket_sub_aggregate.get_key(), ); + for id in ids { - assert!(id < total_activities); + assert!(id < number_of_leaves); } }); } #[test] -fn test_challenge_sub_aggregates_not_in_consensus() { +fn challenge_bucket_sub_aggregate_works() { let mut ext = new_test_ext(); let (offchain, offchain_state) = TestOffchainExt::new(); let (pool, _pool_state) = TestTransactionPoolExt::new(); @@ -2435,10 +2873,7 @@ fn test_challenge_sub_aggregates_not_in_consensus() { b"9ef98ad9c3626ba725e78d76cfcfc4b4d07e84f0388465bc7eb992e3e117234a".as_ref(), ); offchain_state.timestamp = Timestamp::from_unix_millis(0); - let host1 = "178.251.228.236"; - let host2 = "95.217.8.119"; - let host3 = "178.251.228.42"; - let host4 = "37.27.30.47"; + let host1 = "178.251.228.165"; let port = 8080; @@ -2453,73 +2888,23 @@ fn test_challenge_sub_aggregates_not_in_consensus() { let pending_request2 = PendingRequest { method: "GET".to_string(), - uri: format!("http://{}:{}/activity/buckets/123229/challenge?eraId=5757773&nodeId=0x1f50f1455f60f5774564233d321a116ca45ae3188b2200999445706d04839d72&merkleTreeNodeId=0,2,1,3", host2, port), - response: Some(br#"{"proofs":[{"merkle_tree_node_id":3,"usage":{"stored_bytes":2097152,"transferred_bytes":1048576,"number_of_puts":1,"number_of_gets":1},"path":["hFnZfjnS5bAzgm5tHcWTxuJa5waDcaiU7OhBRofylhQ="],"leafs":[{"record":{"id":"17Z3vSjjRm6mWN3Swpw3Cw==","upstream":{"request":{"requestId":"e9920157-6c6a-485e-9f5a-1685ea6d4ef5","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_PIECE","bucketId":"1","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"0","timestamp":"1727346880632","signature":{"algorithm":"ED_25519","signer":"iNw0F9UFjsS0UD4MEuoaCom+IA/piSJCPUM0AU+msO4=","value":"KPDnQH5KZZQ2hksJ8F/w3GHwWloAm1QKoLt+SuUNYt3HxsGrh3r3q77COiu0jrwQ7mEsp/FFJp4pDp2Y1j2sDA=="}}},"downstream":[{"request":{"requestId":"a5bcaa37-97a4-45d2-beb9-c11cc955fb78","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_MERKLE_TREE","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"0","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"ulpjaksvopDDRRfYnrccUg5spkoRpfZlDARbjgfL4Y/X4HZNUp2cL5qQMHUosREB6PSMXr9rQvXYGA9kmrUBDg=="}}},{"request":{"requestId":"8af9ba14-4c49-438c-957d-d1a108a58b85","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_SEGMENT","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"524288","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"CLdw3HaQWVWdDHeog2SZjiEA4NZN6PD8vyw58JuQI7gMDpDXLFslMOcI7p/uNEyeDfNoKTAgNZpWbNR4vSZ/AA=="}}},{"request":{"requestId":"b3dc8833-d5aa-4e33-9afa-54584da29cda","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_SEGMENT","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"524288","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"5XTnDU/85DqWWpMy1kGRVK6ZHe/EYDeg2p07UbFnIr6xLX7n50k9MslwuF8jMl2/QoBrPnndHdCd5ssqV90kDg=="}}}],"timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"8WWGHaL3n8+bkuYQhTua3l+i3W//XXhlnzCpQ7VJ/BmfXQPFGEjIZsXw0kKr4+VXh/kWAncF3VrvW9nEi6G2CQ=="}},"transferred_bytes":1048576,"stored_bytes":0},{"record":{"id":"8Rg6VlRrSE65NsCY02OnlA==","upstream":{"request":{"requestId":"aacf30c4-b2e9-4f37-826d-0016c280f39b","requestType":"REQUEST_TYPE_PUT","contentType":"CONTENT_TYPE_METADATA","bucketId":"0","pieceCid":"AAAAAAAAAAEBAh4gaLfPG3AA1QwNFQc3VvJYsMAINAN6mMkvo5vk5HP8g/0=","offset":"0","size":"385","timestamp":"1727346880673","signature":{"algorithm":"ED_25519","signer":"xHUfclv0KTLyCz1NjsLAdMrEBfKdlta130WiEBvB14s=","value":"yPZt7Fyfp1aiJL+hYOg5rRtPPTNDMZwgReX2RX4bWbP8+ivreh1cNvSwnM5ln0EFqxTn53iVQpZeMWXUSiJeCw=="}}},"downstream":[],"timestamp":"1727346880673","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"zX0aGW/FuhddMAtGvN4Gjf6P1JaFGasrwf5yCrQPFv4qUB1GyACynb1s1+Mv0zpMAGOtIOcwaemoPu4fnOByBA=="}},"transferred_bytes":1048576,"stored_bytes":1048576}]}]}"#.to_vec()), - sent: true, - ..Default::default() - }; - - let pending_request3 = PendingRequest { - method: "GET".to_string(), - uri: format!("http://{}:{}/activity/buckets/123229/challenge?eraId=5757773&nodeId=0x1f50f1455f60f5774564233d321a116ca45ae3188b2200999445706d04839d72&merkleTreeNodeId=0,2,1,3", host3, port), - response: Some(br#"{"proofs":[{"merkle_tree_node_id":3,"usage":{"stored_bytes":2097152,"transferred_bytes":1048576,"number_of_puts":1,"number_of_gets":1},"path":["hFnZfjnS5bAzgm5tHcWTxuJa5waDcaiU7OhBRofylhQ="],"leafs":[{"record":{"id":"17Z3vSjjRm6mWN3Swpw3Cw==","upstream":{"request":{"requestId":"e9920157-6c6a-485e-9f5a-1685ea6d4ef5","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_PIECE","bucketId":"1","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"0","timestamp":"1727346880632","signature":{"algorithm":"ED_25519","signer":"iNw0F9UFjsS0UD4MEuoaCom+IA/piSJCPUM0AU+msO4=","value":"KPDnQH5KZZQ2hksJ8F/w3GHwWloAm1QKoLt+SuUNYt3HxsGrh3r3q77COiu0jrwQ7mEsp/FFJp4pDp2Y1j2sDA=="}}},"downstream":[{"request":{"requestId":"a5bcaa37-97a4-45d2-beb9-c11cc955fb78","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_MERKLE_TREE","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"0","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"ulpjaksvopDDRRfYnrccUg5spkoRpfZlDARbjgfL4Y/X4HZNUp2cL5qQMHUosREB6PSMXr9rQvXYGA9kmrUBDg=="}}},{"request":{"requestId":"8af9ba14-4c49-438c-957d-d1a108a58b85","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_SEGMENT","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"524288","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"CLdw3HaQWVWdDHeog2SZjiEA4NZN6PD8vyw58JuQI7gMDpDXLFslMOcI7p/uNEyeDfNoKTAgNZpWbNR4vSZ/AA=="}}},{"request":{"requestId":"b3dc8833-d5aa-4e33-9afa-54584da29cda","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_SEGMENT","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"524288","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"5XTnDU/85DqWWpMy1kGRVK6ZHe/EYDeg2p07UbFnIr6xLX7n50k9MslwuF8jMl2/QoBrPnndHdCd5ssqV90kDg=="}}}],"timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"8WWGHaL3n8+bkuYQhTua3l+i3W//XXhlnzCpQ7VJ/BmfXQPFGEjIZsXw0kKr4+VXh/kWAncF3VrvW9nEi6G2CQ=="}},"transferred_bytes":1048576,"stored_bytes":0},{"record":{"id":"8Rg6VlRrSE65NsCY02OnlA==","upstream":{"request":{"requestId":"aacf30c4-b2e9-4f37-826d-0016c280f39b","requestType":"REQUEST_TYPE_PUT","contentType":"CONTENT_TYPE_METADATA","bucketId":"0","pieceCid":"AAAAAAAAAAEBAh4gaLfPG3AA1QwNFQc3VvJYsMAINAN6mMkvo5vk5HP8g/0=","offset":"0","size":"385","timestamp":"1727346880673","signature":{"algorithm":"ED_25519","signer":"xHUfclv0KTLyCz1NjsLAdMrEBfKdlta130WiEBvB14s=","value":"yPZt7Fyfp1aiJL+hYOg5rRtPPTNDMZwgReX2RX4bWbP8+ivreh1cNvSwnM5ln0EFqxTn53iVQpZeMWXUSiJeCw=="}}},"downstream":[],"timestamp":"1727346880673","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"zX0aGW/FuhddMAtGvN4Gjf6P1JaFGasrwf5yCrQPFv4qUB1GyACynb1s1+Mv0zpMAGOtIOcwaemoPu4fnOByBA=="}},"transferred_bytes":1048576,"stored_bytes":1048576}]}]}"#.to_vec()), - sent: true, - ..Default::default() - }; - - let pending_request4 = PendingRequest { - method: "GET".to_string(), - uri: format!("http://{}:{}/activity/buckets/123229/challenge?eraId=5757773&nodeId=0x1f50f1455f60f5774564233d321a116ca45ae3188b2200999445706d04839d72&merkleTreeNodeId=0,2,1,3", host4, port), - response: Some(br#"{"proofs":[{"merkle_tree_node_id":3,"usage":{"stored_bytes":2097152,"transferred_bytes":1048576,"number_of_puts":1,"number_of_gets":1},"path":["hFnZfjnS5bAzgm5tHcWTxuJa5waDcaiU7OhBRofylhQ="],"leafs":[{"record":{"id":"17Z3vSjjRm6mWN3Swpw3Cw==","upstream":{"request":{"requestId":"e9920157-6c6a-485e-9f5a-1685ea6d4ef5","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_PIECE","bucketId":"1","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"0","timestamp":"1727346880632","signature":{"algorithm":"ED_25519","signer":"iNw0F9UFjsS0UD4MEuoaCom+IA/piSJCPUM0AU+msO4=","value":"KPDnQH5KZZQ2hksJ8F/w3GHwWloAm1QKoLt+SuUNYt3HxsGrh3r3q77COiu0jrwQ7mEsp/FFJp4pDp2Y1j2sDA=="}}},"downstream":[{"request":{"requestId":"a5bcaa37-97a4-45d2-beb9-c11cc955fb78","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_MERKLE_TREE","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"0","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"ulpjaksvopDDRRfYnrccUg5spkoRpfZlDARbjgfL4Y/X4HZNUp2cL5qQMHUosREB6PSMXr9rQvXYGA9kmrUBDg=="}}},{"request":{"requestId":"8af9ba14-4c49-438c-957d-d1a108a58b85","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_SEGMENT","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"524288","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"CLdw3HaQWVWdDHeog2SZjiEA4NZN6PD8vyw58JuQI7gMDpDXLFslMOcI7p/uNEyeDfNoKTAgNZpWbNR4vSZ/AA=="}}},{"request":{"requestId":"b3dc8833-d5aa-4e33-9afa-54584da29cda","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_SEGMENT","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"524288","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"5XTnDU/85DqWWpMy1kGRVK6ZHe/EYDeg2p07UbFnIr6xLX7n50k9MslwuF8jMl2/QoBrPnndHdCd5ssqV90kDg=="}}}],"timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"8WWGHaL3n8+bkuYQhTua3l+i3W//XXhlnzCpQ7VJ/BmfXQPFGEjIZsXw0kKr4+VXh/kWAncF3VrvW9nEi6G2CQ=="}},"transferred_bytes":1048576,"stored_bytes":0},{"record":{"id":"8Rg6VlRrSE65NsCY02OnlA==","upstream":{"request":{"requestId":"aacf30c4-b2e9-4f37-826d-0016c280f39b","requestType":"REQUEST_TYPE_PUT","contentType":"CONTENT_TYPE_METADATA","bucketId":"0","pieceCid":"AAAAAAAAAAEBAh4gaLfPG3AA1QwNFQc3VvJYsMAINAN6mMkvo5vk5HP8g/0=","offset":"0","size":"385","timestamp":"1727346880673","signature":{"algorithm":"ED_25519","signer":"xHUfclv0KTLyCz1NjsLAdMrEBfKdlta130WiEBvB14s=","value":"yPZt7Fyfp1aiJL+hYOg5rRtPPTNDMZwgReX2RX4bWbP8+ivreh1cNvSwnM5ln0EFqxTn53iVQpZeMWXUSiJeCw=="}}},"downstream":[],"timestamp":"1727346880673","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"zX0aGW/FuhddMAtGvN4Gjf6P1JaFGasrwf5yCrQPFv4qUB1GyACynb1s1+Mv0zpMAGOtIOcwaemoPu4fnOByBA=="}},"transferred_bytes":1048576,"stored_bytes":1048576}]}]}"#.to_vec()), - sent: true, - ..Default::default() - }; - - let pending_request5 = PendingRequest { - method: "GET".to_string(), - uri: format!("http://{}:{}/activity/buckets/123229/challenge?eraId=5757773&nodeId=0x1f50f1455f60f5774564233d321a116ca45ae3188b2200999445706d04839d72&merkleTreeNodeId=1", host1, port), - response: Some(br#"{"proofs":[{"merkle_tree_node_id":3,"usage":{"stored_bytes":2097152,"transferred_bytes":1048576,"number_of_puts":1,"number_of_gets":1},"path":["hFnZfjnS5bAzgm5tHcWTxuJa5waDcaiU7OhBRofylhQ="],"leafs":[{"record":{"id":"17Z3vSjjRm6mWN3Swpw3Cw==","upstream":{"request":{"requestId":"e9920157-6c6a-485e-9f5a-1685ea6d4ef5","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_PIECE","bucketId":"1","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"0","timestamp":"1727346880632","signature":{"algorithm":"ED_25519","signer":"iNw0F9UFjsS0UD4MEuoaCom+IA/piSJCPUM0AU+msO4=","value":"KPDnQH5KZZQ2hksJ8F/w3GHwWloAm1QKoLt+SuUNYt3HxsGrh3r3q77COiu0jrwQ7mEsp/FFJp4pDp2Y1j2sDA=="}}},"downstream":[{"request":{"requestId":"a5bcaa37-97a4-45d2-beb9-c11cc955fb78","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_MERKLE_TREE","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"0","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"ulpjaksvopDDRRfYnrccUg5spkoRpfZlDARbjgfL4Y/X4HZNUp2cL5qQMHUosREB6PSMXr9rQvXYGA9kmrUBDg=="}}},{"request":{"requestId":"8af9ba14-4c49-438c-957d-d1a108a58b85","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_SEGMENT","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"524288","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"CLdw3HaQWVWdDHeog2SZjiEA4NZN6PD8vyw58JuQI7gMDpDXLFslMOcI7p/uNEyeDfNoKTAgNZpWbNR4vSZ/AA=="}}},{"request":{"requestId":"b3dc8833-d5aa-4e33-9afa-54584da29cda","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_SEGMENT","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"524288","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"5XTnDU/85DqWWpMy1kGRVK6ZHe/EYDeg2p07UbFnIr6xLX7n50k9MslwuF8jMl2/QoBrPnndHdCd5ssqV90kDg=="}}}],"timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"8WWGHaL3n8+bkuYQhTua3l+i3W//XXhlnzCpQ7VJ/BmfXQPFGEjIZsXw0kKr4+VXh/kWAncF3VrvW9nEi6G2CQ=="}},"transferred_bytes":1048576,"stored_bytes":0},{"record":{"id":"8Rg6VlRrSE65NsCY02OnlA==","upstream":{"request":{"requestId":"aacf30c4-b2e9-4f37-826d-0016c280f39b","requestType":"REQUEST_TYPE_PUT","contentType":"CONTENT_TYPE_METADATA","bucketId":"0","pieceCid":"AAAAAAAAAAEBAh4gaLfPG3AA1QwNFQc3VvJYsMAINAN6mMkvo5vk5HP8g/0=","offset":"0","size":"385","timestamp":"1727346880673","signature":{"algorithm":"ED_25519","signer":"xHUfclv0KTLyCz1NjsLAdMrEBfKdlta130WiEBvB14s=","value":"yPZt7Fyfp1aiJL+hYOg5rRtPPTNDMZwgReX2RX4bWbP8+ivreh1cNvSwnM5ln0EFqxTn53iVQpZeMWXUSiJeCw=="}}},"downstream":[],"timestamp":"1727346880673","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"zX0aGW/FuhddMAtGvN4Gjf6P1JaFGasrwf5yCrQPFv4qUB1GyACynb1s1+Mv0zpMAGOtIOcwaemoPu4fnOByBA=="}},"transferred_bytes":1048576,"stored_bytes":1048576}]}]}"#.to_vec()), - sent: true, - ..Default::default() - }; - - let pending_request6 = PendingRequest { - method: "GET".to_string(), - uri: format!("http://{}:{}/activity/buckets/123229/challenge?eraId=5757773&nodeId=0x1f50f1455f60f5774564233d321a116ca45ae3188b2200999445706d04839d72&merkleTreeNodeId=1", host2, port), - response: Some(br#"{"proofs":[{"merkle_tree_node_id":3,"usage":{"stored_bytes":2097152,"transferred_bytes":1048576,"number_of_puts":1,"number_of_gets":1},"path":["hFnZfjnS5bAzgm5tHcWTxuJa5waDcaiU7OhBRofylhQ="],"leafs":[{"record":{"id":"17Z3vSjjRm6mWN3Swpw3Cw==","upstream":{"request":{"requestId":"e9920157-6c6a-485e-9f5a-1685ea6d4ef5","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_PIECE","bucketId":"1","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"0","timestamp":"1727346880632","signature":{"algorithm":"ED_25519","signer":"iNw0F9UFjsS0UD4MEuoaCom+IA/piSJCPUM0AU+msO4=","value":"KPDnQH5KZZQ2hksJ8F/w3GHwWloAm1QKoLt+SuUNYt3HxsGrh3r3q77COiu0jrwQ7mEsp/FFJp4pDp2Y1j2sDA=="}}},"downstream":[{"request":{"requestId":"a5bcaa37-97a4-45d2-beb9-c11cc955fb78","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_MERKLE_TREE","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"0","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"ulpjaksvopDDRRfYnrccUg5spkoRpfZlDARbjgfL4Y/X4HZNUp2cL5qQMHUosREB6PSMXr9rQvXYGA9kmrUBDg=="}}},{"request":{"requestId":"8af9ba14-4c49-438c-957d-d1a108a58b85","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_SEGMENT","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"524288","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"CLdw3HaQWVWdDHeog2SZjiEA4NZN6PD8vyw58JuQI7gMDpDXLFslMOcI7p/uNEyeDfNoKTAgNZpWbNR4vSZ/AA=="}}},{"request":{"requestId":"b3dc8833-d5aa-4e33-9afa-54584da29cda","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_SEGMENT","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"524288","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"5XTnDU/85DqWWpMy1kGRVK6ZHe/EYDeg2p07UbFnIr6xLX7n50k9MslwuF8jMl2/QoBrPnndHdCd5ssqV90kDg=="}}}],"timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"8WWGHaL3n8+bkuYQhTua3l+i3W//XXhlnzCpQ7VJ/BmfXQPFGEjIZsXw0kKr4+VXh/kWAncF3VrvW9nEi6G2CQ=="}},"transferred_bytes":1048576,"stored_bytes":0},{"record":{"id":"8Rg6VlRrSE65NsCY02OnlA==","upstream":{"request":{"requestId":"aacf30c4-b2e9-4f37-826d-0016c280f39b","requestType":"REQUEST_TYPE_PUT","contentType":"CONTENT_TYPE_METADATA","bucketId":"0","pieceCid":"AAAAAAAAAAEBAh4gaLfPG3AA1QwNFQc3VvJYsMAINAN6mMkvo5vk5HP8g/0=","offset":"0","size":"385","timestamp":"1727346880673","signature":{"algorithm":"ED_25519","signer":"xHUfclv0KTLyCz1NjsLAdMrEBfKdlta130WiEBvB14s=","value":"yPZt7Fyfp1aiJL+hYOg5rRtPPTNDMZwgReX2RX4bWbP8+ivreh1cNvSwnM5ln0EFqxTn53iVQpZeMWXUSiJeCw=="}}},"downstream":[],"timestamp":"1727346880673","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"zX0aGW/FuhddMAtGvN4Gjf6P1JaFGasrwf5yCrQPFv4qUB1GyACynb1s1+Mv0zpMAGOtIOcwaemoPu4fnOByBA=="}},"transferred_bytes":1048576,"stored_bytes":1048576}]}]}"#.to_vec()), - sent: true, - ..Default::default() - }; - - let pending_request7 = PendingRequest { - method: "GET".to_string(), - uri: format!("http://{}:{}/activity/buckets/123229/challenge?eraId=5757773&nodeId=0x1f50f1455f60f5774564233d321a116ca45ae3188b2200999445706d04839d72&merkleTreeNodeId=1", host3, port), - response: Some(br#"{"proofs":[{"merkle_tree_node_id":3,"usage":{"stored_bytes":2097152,"transferred_bytes":1048576,"number_of_puts":1,"number_of_gets":1},"path":["hFnZfjnS5bAzgm5tHcWTxuJa5waDcaiU7OhBRofylhQ="],"leafs":[{"record":{"id":"17Z3vSjjRm6mWN3Swpw3Cw==","upstream":{"request":{"requestId":"e9920157-6c6a-485e-9f5a-1685ea6d4ef5","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_PIECE","bucketId":"1","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"0","timestamp":"1727346880632","signature":{"algorithm":"ED_25519","signer":"iNw0F9UFjsS0UD4MEuoaCom+IA/piSJCPUM0AU+msO4=","value":"KPDnQH5KZZQ2hksJ8F/w3GHwWloAm1QKoLt+SuUNYt3HxsGrh3r3q77COiu0jrwQ7mEsp/FFJp4pDp2Y1j2sDA=="}}},"downstream":[{"request":{"requestId":"a5bcaa37-97a4-45d2-beb9-c11cc955fb78","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_MERKLE_TREE","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"0","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"ulpjaksvopDDRRfYnrccUg5spkoRpfZlDARbjgfL4Y/X4HZNUp2cL5qQMHUosREB6PSMXr9rQvXYGA9kmrUBDg=="}}},{"request":{"requestId":"8af9ba14-4c49-438c-957d-d1a108a58b85","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_SEGMENT","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"524288","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"CLdw3HaQWVWdDHeog2SZjiEA4NZN6PD8vyw58JuQI7gMDpDXLFslMOcI7p/uNEyeDfNoKTAgNZpWbNR4vSZ/AA=="}}},{"request":{"requestId":"b3dc8833-d5aa-4e33-9afa-54584da29cda","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_SEGMENT","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"524288","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"5XTnDU/85DqWWpMy1kGRVK6ZHe/EYDeg2p07UbFnIr6xLX7n50k9MslwuF8jMl2/QoBrPnndHdCd5ssqV90kDg=="}}}],"timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"8WWGHaL3n8+bkuYQhTua3l+i3W//XXhlnzCpQ7VJ/BmfXQPFGEjIZsXw0kKr4+VXh/kWAncF3VrvW9nEi6G2CQ=="}},"transferred_bytes":1048576,"stored_bytes":0},{"record":{"id":"8Rg6VlRrSE65NsCY02OnlA==","upstream":{"request":{"requestId":"aacf30c4-b2e9-4f37-826d-0016c280f39b","requestType":"REQUEST_TYPE_PUT","contentType":"CONTENT_TYPE_METADATA","bucketId":"0","pieceCid":"AAAAAAAAAAEBAh4gaLfPG3AA1QwNFQc3VvJYsMAINAN6mMkvo5vk5HP8g/0=","offset":"0","size":"385","timestamp":"1727346880673","signature":{"algorithm":"ED_25519","signer":"xHUfclv0KTLyCz1NjsLAdMrEBfKdlta130WiEBvB14s=","value":"yPZt7Fyfp1aiJL+hYOg5rRtPPTNDMZwgReX2RX4bWbP8+ivreh1cNvSwnM5ln0EFqxTn53iVQpZeMWXUSiJeCw=="}}},"downstream":[],"timestamp":"1727346880673","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"zX0aGW/FuhddMAtGvN4Gjf6P1JaFGasrwf5yCrQPFv4qUB1GyACynb1s1+Mv0zpMAGOtIOcwaemoPu4fnOByBA=="}},"transferred_bytes":1048576,"stored_bytes":1048576}]}]}"#.to_vec()), - sent: true, - ..Default::default() - }; - - let pending_request8 = PendingRequest { - method: "GET".to_string(), - uri: format!("http://{}:{}/activity/buckets/123229/challenge?eraId=5757773&nodeId=0x1f50f1455f60f5774564233d321a116ca45ae3188b2200999445706d04839d72&merkleTreeNodeId=1", host4, port), - response: Some(br#"{"proofs":[{"merkle_tree_node_id":3,"usage":{"stored_bytes":2097152,"transferred_bytes":1048576,"number_of_puts":1,"number_of_gets":1},"path":["hFnZfjnS5bAzgm5tHcWTxuJa5waDcaiU7OhBRofylhQ="],"leafs":[{"record":{"id":"17Z3vSjjRm6mWN3Swpw3Cw==","upstream":{"request":{"requestId":"e9920157-6c6a-485e-9f5a-1685ea6d4ef5","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_PIECE","bucketId":"1","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"0","timestamp":"1727346880632","signature":{"algorithm":"ED_25519","signer":"iNw0F9UFjsS0UD4MEuoaCom+IA/piSJCPUM0AU+msO4=","value":"KPDnQH5KZZQ2hksJ8F/w3GHwWloAm1QKoLt+SuUNYt3HxsGrh3r3q77COiu0jrwQ7mEsp/FFJp4pDp2Y1j2sDA=="}}},"downstream":[{"request":{"requestId":"a5bcaa37-97a4-45d2-beb9-c11cc955fb78","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_MERKLE_TREE","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"0","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"ulpjaksvopDDRRfYnrccUg5spkoRpfZlDARbjgfL4Y/X4HZNUp2cL5qQMHUosREB6PSMXr9rQvXYGA9kmrUBDg=="}}},{"request":{"requestId":"8af9ba14-4c49-438c-957d-d1a108a58b85","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_SEGMENT","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"524288","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"CLdw3HaQWVWdDHeog2SZjiEA4NZN6PD8vyw58JuQI7gMDpDXLFslMOcI7p/uNEyeDfNoKTAgNZpWbNR4vSZ/AA=="}}},{"request":{"requestId":"b3dc8833-d5aa-4e33-9afa-54584da29cda","requestType":"REQUEST_TYPE_GET","contentType":"CONTENT_TYPE_SEGMENT","bucketId":"0","pieceCid":"AQIeIKLbs3OibO5qbLJ/PLCo1m02oFHWCl4s7S59GWgxDUbk","offset":"0","size":"524288","timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"5XTnDU/85DqWWpMy1kGRVK6ZHe/EYDeg2p07UbFnIr6xLX7n50k9MslwuF8jMl2/QoBrPnndHdCd5ssqV90kDg=="}}}],"timestamp":"1727346880633","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"8WWGHaL3n8+bkuYQhTua3l+i3W//XXhlnzCpQ7VJ/BmfXQPFGEjIZsXw0kKr4+VXh/kWAncF3VrvW9nEi6G2CQ=="}},"transferred_bytes":1048576,"stored_bytes":0},{"record":{"id":"8Rg6VlRrSE65NsCY02OnlA==","upstream":{"request":{"requestId":"aacf30c4-b2e9-4f37-826d-0016c280f39b","requestType":"REQUEST_TYPE_PUT","contentType":"CONTENT_TYPE_METADATA","bucketId":"0","pieceCid":"AAAAAAAAAAEBAh4gaLfPG3AA1QwNFQc3VvJYsMAINAN6mMkvo5vk5HP8g/0=","offset":"0","size":"385","timestamp":"1727346880673","signature":{"algorithm":"ED_25519","signer":"xHUfclv0KTLyCz1NjsLAdMrEBfKdlta130WiEBvB14s=","value":"yPZt7Fyfp1aiJL+hYOg5rRtPPTNDMZwgReX2RX4bWbP8+ivreh1cNvSwnM5ln0EFqxTn53iVQpZeMWXUSiJeCw=="}}},"downstream":[],"timestamp":"1727346880673","signature":{"algorithm":"ED_25519","signer":"CsfLnFNZTp9TjZlQxrzyjwwMe4OF3uouviQGK8ZA574=","value":"zX0aGW/FuhddMAtGvN4Gjf6P1JaFGasrwf5yCrQPFv4qUB1GyACynb1s1+Mv0zpMAGOtIOcwaemoPu4fnOByBA=="}},"transferred_bytes":1048576,"stored_bytes":1048576}]}]}"#.to_vec()), + uri: format!("http://{}:{}/activity/buckets/123229/traverse?eraId=5757773&nodeId=0x1f50f1455f60f5774564233d321a116ca45ae3188b2200999445706d04839d72&merkleTreeNodeId=1&levels=1", host1, port), + response: Some(br#"[{"merkle_tree_node_id":2,"hash":"hkujtYgWP21CrXdRP1rhRPrYR2ooIYCnP5zwCERTePI=","stored_bytes":20913291,"transferred_bytes":20913291,"number_of_puts":61,"number_of_gets":3},{"merkle_tree_node_id":3,"hash":"ZgWwK2LgWkHpx5JlXZn/Rouq6uE9DhOnRH6EA1+QO6o=","stored_bytes":23778084,"transferred_bytes":23778084,"number_of_puts":46,"number_of_gets":2}]"#.to_vec()), sent: true, ..Default::default() }; offchain_state.expect_request(pending_request1); offchain_state.expect_request(pending_request2); - offchain_state.expect_request(pending_request3); - offchain_state.expect_request(pending_request4); - offchain_state.expect_request(pending_request5); - offchain_state.expect_request(pending_request6); - offchain_state.expect_request(pending_request7); - offchain_state.expect_request(pending_request8); + drop(offchain_state); let cluster_id = ClusterId::from([1; 20]); let era_id = 5757773; + let host1 = "178.251.228.165"; + + let port = 8080; let node_params1 = StorageNodeParams { ssl: false, host: host1.as_bytes().to_vec(), @@ -2530,56 +2915,22 @@ fn test_challenge_sub_aggregates_not_in_consensus() { domain: b"example2.com".to_vec(), }; - let node_params2 = StorageNodeParams { - ssl: false, - host: host2.as_bytes().to_vec(), - http_port: port, - mode: StorageNodeMode::DAC, - p2p_port: 5555, - grpc_port: 4444, - domain: b"example3.com".to_vec(), - }; - - let node_params3 = StorageNodeParams { - ssl: false, - host: host3.as_bytes().to_vec(), - http_port: port, - mode: StorageNodeMode::DAC, - p2p_port: 5555, - grpc_port: 4444, - domain: b"example4.com".to_vec(), - }; - - let node_params4 = StorageNodeParams { - ssl: false, - host: host4.as_bytes().to_vec(), - http_port: port, - mode: StorageNodeMode::DAC, - p2p_port: 5555, - grpc_port: 4444, - domain: b"example5.com".to_vec(), + let deffective_bucket_sub_aggregate = BucketSubAggregate { + bucket_id: 123229, + node_id: "0x1f50f1455f60f5774564233d321a116ca45ae3188b2200999445706d04839d72" + .to_string(), + stored_bytes: 0, + transferred_bytes: 25143977, + number_of_puts: 0, + number_of_gets: 10, + aggregator: AggregatorInfo { + node_pub_key: NodePubKey::StoragePubKey(AccountId32::new([0; 32])), + node_params: node_params1.clone(), + }, }; - let dac_nodes: Vec<(NodePubKey, StorageNodeParams)> = vec![ - (NodePubKey::StoragePubKey(StorageNodePubKey::new([1; 32])), node_params1), - (NodePubKey::StoragePubKey(StorageNodePubKey::new([2; 32])), node_params2), - (NodePubKey::StoragePubKey(StorageNodePubKey::new([3; 32])), node_params3), - (NodePubKey::StoragePubKey(StorageNodePubKey::new([4; 32])), node_params4), - ]; - - let bucket_node_aggregates_not_in_consensus: Vec = - vec![BucketNodeAggregatesActivity { - bucket_id: 123229, - node_id: "0x1f50f1455f60f5774564233d321a116ca45ae3188b2200999445706d04839d72" - .to_string(), - stored_bytes: 0, - transferred_bytes: 25143977, - number_of_puts: 0, - number_of_gets: 10, - }]; - let result = - DdcVerification::challenge_and_find_valid_sub_aggregates_not_in_consensus(&cluster_id, era_id, &dac_nodes, bucket_node_aggregates_not_in_consensus); + DdcVerification::_challenge_aggregate(&cluster_id, era_id, &deffective_bucket_sub_aggregate); assert!(result.is_ok()); diff --git a/primitives/src/lib.rs b/primitives/src/lib.rs index 07cdb12a2..894bdeb40 100644 --- a/primitives/src/lib.rs +++ b/primitives/src/lib.rs @@ -108,7 +108,17 @@ pub struct ClusterBondingParams { pub storage_unbonding_delay: BlockNumber, } -#[derive(Clone, Encode, Decode, RuntimeDebug, TypeInfo, PartialEq, Serialize, Deserialize)] +#[derive( + Debug, Serialize, Deserialize, Clone, Ord, PartialOrd, PartialEq, Eq, Encode, Decode, TypeInfo, +)] +pub struct AggregatorInfo { + pub node_pub_key: NodePubKey, + pub node_params: StorageNodeParams, +} + +#[derive( + Debug, Serialize, Deserialize, Clone, Ord, PartialOrd, PartialEq, Eq, Encode, Decode, TypeInfo, +)] pub enum NodePubKey { StoragePubKey(StorageNodePubKey), } @@ -136,7 +146,20 @@ impl TryFrom for NodeType { } } -#[derive(Clone, Encode, Decode, RuntimeDebug, TypeInfo, PartialEq, Serialize, Deserialize)] +#[derive( + Debug, + Serialize, + Deserialize, + Clone, + Hash, + Ord, + PartialOrd, + PartialEq, + Eq, + Encode, + Decode, + TypeInfo, +)] pub enum StorageNodeMode { /// DDC Storage node operates with enabled caching in RAM and stores data in Hard Drive Full = 1, @@ -148,8 +171,20 @@ pub enum StorageNodeMode { DAC = 4, } -#[cfg_attr(feature = "std", derive(Serialize, Deserialize))] -#[derive(Clone, Encode, Decode, RuntimeDebug, TypeInfo, PartialEq)] +#[derive( + Debug, + Serialize, + Deserialize, + Clone, + Hash, + Ord, + PartialOrd, + PartialEq, + Eq, + Encode, + Decode, + TypeInfo, +)] pub struct StorageNodeParams { pub mode: StorageNodeMode, pub host: Vec, diff --git a/runtime/cere-dev/src/lib.rs b/runtime/cere-dev/src/lib.rs index b2a7278ac..e7ed99483 100644 --- a/runtime/cere-dev/src/lib.rs +++ b/runtime/cere-dev/src/lib.rs @@ -149,7 +149,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // and set impl_version to 0. If only runtime // implementation changes and behavior does not, then leave spec_version as // is and increment impl_version. - spec_version: 54128, + spec_version: 54129, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 19, @@ -1294,6 +1294,7 @@ impl, T: frame_system::Config> GetDdcOrigin parameter_types! { pub const VerificationPalletId: PalletId = PalletId(*b"verifypa"); + pub const MajorityOfAggregators: Percent = Percent::from_percent(67); } impl pallet_ddc_verification::Config for Runtime { type RuntimeEvent = RuntimeEvent; @@ -1308,7 +1309,8 @@ impl pallet_ddc_verification::Config for Runtime { type ActivityHasher = BlakeTwo256; const MAJORITY: u8 = 67; const BLOCK_TO_START: u16 = 30; // every 100 blocks - const MIN_DAC_NODES_FOR_CONSENSUS: u16 = 3; + const DAC_REDUNDANCY_FACTOR: u16 = 3; + type AggregatorsQuorum = MajorityOfAggregators; const MAX_PAYOUT_BATCH_SIZE: u16 = MAX_PAYOUT_BATCH_SIZE; const MAX_PAYOUT_BATCH_COUNT: u16 = MAX_PAYOUT_BATCH_COUNT; type ActivityHash = H256; diff --git a/runtime/cere/src/lib.rs b/runtime/cere/src/lib.rs index 8af9b762f..7f74445ce 100644 --- a/runtime/cere/src/lib.rs +++ b/runtime/cere/src/lib.rs @@ -143,7 +143,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { // and set impl_version to 0. If only runtime // implementation changes and behavior does not, then leave spec_version as // is and increment impl_version. - spec_version: 54128, + spec_version: 54129, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 19, @@ -1301,6 +1301,7 @@ impl, T: frame_system::Config> GetDdcOrigin parameter_types! { pub const VerificationPalletId: PalletId = PalletId(*b"verifypa"); + pub const MajorityOfAggregators: Percent = Percent::from_percent(67); } impl pallet_ddc_verification::Config for Runtime { @@ -1316,7 +1317,8 @@ impl pallet_ddc_verification::Config for Runtime { type ActivityHasher = BlakeTwo256; const MAJORITY: u8 = 67; const BLOCK_TO_START: u16 = 100; // every 100 blocks - const MIN_DAC_NODES_FOR_CONSENSUS: u16 = 3; + const DAC_REDUNDANCY_FACTOR: u16 = 3; + type AggregatorsQuorum = MajorityOfAggregators; const MAX_PAYOUT_BATCH_SIZE: u16 = MAX_PAYOUT_BATCH_SIZE; const MAX_PAYOUT_BATCH_COUNT: u16 = MAX_PAYOUT_BATCH_COUNT; type ActivityHash = H256;