From ee8345401f4b54b58785aba88588a998d0ca6d93 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Tue, 28 Feb 2023 18:08:31 +0400 Subject: [PATCH 01/59] feat(db): add ss58 migration for account table --- backend/package.json | 2 + db/migrations/20230227122704-ss58-prefix.js | 48 +++++ .../sqls/20230227122704-ss58-prefix-down.sql | 0 .../sqls/20230227122704-ss58-prefix-up.sql | 0 yarn.lock | 167 ++++++++++++++++++ 5 files changed, 217 insertions(+) create mode 100644 db/migrations/20230227122704-ss58-prefix.js create mode 100644 db/migrations/sqls/20230227122704-ss58-prefix-down.sql create mode 100644 db/migrations/sqls/20230227122704-ss58-prefix-up.sql diff --git a/backend/package.json b/backend/package.json index 3f095c88..c42db65e 100644 --- a/backend/package.json +++ b/backend/package.json @@ -27,6 +27,8 @@ "@cere-ddc-sdk/smart-contract": "0.5.0", "@polkadot/api": "8.2.1", "@polkadot/api-contract": "8.2.1", + "@polkadot/keyring": "^10.4.2", + "@polkadot/util-crypto": "^10.4.2", "axios": "^0.21.1", "bignumber.js": "^9.0.1", "db-migrate": "^0.11.13", diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js new file mode 100644 index 00000000..28911505 --- /dev/null +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -0,0 +1,48 @@ +'use strict'; + +var dbm; +var type; +var seed; +var keyring = require('@polkadot/keyring'); +var Promise; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function (options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; + Promise = options.Promise; +}; + +const convertAccountTable = (db, ss58Format) => { + return db.runSql('SELECT * from account;', (_, result) => { + + result.rows.forEach((row) => { + const currentAccountId = row.account_id; + + const nextAddress = keyring.encodeAddress( + keyring.decodeAddress(currentAccountId), + ss58Format); + + const queryString = `UPDATE account SET account_id='${nextAddress}' WHERE account_id='${currentAccountId}';` + + db.runSql(queryString); + }); + + }); +} + +exports.up = function (db) { + return convertAccountTable(db, 54); +}; + +exports.down = function (db) { + return convertAccountTable(db, 42); +}; + +exports._meta = { + "version": 1 +}; diff --git a/db/migrations/sqls/20230227122704-ss58-prefix-down.sql b/db/migrations/sqls/20230227122704-ss58-prefix-down.sql new file mode 100644 index 00000000..e69de29b diff --git a/db/migrations/sqls/20230227122704-ss58-prefix-up.sql b/db/migrations/sqls/20230227122704-ss58-prefix-up.sql new file mode 100644 index 00000000..e69de29b diff --git a/yarn.lock b/yarn.lock index a3f7f78a..3dd630b7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1224,6 +1224,13 @@ dependencies: regenerator-runtime "^0.13.10" +"@babel/runtime@^7.20.13", "@babel/runtime@^7.20.6": + version "7.21.0" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.21.0.tgz#5b55c9d394e5fcf304909a8b00c07dc217b56673" + integrity sha512-xwII0//EObnq89Ji5AKYQaRYiW/nZ3llSv29d49IuxPhKbtJoLP+9QUUZ4nVragQVtaVGeZrpB+ZtG/Pdy/POw== + dependencies: + regenerator-runtime "^0.13.11" + "@babel/template@^7.14.5": version "7.14.5" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.14.5.tgz#a9bc9d8b33354ff6e55a9c60d1109200a68974f4" @@ -2289,6 +2296,11 @@ resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-1.1.2.tgz#e9e035b9b166ca0af657a7848eb2718f0f22f183" integrity sha512-KYRCASVTv6aeUi1tsF8/vpyR7zpfs3FUzy2Jqm+MU+LmUKhQ0y2FpfwqkCcxSg2ua4GALJd8k2R76WxwZGbQpA== +"@noble/hashes@1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-1.2.0.tgz#a3150eeb09cc7ab207ebf6d7b9ad311a9bdbed12" + integrity sha512-FZfhjEDbT5GRswV3C6uvLPHMiVD6lQBmpoX5+eSiPaMTXte/IKqI5dykDxzZB/WBeK/CDuQRBWarPdi3FNY2zQ== + "@noble/hashes@^0.4.5": version "0.4.5" resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-0.4.5.tgz#f69a963b0c59c1145bc5aca1f3eef58a48bf9a59" @@ -2299,6 +2311,11 @@ resolved "https://registry.yarnpkg.com/@noble/secp256k1/-/secp256k1-1.6.0.tgz#602afbbfcfb7e169210469b697365ef740d7e930" integrity sha512-DWSsg8zMHOYMYBqIQi96BQuthZrp98LCeMNcUOaffCIVYQ5yxDbNikLF+H7jEnmNNmXbtVic46iCuVWzar+MgA== +"@noble/secp256k1@1.7.1": + version "1.7.1" + resolved "https://registry.yarnpkg.com/@noble/secp256k1/-/secp256k1-1.7.1.tgz#b251c70f824ce3ca7f8dc3df08d58f005cc0507c" + integrity sha512-hOUk6AyBFmqVrv7k5WAw/LpszxVbj9gGN4JRkIX52fdFAj1UA61KXmZDvqVEm+pOyec3+fIeZB02LYa/pWOArw== + "@noble/secp256k1@^1.3.4": version "1.7.0" resolved "https://registry.yarnpkg.com/@noble/secp256k1/-/secp256k1-1.7.0.tgz#d15357f7c227e751d90aa06b05a0e5cf993ba8c1" @@ -3034,6 +3051,15 @@ "@polkadot/util" "8.2.2" "@polkadot/util-crypto" "8.2.2" +"@polkadot/keyring@^10.4.2": + version "10.4.2" + resolved "https://registry.yarnpkg.com/@polkadot/keyring/-/keyring-10.4.2.tgz#793377fdb9076df0af771df11388faa6be03c70d" + integrity sha512-7iHhJuXaHrRTG6cJDbZE9G+c1ts1dujp0qbO4RfAPmT7YUvphHvAtCKueN9UKPz5+TYDL+rP/jDEaSKU8jl/qQ== + dependencies: + "@babel/runtime" "^7.20.13" + "@polkadot/util" "10.4.2" + "@polkadot/util-crypto" "10.4.2" + "@polkadot/keyring@^6.0.5": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/keyring/-/keyring-6.11.1.tgz#2510c349c965c74cc2f108f114f1048856940604" @@ -3064,6 +3090,15 @@ "@polkadot/util-crypto" "^6.0.5" bn.js "^4.11.9" +"@polkadot/networks@10.4.2": + version "10.4.2" + resolved "https://registry.yarnpkg.com/@polkadot/networks/-/networks-10.4.2.tgz#d7878c6aad8173c800a21140bfe5459261724456" + integrity sha512-FAh/znrEvWBiA/LbcT5GXHsCFUl//y9KqxLghSr/CreAmAergiJNT0MVUezC7Y36nkATgmsr4ylFwIxhVtuuCw== + dependencies: + "@babel/runtime" "^7.20.13" + "@polkadot/util" "10.4.2" + "@substrate/ss58-registry" "^1.38.0" + "@polkadot/networks@6.11.1", "@polkadot/networks@^6.0.5": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/networks/-/networks-6.11.1.tgz#8fd189593f6ee4f8bf64378d0aaae09e39a37d35" @@ -3256,6 +3291,23 @@ "@babel/runtime" "^7.15.3" color "^3.2.1" +"@polkadot/util-crypto@10.4.2", "@polkadot/util-crypto@^10.4.2": + version "10.4.2" + resolved "https://registry.yarnpkg.com/@polkadot/util-crypto/-/util-crypto-10.4.2.tgz#871fb69c65768bd48c57bb5c1f76a85d979fb8b5" + integrity sha512-RxZvF7C4+EF3fzQv8hZOLrYCBq5+wA+2LWv98nECkroChY3C2ZZvyWDqn8+aonNULt4dCVTWDZM0QIY6y4LUAQ== + dependencies: + "@babel/runtime" "^7.20.13" + "@noble/hashes" "1.2.0" + "@noble/secp256k1" "1.7.1" + "@polkadot/networks" "10.4.2" + "@polkadot/util" "10.4.2" + "@polkadot/wasm-crypto" "^6.4.1" + "@polkadot/x-bigint" "10.4.2" + "@polkadot/x-randomvalues" "10.4.2" + "@scure/base" "1.1.1" + ed2curve "^0.3.0" + tweetnacl "^1.0.3" + "@polkadot/util-crypto@6.11.1", "@polkadot/util-crypto@^6.0.5": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/util-crypto/-/util-crypto-6.11.1.tgz#7a36acf5c8bf52541609ec0b0b2a69af295d652e" @@ -3335,6 +3387,19 @@ tweetnacl "^1.0.3" xxhashjs "^0.2.2" +"@polkadot/util@10.4.2": + version "10.4.2" + resolved "https://registry.yarnpkg.com/@polkadot/util/-/util-10.4.2.tgz#df41805cb27f46b2b4dad24c371fa2a68761baa1" + integrity sha512-0r5MGICYiaCdWnx+7Axlpvzisy/bi1wZGXgCSw5+ZTyPTOqvsYRqM2X879yxvMsGfibxzWqNzaiVjToz1jvUaA== + dependencies: + "@babel/runtime" "^7.20.13" + "@polkadot/x-bigint" "10.4.2" + "@polkadot/x-global" "10.4.2" + "@polkadot/x-textdecoder" "10.4.2" + "@polkadot/x-textencoder" "10.4.2" + "@types/bn.js" "^5.1.1" + bn.js "^5.2.1" + "@polkadot/util@6.11.1", "@polkadot/util@^6.0.5": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/util/-/util-6.11.1.tgz#8950b038ba3e6ebfc0a7ff47feeb972e81b2626c" @@ -3407,6 +3472,13 @@ dependencies: "@babel/runtime" "^7.18.9" +"@polkadot/wasm-bridge@6.4.1": + version "6.4.1" + resolved "https://registry.yarnpkg.com/@polkadot/wasm-bridge/-/wasm-bridge-6.4.1.tgz#e97915dd67ba543ec3381299c2a5b9330686e27e" + integrity sha512-QZDvz6dsUlbYsaMV5biZgZWkYH9BC5AfhT0f0/knv8+LrbAoQdP3Asbvddw8vyU9sbpuCHXrd4bDLBwUCRfrBQ== + dependencies: + "@babel/runtime" "^7.20.6" + "@polkadot/wasm-crypto-asmjs@6.3.1": version "6.3.1" resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-asmjs/-/wasm-crypto-asmjs-6.3.1.tgz#e8f469c9cf4a7709c8131a96f857291953f3e30a" @@ -3414,6 +3486,13 @@ dependencies: "@babel/runtime" "^7.18.9" +"@polkadot/wasm-crypto-asmjs@6.4.1": + version "6.4.1" + resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-asmjs/-/wasm-crypto-asmjs-6.4.1.tgz#3cc76bbda5ea4a7a860982c64f9565907b312253" + integrity sha512-UxZTwuBZlnODGIQdCsE2Sn/jU0O2xrNQ/TkhRFELfkZXEXTNu4lw6NpaKq7Iey4L+wKd8h4lT3VPVkMcPBLOvA== + dependencies: + "@babel/runtime" "^7.20.6" + "@polkadot/wasm-crypto-asmjs@^4.2.1": version "4.2.1" resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-asmjs/-/wasm-crypto-asmjs-4.2.1.tgz#6b7eae1c011709f8042dfd30872a5fc5e9e021c0" @@ -3438,6 +3517,16 @@ "@polkadot/wasm-crypto-asmjs" "6.3.1" "@polkadot/wasm-crypto-wasm" "6.3.1" +"@polkadot/wasm-crypto-init@6.4.1": + version "6.4.1" + resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-init/-/wasm-crypto-init-6.4.1.tgz#4d9ab0030db52cf177bf707ef8e77aa4ca721668" + integrity sha512-1ALagSi/nfkyFaH6JDYfy/QbicVbSn99K8PV9rctDUfxc7P06R7CoqbjGQ4OMPX6w1WYVPU7B4jPHGLYBlVuMw== + dependencies: + "@babel/runtime" "^7.20.6" + "@polkadot/wasm-bridge" "6.4.1" + "@polkadot/wasm-crypto-asmjs" "6.4.1" + "@polkadot/wasm-crypto-wasm" "6.4.1" + "@polkadot/wasm-crypto-wasm@6.3.1": version "6.3.1" resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-wasm/-/wasm-crypto-wasm-6.3.1.tgz#67f720e7f9694fef096abe9d60abbac02e032383" @@ -3446,6 +3535,14 @@ "@babel/runtime" "^7.18.9" "@polkadot/wasm-util" "6.3.1" +"@polkadot/wasm-crypto-wasm@6.4.1": + version "6.4.1" + resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-wasm/-/wasm-crypto-wasm-6.4.1.tgz#97180f80583b18f6a13c1054fa5f7e8da40b1028" + integrity sha512-3VV9ZGzh0ZY3SmkkSw+0TRXxIpiO0nB8lFwlRgcwaCihwrvLfRnH9GI8WE12mKsHVjWTEVR3ogzILJxccAUjDA== + dependencies: + "@babel/runtime" "^7.20.6" + "@polkadot/wasm-util" "6.4.1" + "@polkadot/wasm-crypto-wasm@^4.2.1": version "4.2.1" resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-wasm/-/wasm-crypto-wasm-4.2.1.tgz#2a86f9b405e7195c3f523798c6ce4afffd19737e" @@ -3490,6 +3587,18 @@ "@polkadot/wasm-crypto-wasm" "6.3.1" "@polkadot/wasm-util" "6.3.1" +"@polkadot/wasm-crypto@^6.4.1": + version "6.4.1" + resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto/-/wasm-crypto-6.4.1.tgz#79310e23ad1ca62362ba893db6a8567154c2536a" + integrity sha512-FH+dcDPdhSLJvwL0pMLtn/LIPd62QDPODZRCmDyw+pFjLOMaRBc7raomWUOqyRWJTnqVf/iscc2rLVLNMyt7ag== + dependencies: + "@babel/runtime" "^7.20.6" + "@polkadot/wasm-bridge" "6.4.1" + "@polkadot/wasm-crypto-asmjs" "6.4.1" + "@polkadot/wasm-crypto-init" "6.4.1" + "@polkadot/wasm-crypto-wasm" "6.4.1" + "@polkadot/wasm-util" "6.4.1" + "@polkadot/wasm-util@6.3.1": version "6.3.1" resolved "https://registry.yarnpkg.com/@polkadot/wasm-util/-/wasm-util-6.3.1.tgz#439ebb68a436317af388ed6438b8f879df3afcda" @@ -3497,6 +3606,21 @@ dependencies: "@babel/runtime" "^7.18.9" +"@polkadot/wasm-util@6.4.1": + version "6.4.1" + resolved "https://registry.yarnpkg.com/@polkadot/wasm-util/-/wasm-util-6.4.1.tgz#74aecc85bec427a9225d9874685944ea3dc3ab76" + integrity sha512-Uwo+WpEsDmFExWC5kTNvsVhvqXMZEKf4gUHXFn4c6Xz4lmieRT5g+1bO1KJ21pl4msuIgdV3Bksfs/oiqMFqlw== + dependencies: + "@babel/runtime" "^7.20.6" + +"@polkadot/x-bigint@10.4.2": + version "10.4.2" + resolved "https://registry.yarnpkg.com/@polkadot/x-bigint/-/x-bigint-10.4.2.tgz#7eb2ec732259df48b5a00f07879a1331e05606ec" + integrity sha512-awRiox+/XSReLzimAU94fPldowiwnnMUkQJe8AebYhNocAj6SJU00GNoj6j6tAho6yleOwrTJXZaWFBaQVJQNg== + dependencies: + "@babel/runtime" "^7.20.13" + "@polkadot/x-global" "10.4.2" + "@polkadot/x-bigint@8.2.2": version "8.2.2" resolved "https://registry.yarnpkg.com/@polkadot/x-bigint/-/x-bigint-8.2.2.tgz#24f4e1b889a1ee899ac9c98fee526e44e2130c14" @@ -3533,6 +3657,13 @@ "@types/node-fetch" "^2.6.2" node-fetch "^2.6.7" +"@polkadot/x-global@10.4.2": + version "10.4.2" + resolved "https://registry.yarnpkg.com/@polkadot/x-global/-/x-global-10.4.2.tgz#5662366e3deda0b4c8f024b2d902fa838f9e60a4" + integrity sha512-g6GXHD/ykZvHap3M6wh19dO70Zm43l4jEhlxf5LtTo5/0/UporFCXr2YJYZqfbn9JbQwl1AU+NroYio+vtJdiA== + dependencies: + "@babel/runtime" "^7.20.13" + "@polkadot/x-global@6.11.1", "@polkadot/x-global@^6.0.5": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/x-global/-/x-global-6.11.1.tgz#c292b3825fea60e9b33fff1790323fc57de1ca5d" @@ -3561,6 +3692,14 @@ dependencies: "@babel/runtime" "^7.18.6" +"@polkadot/x-randomvalues@10.4.2": + version "10.4.2" + resolved "https://registry.yarnpkg.com/@polkadot/x-randomvalues/-/x-randomvalues-10.4.2.tgz#895f1220d5a4522a83d8d5014e3c1e03b129893e" + integrity sha512-mf1Wbpe7pRZHO0V3V89isPLqZOy5XGX2bCqsfUWHgb1NvV1MMx5TjVjdaYyNlGTiOkAmJKlOHshcfPU2sYWpNg== + dependencies: + "@babel/runtime" "^7.20.13" + "@polkadot/x-global" "10.4.2" + "@polkadot/x-randomvalues@6.11.1": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/x-randomvalues/-/x-randomvalues-6.11.1.tgz#f006fa250c8e82c92ccb769976a45a8e7f3df28b" @@ -3601,6 +3740,14 @@ "@babel/runtime" "^7.14.6" rxjs "^6.6.7" +"@polkadot/x-textdecoder@10.4.2": + version "10.4.2" + resolved "https://registry.yarnpkg.com/@polkadot/x-textdecoder/-/x-textdecoder-10.4.2.tgz#93202f3e5ad0e7f75a3fa02d2b8a3343091b341b" + integrity sha512-d3ADduOKUTU+cliz839+KCFmi23pxTlabH7qh7Vs1GZQvXOELWdqFOqakdiAjtMn68n1KVF4O14Y+OUm7gp/zA== + dependencies: + "@babel/runtime" "^7.20.13" + "@polkadot/x-global" "10.4.2" + "@polkadot/x-textdecoder@6.11.1": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/x-textdecoder/-/x-textdecoder-6.11.1.tgz#6cc314645681cc4639085c03b65328671c7f182c" @@ -3633,6 +3780,14 @@ "@babel/runtime" "^7.18.6" "@polkadot/x-global" "9.7.2" +"@polkadot/x-textencoder@10.4.2": + version "10.4.2" + resolved "https://registry.yarnpkg.com/@polkadot/x-textencoder/-/x-textencoder-10.4.2.tgz#cd2e6c8a66b0b400a73f0164e99c510fb5c83501" + integrity sha512-mxcQuA1exnyv74Kasl5vxBq01QwckG088lYjc3KwmND6+pPrW2OWagbxFX5VFoDLDAE+UJtnUHsjdWyOTDhpQA== + dependencies: + "@babel/runtime" "^7.20.13" + "@polkadot/x-global" "10.4.2" + "@polkadot/x-textencoder@6.11.1": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/x-textencoder/-/x-textencoder-6.11.1.tgz#73e89da5b91954ae380042c19314c90472f59d9e" @@ -3812,6 +3967,11 @@ resolved "https://registry.yarnpkg.com/@substrate/ss58-registry/-/ss58-registry-1.34.0.tgz#b6faed02343da7a8956444f5db23bc7246dd5fb5" integrity sha512-8Df5usnWvjnw/WRAmKOqHXRPPRfiCd1kIN8ttH4YmBrRTERjVInsdu0xvLdbyUYKyvgK6zKhHWQfYohXqllHhg== +"@substrate/ss58-registry@^1.38.0": + version "1.39.0" + resolved "https://registry.yarnpkg.com/@substrate/ss58-registry/-/ss58-registry-1.39.0.tgz#eb916ff5fea7fa02e77745823fde21af979273d2" + integrity sha512-qZYpuE6n+mwew+X71dOur/CbMXj6rNW27o63JeJwdQH/GvcSKm3JLNhd+bGzwUKg0D/zD30Qc6p4JykArzM+tA== + "@szmarczak/http-timer@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@szmarczak/http-timer/-/http-timer-1.1.2.tgz#b1665e2c461a2cd92f4c1bbf50d5454de0d4b421" @@ -3878,6 +4038,13 @@ dependencies: "@types/node" "*" +"@types/bn.js@^5.1.1": + version "5.1.1" + resolved "https://registry.yarnpkg.com/@types/bn.js/-/bn.js-5.1.1.tgz#b51e1b55920a4ca26e9285ff79936bbdec910682" + integrity sha512-qNrYbZqMx0uJAfKnKclPh+dTwK33KfLHYqtyODwd5HnXOjnkhc4qgn3BrK6RWyGZm5+sIFE7Q7Vz6QQtJB7w7g== + dependencies: + "@types/node" "*" + "@types/body-parser@*", "@types/body-parser@1.19.0": version "1.19.0" resolved "https://registry.yarnpkg.com/@types/body-parser/-/body-parser-1.19.0.tgz#0685b3c47eb3006ffed117cdd55164b61f80538f" From 62c7285f66ac5d7caf245059f3197162a77079d2 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Wed, 1 Mar 2023 14:05:07 +0400 Subject: [PATCH 02/59] feat(db): wrapped migration into one single transaction --- db/migrations/20230227122704-ss58-prefix.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 28911505..15a761cb 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -36,11 +36,15 @@ const convertAccountTable = (db, ss58Format) => { } exports.up = function (db) { - return convertAccountTable(db, 54); + db.startMigration(); + convertAccountTable(db, 54); + return db.endMigration(); }; exports.down = function (db) { - return convertAccountTable(db, 42); + db.startMigration(); + convertAccountTable(db, 42); + return db.endMigration(); }; exports._meta = { From b3dd4fad8137d0f211f47414a83f8159fad0fc09 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Wed, 1 Mar 2023 14:41:33 +0400 Subject: [PATCH 03/59] feat(db): add migration for era_* and featured tables --- db/migrations/20230227122704-ss58-prefix.js | 33 +++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 15a761cb..9f902762 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -17,6 +17,7 @@ exports.setup = function (options, seedLink) { Promise = options.Promise; }; +// Migration script for Account Table const convertAccountTable = (db, ss58Format) => { return db.runSql('SELECT * from account;', (_, result) => { @@ -35,15 +36,47 @@ const convertAccountTable = (db, ss58Format) => { }); } +// Migration script for featured, era_commission, era_points, era_relative_performance, era_self_stake, era_vrc_score tables +const convertEraTables = (db, ss58Format) => { + const tables = [ + 'featured', + 'era_commission', + 'era_points', + 'era_relative_performance', + 'era_self_stake', + 'era_vrc_score', + ]; + tables.forEach((table) => { + db.runSql(`SELECT * from ${table}`, (_, result) => { + + result.rows.forEach((row) => { + const currentAccountId = row.stash_address; + + const nextAddress = keyring.encodeAddress( + keyring.decodeAddress(currentAccountId), + ss58Format); + + const queryString = `UPDATE ${table} SET stash_address='${nextAddress}' WHERE stash_address='${currentAccountId}';` + + db.runSql(queryString); + + }); + + }); + }); +}; + exports.up = function (db) { db.startMigration(); convertAccountTable(db, 54); + convertEraTables(db, 54); return db.endMigration(); }; exports.down = function (db) { db.startMigration(); convertAccountTable(db, 42); + convertEraTables(db, 42); return db.endMigration(); }; From df9f054f97d0391b6febd0f05ec0b48a7a63f9bc Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Wed, 1 Mar 2023 18:29:18 +0400 Subject: [PATCH 04/59] feat(db): add balances property migration for account table --- db/migrations/20230227122704-ss58-prefix.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 9f902762..4f4328a3 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -21,14 +21,15 @@ exports.setup = function (options, seedLink) { const convertAccountTable = (db, ss58Format) => { return db.runSql('SELECT * from account;', (_, result) => { - result.rows.forEach((row) => { - const currentAccountId = row.account_id; + result.rows.forEach(({ account_id, balances }) => { const nextAddress = keyring.encodeAddress( - keyring.decodeAddress(currentAccountId), + keyring.decodeAddress(account_id), ss58Format); - const queryString = `UPDATE account SET account_id='${nextAddress}' WHERE account_id='${currentAccountId}';` + const nextBalances = balances.replace(account_id, nextAddress); + + const queryString = `UPDATE account SET account_id='${nextAddress}', balances='${nextBalances}' WHERE account_id='${account_id}';` db.runSql(queryString); }); From 0b9202fd8e05b307bbea469fe8ab2a1e963d5f5d Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Mon, 6 Mar 2023 15:25:20 +0400 Subject: [PATCH 05/59] feat(db): add migration for ranking and draft for block table --- db/migrations/20230227122704-ss58-prefix.js | 67 ++++++++++++++++++--- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 4f4328a3..710deca7 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -20,7 +20,6 @@ exports.setup = function (options, seedLink) { // Migration script for Account Table const convertAccountTable = (db, ss58Format) => { return db.runSql('SELECT * from account;', (_, result) => { - result.rows.forEach(({ account_id, balances }) => { const nextAddress = keyring.encodeAddress( @@ -33,7 +32,6 @@ const convertAccountTable = (db, ss58Format) => { db.runSql(queryString); }); - }); } @@ -67,18 +65,71 @@ const convertEraTables = (db, ss58Format) => { }); }; -exports.up = function (db) { - db.startMigration(); +const convertRankingTable = (db, ss58Format) => { + return db.runSql('SELECT * from ranking;', (_, result) => { + + result.rows.forEach(({ identity, stash_address, controller_address }) => { + const nextStashAddress = keyring.encodeAddress( + keyring.decodeAddress(stash_address), + ss58Format); + + const nextControllerAddress = keyring.encodeAddress( + keyring.decodeAddress(controller_address), + ss58Format); + + // TODO: Think how to deal with identity? Regex for addresses + const nextIdentity = identity; + + const queryString = `UPDATE ranking SET identity='${nextIdentity}', stash_address='${nextStashAddress}', controller_address='${nextControllerAddress}' WHERE stash_address='${stash_address}';` + console.log("Prev and next", queryString); + + db.runSql(queryString); + }); + + }); +} + +const convertBlockTable = async (db, ss55Format) => { + + return db.runSql('SELECT block_author from block;', (_, result) => { + const totalRows = result.rows.length; + const batchSize = 10000; + let currentIndex = 0; + + const updateBlock = (block_author) => { + const nextBlockAuthor = keyring.encodeAddress( + keyring.decodeAddress(block_author), + ss55Format); + + const queryString = `UPDATE block SET block_author='${nextBlockAuthor}' WHERE block_author='${block_author}';` + + db.runSql(queryString); + }; + + while(currentIndex < totalRows) { + const currentBatch = result.rows.slice(currentIndex, currentIndex + batchSize); + + currentBatch.forEach(({ block_author }) => { + updateBlock(block_author); + }); + + currentIndex+= batchSize; + }; + }); +}; + +exports.up = async function (db) { convertAccountTable(db, 54); convertEraTables(db, 54); - return db.endMigration(); + convertRankingTable(db, 54); + // convertBlockTable(db, 54); }; -exports.down = function (db) { - db.startMigration(); +exports.down = async function (db) { convertAccountTable(db, 42); convertEraTables(db, 42); - return db.endMigration(); + convertRankingTable(db, 42); + // convertBlockTable(db, 42); }; exports._meta = { From 87723cadb99a2f9ccc6a9fa41f9b529accf8e1f0 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Mon, 6 Mar 2023 17:56:12 +0400 Subject: [PATCH 06/59] refactor(db): refactor codebase --- db/migrations/20230227122704-ss58-prefix.js | 48 ++++++++++----------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 710deca7..21debf47 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -19,8 +19,8 @@ exports.setup = function (options, seedLink) { // Migration script for Account Table const convertAccountTable = (db, ss58Format) => { - return db.runSql('SELECT * from account;', (_, result) => { - result.rows.forEach(({ account_id, balances }) => { + db.runSql('SELECT * from account;', (_, result) => { + result.rows.forEach(({account_id, balances}) => { const nextAddress = keyring.encodeAddress( keyring.decodeAddress(account_id), @@ -48,14 +48,13 @@ const convertEraTables = (db, ss58Format) => { tables.forEach((table) => { db.runSql(`SELECT * from ${table}`, (_, result) => { - result.rows.forEach((row) => { - const currentAccountId = row.stash_address; + result.rows.forEach(({stash_address}) => { const nextAddress = keyring.encodeAddress( - keyring.decodeAddress(currentAccountId), + keyring.decodeAddress(stash_address), ss58Format); - const queryString = `UPDATE ${table} SET stash_address='${nextAddress}' WHERE stash_address='${currentAccountId}';` + const queryString = `UPDATE ${table} SET stash_address='${nextAddress}' WHERE stash_address='${stash_address}';` db.runSql(queryString); @@ -66,9 +65,9 @@ const convertEraTables = (db, ss58Format) => { }; const convertRankingTable = (db, ss58Format) => { - return db.runSql('SELECT * from ranking;', (_, result) => { + db.runSql('SELECT * from ranking;', (_, result) => { - result.rows.forEach(({ identity, stash_address, controller_address }) => { + result.rows.forEach(({identity, stash_address, controller_address}) => { const nextStashAddress = keyring.encodeAddress( keyring.decodeAddress(stash_address), ss58Format); @@ -81,7 +80,6 @@ const convertRankingTable = (db, ss58Format) => { const nextIdentity = identity; const queryString = `UPDATE ranking SET identity='${nextIdentity}', stash_address='${nextStashAddress}', controller_address='${nextControllerAddress}' WHERE stash_address='${stash_address}';` - console.log("Prev and next", queryString); db.runSql(queryString); }); @@ -89,9 +87,9 @@ const convertRankingTable = (db, ss58Format) => { }); } -const convertBlockTable = async (db, ss55Format) => { - - return db.runSql('SELECT block_author from block;', (_, result) => { +// This batching approach is very slow, it should be replaced with creating temp. table and joining https://stackoverflow.com/questions/35903375/how-to-update-large-table-with-millions-of-rows-in-sql-server +const convertBlockTable = (db, ss55Format) => { + db.runSql('SELECT block_author from block;', (_, result) => { const totalRows = result.rows.length; const batchSize = 10000; let currentIndex = 0; @@ -102,34 +100,34 @@ const convertBlockTable = async (db, ss55Format) => { ss55Format); const queryString = `UPDATE block SET block_author='${nextBlockAuthor}' WHERE block_author='${block_author}';` - db.runSql(queryString); }; - while(currentIndex < totalRows) { + while (currentIndex < totalRows) { const currentBatch = result.rows.slice(currentIndex, currentIndex + batchSize); - currentBatch.forEach(({ block_author }) => { + currentBatch.forEach(({block_author}) => { updateBlock(block_author); }); - currentIndex+= batchSize; - }; + currentIndex += batchSize; + } }); }; +const convertTables = (db, prefix) => { + convertAccountTable(db, prefix); + convertEraTables(db, prefix); + convertRankingTable(db, prefix); + // convertBlockTable(db, prefix); +} + exports.up = async function (db) { - convertAccountTable(db, 54); - convertEraTables(db, 54); - convertRankingTable(db, 54); - // convertBlockTable(db, 54); + convertTables(db, 54); }; exports.down = async function (db) { - convertAccountTable(db, 42); - convertEraTables(db, 42); - convertRankingTable(db, 42); - // convertBlockTable(db, 42); + convertTables(db, 42); }; exports._meta = { From 27fb7f7c529da7012e4361e72a0bab8f76caf389 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Mon, 6 Mar 2023 18:07:44 +0400 Subject: [PATCH 07/59] refactor(db): remove unused sql files --- db/migrations/sqls/20230227122704-ss58-prefix-down.sql | 0 db/migrations/sqls/20230227122704-ss58-prefix-up.sql | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 db/migrations/sqls/20230227122704-ss58-prefix-down.sql delete mode 100644 db/migrations/sqls/20230227122704-ss58-prefix-up.sql diff --git a/db/migrations/sqls/20230227122704-ss58-prefix-down.sql b/db/migrations/sqls/20230227122704-ss58-prefix-down.sql deleted file mode 100644 index e69de29b..00000000 diff --git a/db/migrations/sqls/20230227122704-ss58-prefix-up.sql b/db/migrations/sqls/20230227122704-ss58-prefix-up.sql deleted file mode 100644 index e69de29b..00000000 From beb82e3dc6a754b4d02a1bdd260acb63b6427eb2 Mon Sep 17 00:00:00 2001 From: Dmitriy Groschovskiy Date: Mon, 6 Mar 2023 18:31:38 +0200 Subject: [PATCH 08/59] Update dev.yaml --- .github/workflows/dev.yaml | 44 ++++++++++---------------------------- 1 file changed, 11 insertions(+), 33 deletions(-) diff --git a/.github/workflows/dev.yaml b/.github/workflows/dev.yaml index e8adff15..42966720 100644 --- a/.github/workflows/dev.yaml +++ b/.github/workflows/dev.yaml @@ -5,53 +5,31 @@ on: - dev-cere workflow_dispatch: +permissions: + id-token: write + contents: read + jobs: build_backend_cerestats_api: - uses: Cerebellum-Network/reusable-workflows/.github/workflows/build-and-push-docker.yaml@1.0.0 + uses: Cerebellum-Network/reusable-workflows/.github/workflows/deploy-to-ecr.yaml@1.3.0 with: runs-on: '["self-hosted", "cere-network-large-workers"]' - org: cerebellumnetwork environment: dev - image: crb-cerestats-api - repository: crb-cerestats-api + aws_account_id: ${{ vars.DEV_NETWORK_AWS_ACCOUNT_ID }} + repository: stats-api file: ./api/Dockerfile secrets: inherit - + build_backend_cerestats_crawler: - uses: Cerebellum-Network/reusable-workflows/.github/workflows/build-and-push-docker.yaml@1.0.0 + uses: Cerebellum-Network/reusable-workflows/.github/workflows/deploy-to-ecr.yaml@1.3.0 with: runs-on: '["self-hosted", "cere-network-large-workers"]' - org: cerebellumnetwork environment: dev - image: crb-cerestats-crawler - repository: crb-cerestats-crawler + aws_account_id: ${{ vars.DEV_NETWORK_AWS_ACCOUNT_ID }} + repository: stats-crawler file: ./backend/docker/crawler/Dockerfile secrets: inherit - deploy_backend_cerestats_api: - uses: Cerebellum-Network/reusable-workflows/.github/workflows/deploy-with-helm.yaml@1.0.0 - needs: build_backend_cerestats_api - with: - runs-on: '["self-hosted", "cere-network-dev-deployer"]' - helm-repo-path: cerestats/cerestats-api - helm-release: cerestats-api - namespace: cerestats - tag: ${{ needs.build_backend_cerestats_api.outputs.version }} - environment: dev - secrets: inherit - - deploy_backend_cerestats_crawler: - uses: Cerebellum-Network/reusable-workflows/.github/workflows/deploy-with-helm.yaml@1.0.0 - needs: build_backend_cerestats_crawler - with: - runs-on: '["self-hosted", "cere-network-dev-deployer"]' - helm-repo-path: cerestats/cerestats-crawler - helm-release: cerestats-crawler - namespace: cerestats - tag: ${{ needs.build_backend_cerestats_crawler.outputs.version }} - environment: dev - secrets: inherit - build_and_deploy_frontend: uses: Cerebellum-Network/reusable-workflows/.github/workflows/build-and-upload-static.yaml@1.0.0 with: From f1e3b8ff98687bd719ac4576e411ee8a452218ec Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Fri, 10 Mar 2023 12:38:50 +0400 Subject: [PATCH 09/59] feat(db): add migration for faucet table --- db/migrations/20230227122704-ss58-prefix.js | 22 +++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 21debf47..e5efa2ed 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -54,6 +54,7 @@ const convertEraTables = (db, ss58Format) => { keyring.decodeAddress(stash_address), ss58Format); + // TODO: check where condition! const queryString = `UPDATE ${table} SET stash_address='${nextAddress}' WHERE stash_address='${stash_address}';` db.runSql(queryString); @@ -87,6 +88,26 @@ const convertRankingTable = (db, ss58Format) => { }); } +const convertFaucetTable = (db, ss58Format) => { + db.runSql('SELECT * from faucet;', (_, result) => { + + result.rows.forEach(({ id, sender, destination }) => { + const nextSender = keyring.encodeAddress( + keyring.decodeAddress(sender), + ss58Format); + + const nextDestination = keyring.encodeAddress( + keyring.decodeAddress(destination), + ss58Format); + + const queryString = `UPDATE faucet SET sender='${nextSender}', destination='${nextDestination}' WHERE id='${id}';` + + db.runSql(queryString); + }); + + }); +} + // This batching approach is very slow, it should be replaced with creating temp. table and joining https://stackoverflow.com/questions/35903375/how-to-update-large-table-with-millions-of-rows-in-sql-server const convertBlockTable = (db, ss55Format) => { db.runSql('SELECT block_author from block;', (_, result) => { @@ -119,6 +140,7 @@ const convertTables = (db, prefix) => { convertAccountTable(db, prefix); convertEraTables(db, prefix); convertRankingTable(db, prefix); + convertFaucetTable(db, prefix); // convertBlockTable(db, prefix); } From 2cf07bd423a696e0ad077ddcbe3d67a1fa28d0c4 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Fri, 10 Mar 2023 14:02:11 +0400 Subject: [PATCH 10/59] feat(db): add identity field migration for accounts --- db/migrations/20230227122704-ss58-prefix.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index e5efa2ed..49f2568c 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -20,7 +20,7 @@ exports.setup = function (options, seedLink) { // Migration script for Account Table const convertAccountTable = (db, ss58Format) => { db.runSql('SELECT * from account;', (_, result) => { - result.rows.forEach(({account_id, balances}) => { + result.rows.forEach(({ account_id, balances, identity }) => { const nextAddress = keyring.encodeAddress( keyring.decodeAddress(account_id), @@ -28,7 +28,21 @@ const convertAccountTable = (db, ss58Format) => { const nextBalances = balances.replace(account_id, nextAddress); - const queryString = `UPDATE account SET account_id='${nextAddress}', balances='${nextBalances}' WHERE account_id='${account_id}';` + let queryString = `UPDATE account SET account_id='${nextAddress}', balances='${nextBalances}'` + + if(identity && identity.includes("parent")) { + const currentParent = JSON.parse(identity).parent; + + const nextParent = keyring.encodeAddress( + keyring.decodeAddress(currentParent), + ss58Format); + + const nextIdentity = identity.replace(currentParent, nextParent); + + queryString+= `, identity='${nextIdentity}'` + } + + queryString+= ` WHERE account_id='${account_id}';` db.runSql(queryString); }); From 5e75b0edb08f1c7f0e0d85036db458143dad9726 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Fri, 10 Mar 2023 14:13:32 +0400 Subject: [PATCH 11/59] feat(db): add identity field migration for ranking --- db/migrations/20230227122704-ss58-prefix.js | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 49f2568c..6094d5c8 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -91,10 +91,22 @@ const convertRankingTable = (db, ss58Format) => { keyring.decodeAddress(controller_address), ss58Format); - // TODO: Think how to deal with identity? Regex for addresses - const nextIdentity = identity; + let queryString = `UPDATE ranking SET stash_address='${nextStashAddress}', controller_address='${nextControllerAddress}'` - const queryString = `UPDATE ranking SET identity='${nextIdentity}', stash_address='${nextStashAddress}', controller_address='${nextControllerAddress}' WHERE stash_address='${stash_address}';` + if(identity && identity.includes("parent")) { + const currentParent = JSON.parse(identity).parent; + + const nextParent = keyring.encodeAddress( + keyring.decodeAddress(currentParent), + ss58Format); + + const nextIdentity = identity.replace(currentParent, nextParent); + + queryString+= `, identity='${nextIdentity}'` + } + + // TODO: check where condition + queryString+= ` WHERE stash_address='${stash_address}'`; db.runSql(queryString); }); From 033298bc4c0e0cb9cf0afaab421470856a13f209 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Fri, 10 Mar 2023 16:39:24 +0400 Subject: [PATCH 12/59] feat(db): add convert for transfers --- db/migrations/20230227122704-ss58-prefix.js | 30 +++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 6094d5c8..1bf2f816 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -162,11 +162,41 @@ const convertBlockTable = (db, ss55Format) => { }); }; +// Transfers are a part of extrinsict table +const convertTransfers = (db, ss58Format) => { + db.runSql(`select * from extrinsic where method like 'transfer%'`, (_, result) => { + + result.rows.forEach(({ block_number, signer, args, method }) => { + const nextSigner = keyring.encodeAddress( + keyring.decodeAddress(signer), + ss58Format); + + let queryString = `UPDATE extrinsic SET signer='${nextSigner}'` + + if(method === 'transfer' || method === 'transferKeepAlive') { + const [account] = JSON.parse(args); + const accountId = typeof account === 'string' ? account : account.id; + const nextAccountId = keyring.encodeAddress( + keyring.decodeAddress(accountId), + ss58Format); + const nextArgs = args.replace(accountId, nextAccountId); + queryString+= `, args='${nextArgs}'`; + } + + queryString+= ` WHERE block_number='${block_number}';` + + db.runSql(queryString); + }); + + }); +}; + const convertTables = (db, prefix) => { convertAccountTable(db, prefix); convertEraTables(db, prefix); convertRankingTable(db, prefix); convertFaucetTable(db, prefix); + convertTransfers(db, prefix); // convertBlockTable(db, prefix); } From 7a74302b26510ca147865f3434e6392bd748f507 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Fri, 10 Mar 2023 16:40:15 +0400 Subject: [PATCH 13/59] feat(db): remove converting block tables --- db/migrations/20230227122704-ss58-prefix.js | 29 --------------------- 1 file changed, 29 deletions(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 1bf2f816..16fea9dd 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -134,34 +134,6 @@ const convertFaucetTable = (db, ss58Format) => { }); } -// This batching approach is very slow, it should be replaced with creating temp. table and joining https://stackoverflow.com/questions/35903375/how-to-update-large-table-with-millions-of-rows-in-sql-server -const convertBlockTable = (db, ss55Format) => { - db.runSql('SELECT block_author from block;', (_, result) => { - const totalRows = result.rows.length; - const batchSize = 10000; - let currentIndex = 0; - - const updateBlock = (block_author) => { - const nextBlockAuthor = keyring.encodeAddress( - keyring.decodeAddress(block_author), - ss55Format); - - const queryString = `UPDATE block SET block_author='${nextBlockAuthor}' WHERE block_author='${block_author}';` - db.runSql(queryString); - }; - - while (currentIndex < totalRows) { - const currentBatch = result.rows.slice(currentIndex, currentIndex + batchSize); - - currentBatch.forEach(({block_author}) => { - updateBlock(block_author); - }); - - currentIndex += batchSize; - } - }); -}; - // Transfers are a part of extrinsict table const convertTransfers = (db, ss58Format) => { db.runSql(`select * from extrinsic where method like 'transfer%'`, (_, result) => { @@ -197,7 +169,6 @@ const convertTables = (db, prefix) => { convertRankingTable(db, prefix); convertFaucetTable(db, prefix); convertTransfers(db, prefix); - // convertBlockTable(db, prefix); } exports.up = async function (db) { From 7caf5de14db50200e40109682145a58a92bf746b Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Fri, 10 Mar 2023 17:08:25 +0400 Subject: [PATCH 14/59] refactor(db): apply code formatting --- db/migrations/20230227122704-ss58-prefix.js | 30 +++++++++++---------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 16fea9dd..1eb3eff5 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -20,7 +20,7 @@ exports.setup = function (options, seedLink) { // Migration script for Account Table const convertAccountTable = (db, ss58Format) => { db.runSql('SELECT * from account;', (_, result) => { - result.rows.forEach(({ account_id, balances, identity }) => { + result.rows.forEach(({account_id, balances, identity}) => { const nextAddress = keyring.encodeAddress( keyring.decodeAddress(account_id), @@ -30,7 +30,7 @@ const convertAccountTable = (db, ss58Format) => { let queryString = `UPDATE account SET account_id='${nextAddress}', balances='${nextBalances}'` - if(identity && identity.includes("parent")) { + if (identity && identity.includes("parent")) { const currentParent = JSON.parse(identity).parent; const nextParent = keyring.encodeAddress( @@ -39,10 +39,10 @@ const convertAccountTable = (db, ss58Format) => { const nextIdentity = identity.replace(currentParent, nextParent); - queryString+= `, identity='${nextIdentity}'` + queryString += `, identity='${nextIdentity}'` } - queryString+= ` WHERE account_id='${account_id}';` + queryString += ` WHERE account_id='${account_id}';` db.runSql(queryString); }); @@ -93,7 +93,7 @@ const convertRankingTable = (db, ss58Format) => { let queryString = `UPDATE ranking SET stash_address='${nextStashAddress}', controller_address='${nextControllerAddress}'` - if(identity && identity.includes("parent")) { + if (identity && identity.includes("parent")) { const currentParent = JSON.parse(identity).parent; const nextParent = keyring.encodeAddress( @@ -102,11 +102,11 @@ const convertRankingTable = (db, ss58Format) => { const nextIdentity = identity.replace(currentParent, nextParent); - queryString+= `, identity='${nextIdentity}'` + queryString += `, identity='${nextIdentity}'` } // TODO: check where condition - queryString+= ` WHERE stash_address='${stash_address}'`; + queryString += ` WHERE stash_address='${stash_address}'`; db.runSql(queryString); }); @@ -117,7 +117,7 @@ const convertRankingTable = (db, ss58Format) => { const convertFaucetTable = (db, ss58Format) => { db.runSql('SELECT * from faucet;', (_, result) => { - result.rows.forEach(({ id, sender, destination }) => { + result.rows.forEach(({id, sender, destination}) => { const nextSender = keyring.encodeAddress( keyring.decodeAddress(sender), ss58Format); @@ -138,24 +138,24 @@ const convertFaucetTable = (db, ss58Format) => { const convertTransfers = (db, ss58Format) => { db.runSql(`select * from extrinsic where method like 'transfer%'`, (_, result) => { - result.rows.forEach(({ block_number, signer, args, method }) => { + result.rows.forEach(({block_number, signer, args, method}) => { const nextSigner = keyring.encodeAddress( keyring.decodeAddress(signer), ss58Format); let queryString = `UPDATE extrinsic SET signer='${nextSigner}'` - if(method === 'transfer' || method === 'transferKeepAlive') { + if (method === 'transfer' || method === 'transferKeepAlive') { const [account] = JSON.parse(args); const accountId = typeof account === 'string' ? account : account.id; const nextAccountId = keyring.encodeAddress( keyring.decodeAddress(accountId), ss58Format); const nextArgs = args.replace(accountId, nextAccountId); - queryString+= `, args='${nextArgs}'`; + queryString += `, args='${nextArgs}'`; } - queryString+= ` WHERE block_number='${block_number}';` + queryString += ` WHERE block_number='${block_number}';` db.runSql(queryString); }); @@ -172,11 +172,13 @@ const convertTables = (db, prefix) => { } exports.up = async function (db) { - convertTables(db, 54); + const newSs58Prefix = 54; + convertTables(db, newSs58Prefix); }; exports.down = async function (db) { - convertTables(db, 42); + const oldSs58Prefix = 42; + convertTables(db, oldSs58Prefix); }; exports._meta = { From 4144bc8fcd981fb055e9e68c37c5819844e8b037 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Fri, 10 Mar 2023 17:15:39 +0400 Subject: [PATCH 15/59] feat(db): speed up convertEraTables --- db/migrations/20230227122704-ss58-prefix.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 1eb3eff5..77036ccc 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -60,7 +60,7 @@ const convertEraTables = (db, ss58Format) => { 'era_vrc_score', ]; tables.forEach((table) => { - db.runSql(`SELECT * from ${table}`, (_, result) => { + db.runSql(`SELECT DISTINCT stash_address from ${table}`, (_, result) => { result.rows.forEach(({stash_address}) => { @@ -68,7 +68,6 @@ const convertEraTables = (db, ss58Format) => { keyring.decodeAddress(stash_address), ss58Format); - // TODO: check where condition! const queryString = `UPDATE ${table} SET stash_address='${nextAddress}' WHERE stash_address='${stash_address}';` db.runSql(queryString); From 524aaf4ae0c9b9ca06b2fb3c1a6a61f8426d8de6 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Fri, 10 Mar 2023 17:19:38 +0400 Subject: [PATCH 16/59] feat(db): change ranking update condition --- db/migrations/20230227122704-ss58-prefix.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 77036ccc..57e96eb2 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -81,7 +81,7 @@ const convertEraTables = (db, ss58Format) => { const convertRankingTable = (db, ss58Format) => { db.runSql('SELECT * from ranking;', (_, result) => { - result.rows.forEach(({identity, stash_address, controller_address}) => { + result.rows.forEach(({identity, stash_address, controller_address, rank}) => { const nextStashAddress = keyring.encodeAddress( keyring.decodeAddress(stash_address), ss58Format); @@ -104,8 +104,7 @@ const convertRankingTable = (db, ss58Format) => { queryString += `, identity='${nextIdentity}'` } - // TODO: check where condition - queryString += ` WHERE stash_address='${stash_address}'`; + queryString += ` WHERE rank='${rank}'`; db.runSql(queryString); }); From a89cdb48c75664ef4a5fe6a8be249582f8bc4775 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Fri, 10 Mar 2023 17:24:33 +0400 Subject: [PATCH 17/59] feat(db): specify row properties for each table in select query --- db/migrations/20230227122704-ss58-prefix.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 57e96eb2..b8f16c85 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -19,7 +19,7 @@ exports.setup = function (options, seedLink) { // Migration script for Account Table const convertAccountTable = (db, ss58Format) => { - db.runSql('SELECT * from account;', (_, result) => { + db.runSql('SELECT account_id, balances, identity from account;', (_, result) => { result.rows.forEach(({account_id, balances, identity}) => { const nextAddress = keyring.encodeAddress( @@ -79,7 +79,7 @@ const convertEraTables = (db, ss58Format) => { }; const convertRankingTable = (db, ss58Format) => { - db.runSql('SELECT * from ranking;', (_, result) => { + db.runSql('SELECT identity, stash_address, controller_address, rank from ranking;', (_, result) => { result.rows.forEach(({identity, stash_address, controller_address, rank}) => { const nextStashAddress = keyring.encodeAddress( @@ -113,7 +113,7 @@ const convertRankingTable = (db, ss58Format) => { } const convertFaucetTable = (db, ss58Format) => { - db.runSql('SELECT * from faucet;', (_, result) => { + db.runSql('SELECT id, sender, destination from faucet;', (_, result) => { result.rows.forEach(({id, sender, destination}) => { const nextSender = keyring.encodeAddress( @@ -134,7 +134,7 @@ const convertFaucetTable = (db, ss58Format) => { // Transfers are a part of extrinsict table const convertTransfers = (db, ss58Format) => { - db.runSql(`select * from extrinsic where method like 'transfer%'`, (_, result) => { + db.runSql(`select block_number, signer, args, method from extrinsic where method like 'transfer%'`, (_, result) => { result.rows.forEach(({block_number, signer, args, method}) => { const nextSigner = keyring.encodeAddress( From 39296eddfcdcd78c41e2834ee3e67db28a0f734a Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Fri, 10 Mar 2023 17:57:59 +0400 Subject: [PATCH 18/59] feat(db): add logger --- db/migrations/20230227122704-ss58-prefix.js | 48 ++++++++++++++++++--- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index b8f16c85..fb6717b3 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -17,9 +17,35 @@ exports.setup = function (options, seedLink) { Promise = options.Promise; }; +class Logger { + constructor(tableName) { + this.tableName = tableName; + this.totalRows = 0; + this.currentRow = 0; + } + + setTotalRows = (totalRows) => { + this.totalRows = totalRows; + } + + log = () => { + this.currentRow++; + const {currentRow, totalRows, tableName} = this; + + if (currentRow === totalRows) { + console.log(`✅ Done for ${tableName} table`); + } else { + console.log(`⏳ Migrated ${Math.trunc(currentRow / totalRows * 10000) / 100}% for ${tableName} table`); + } + } +} + // Migration script for Account Table const convertAccountTable = (db, ss58Format) => { + const logger = new Logger('account'); + db.runSql('SELECT account_id, balances, identity from account;', (_, result) => { + logger.setTotalRows(result.rows.length); result.rows.forEach(({account_id, balances, identity}) => { const nextAddress = keyring.encodeAddress( @@ -44,7 +70,7 @@ const convertAccountTable = (db, ss58Format) => { queryString += ` WHERE account_id='${account_id}';` - db.runSql(queryString); + db.runSql(queryString, logger.log); }); }); } @@ -60,7 +86,10 @@ const convertEraTables = (db, ss58Format) => { 'era_vrc_score', ]; tables.forEach((table) => { + const logger = new Logger(table); + db.runSql(`SELECT DISTINCT stash_address from ${table}`, (_, result) => { + logger.setTotalRows(result.rows.length); result.rows.forEach(({stash_address}) => { @@ -70,7 +99,7 @@ const convertEraTables = (db, ss58Format) => { const queryString = `UPDATE ${table} SET stash_address='${nextAddress}' WHERE stash_address='${stash_address}';` - db.runSql(queryString); + db.runSql(queryString, logger.log); }); @@ -79,7 +108,10 @@ const convertEraTables = (db, ss58Format) => { }; const convertRankingTable = (db, ss58Format) => { + const logger = new Logger('ranking'); + db.runSql('SELECT identity, stash_address, controller_address, rank from ranking;', (_, result) => { + logger.setTotalRows(result.rows.length); result.rows.forEach(({identity, stash_address, controller_address, rank}) => { const nextStashAddress = keyring.encodeAddress( @@ -106,14 +138,17 @@ const convertRankingTable = (db, ss58Format) => { queryString += ` WHERE rank='${rank}'`; - db.runSql(queryString); + db.runSql(queryString, logger.log); }); }); } const convertFaucetTable = (db, ss58Format) => { + const logger = new Logger('faucet'); + db.runSql('SELECT id, sender, destination from faucet;', (_, result) => { + logger.setTotalRows(result.rows.length); result.rows.forEach(({id, sender, destination}) => { const nextSender = keyring.encodeAddress( @@ -126,7 +161,7 @@ const convertFaucetTable = (db, ss58Format) => { const queryString = `UPDATE faucet SET sender='${nextSender}', destination='${nextDestination}' WHERE id='${id}';` - db.runSql(queryString); + db.runSql(queryString, logger.log); }); }); @@ -134,7 +169,10 @@ const convertFaucetTable = (db, ss58Format) => { // Transfers are a part of extrinsict table const convertTransfers = (db, ss58Format) => { + const logger = new Logger('transfer'); + db.runSql(`select block_number, signer, args, method from extrinsic where method like 'transfer%'`, (_, result) => { + logger.setTotalRows(result.rows.length); result.rows.forEach(({block_number, signer, args, method}) => { const nextSigner = keyring.encodeAddress( @@ -155,7 +193,7 @@ const convertTransfers = (db, ss58Format) => { queryString += ` WHERE block_number='${block_number}';` - db.runSql(queryString); + db.runSql(queryString, logger.log); }); }); From c973a976b4ffed30977402d062b5533e5ebe9011 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Tue, 14 Mar 2023 14:57:17 +0400 Subject: [PATCH 19/59] refactor(db): replace identity parent check --- db/migrations/20230227122704-ss58-prefix.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index fb6717b3..d4246f74 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -56,7 +56,7 @@ const convertAccountTable = (db, ss58Format) => { let queryString = `UPDATE account SET account_id='${nextAddress}', balances='${nextBalances}'` - if (identity && identity.includes("parent")) { + if (identity?.includes("parent")) { const currentParent = JSON.parse(identity).parent; const nextParent = keyring.encodeAddress( @@ -124,7 +124,7 @@ const convertRankingTable = (db, ss58Format) => { let queryString = `UPDATE ranking SET stash_address='${nextStashAddress}', controller_address='${nextControllerAddress}'` - if (identity && identity.includes("parent")) { + if (identity?.includes("parent")) { const currentParent = JSON.parse(identity).parent; const nextParent = keyring.encodeAddress( From f936323c09ef38114ffeb2e36f179950e84b7a01 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Tue, 14 Mar 2023 15:00:52 +0400 Subject: [PATCH 20/59] docs(db): update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72d3a118..47bf6efc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ ## vNext - ... +## v0.25.0 +- [BE] Add ss58 prefix migration for first priority data + ## v0.24.3 - [BE] Add missing `next` functions to prevent Stats API crash From fe48b217a3a5443984ae7324f4e3c86ac6f17266 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Fri, 17 Mar 2023 19:29:50 +0400 Subject: [PATCH 21/59] feat(db): add transferAll check --- db/migrations/20230227122704-ss58-prefix.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index d4246f74..e59f77f7 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -181,7 +181,7 @@ const convertTransfers = (db, ss58Format) => { let queryString = `UPDATE extrinsic SET signer='${nextSigner}'` - if (method === 'transfer' || method === 'transferKeepAlive') { + if (method === 'transfer' || method === 'transferKeepAlive' || method === 'transferAll') { const [account] = JSON.parse(args); const accountId = typeof account === 'string' ? account : account.id; const nextAccountId = keyring.encodeAddress( From 75e26be78c08f0d26ac4032ab4650a7f08712b09 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Fri, 17 Mar 2023 20:01:17 +0400 Subject: [PATCH 22/59] refactor(db): add decode function --- db/migrations/20230227122704-ss58-prefix.js | 57 ++++++++++----------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index e59f77f7..6faf6a9a 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -40,6 +40,17 @@ class Logger { } } +const decode = (address, ss58) => { + if (address.startsWith('0x')) { + console.log('Address starts with 0x', address); + return address; + } else { + return keyring.encodeAddress( + keyring.decodeAddress(address), + ss58); + } +}; + // Migration script for Account Table const convertAccountTable = (db, ss58Format) => { const logger = new Logger('account'); @@ -48,9 +59,7 @@ const convertAccountTable = (db, ss58Format) => { logger.setTotalRows(result.rows.length); result.rows.forEach(({account_id, balances, identity}) => { - const nextAddress = keyring.encodeAddress( - keyring.decodeAddress(account_id), - ss58Format); + const nextAddress = decode(account_id, ss58Format); const nextBalances = balances.replace(account_id, nextAddress); @@ -59,9 +68,7 @@ const convertAccountTable = (db, ss58Format) => { if (identity?.includes("parent")) { const currentParent = JSON.parse(identity).parent; - const nextParent = keyring.encodeAddress( - keyring.decodeAddress(currentParent), - ss58Format); + const nextParent = decode(currentParent, ss58Format); const nextIdentity = identity.replace(currentParent, nextParent); @@ -93,9 +100,7 @@ const convertEraTables = (db, ss58Format) => { result.rows.forEach(({stash_address}) => { - const nextAddress = keyring.encodeAddress( - keyring.decodeAddress(stash_address), - ss58Format); + const nextAddress = decode(stash_address, ss58Format) const queryString = `UPDATE ${table} SET stash_address='${nextAddress}' WHERE stash_address='${stash_address}';` @@ -114,22 +119,17 @@ const convertRankingTable = (db, ss58Format) => { logger.setTotalRows(result.rows.length); result.rows.forEach(({identity, stash_address, controller_address, rank}) => { - const nextStashAddress = keyring.encodeAddress( - keyring.decodeAddress(stash_address), - ss58Format); - const nextControllerAddress = keyring.encodeAddress( - keyring.decodeAddress(controller_address), - ss58Format); + const nextStashAddress = decode(stash_address, ss58Format); + + const nextControllerAddress = decode(controller_address, ss58Format); let queryString = `UPDATE ranking SET stash_address='${nextStashAddress}', controller_address='${nextControllerAddress}'` if (identity?.includes("parent")) { const currentParent = JSON.parse(identity).parent; - const nextParent = keyring.encodeAddress( - keyring.decodeAddress(currentParent), - ss58Format); + const nextParent = decode(currentParent, ss58Format); const nextIdentity = identity.replace(currentParent, nextParent); @@ -151,13 +151,10 @@ const convertFaucetTable = (db, ss58Format) => { logger.setTotalRows(result.rows.length); result.rows.forEach(({id, sender, destination}) => { - const nextSender = keyring.encodeAddress( - keyring.decodeAddress(sender), - ss58Format); - const nextDestination = keyring.encodeAddress( - keyring.decodeAddress(destination), - ss58Format); + const nextSender = decode(sender, ss58Format); + + const nextDestination = decode(destination, ss58Format); const queryString = `UPDATE faucet SET sender='${nextSender}', destination='${nextDestination}' WHERE id='${id}';` @@ -175,18 +172,16 @@ const convertTransfers = (db, ss58Format) => { logger.setTotalRows(result.rows.length); result.rows.forEach(({block_number, signer, args, method}) => { - const nextSigner = keyring.encodeAddress( - keyring.decodeAddress(signer), - ss58Format); + const nextSigner = decode(signer, ss58Format); let queryString = `UPDATE extrinsic SET signer='${nextSigner}'` if (method === 'transfer' || method === 'transferKeepAlive' || method === 'transferAll') { const [account] = JSON.parse(args); - const accountId = typeof account === 'string' ? account : account.id; - const nextAccountId = keyring.encodeAddress( - keyring.decodeAddress(accountId), - ss58Format); + const accountId = typeof account === 'string' ? account : account.id ? account.id : account.address20; + + const nextAccountId = decode(accountId, ss58Format); + const nextArgs = args.replace(accountId, nextAccountId); queryString += `, args='${nextArgs}'`; } From 47a6422439c961b3088378e1138747ee43ec28fd Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Tue, 21 Mar 2023 15:21:01 +0400 Subject: [PATCH 23/59] fix(api): add @polkadot/keyring dependency --- api/package.json | 1 + yarn.lock | 149 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 149 insertions(+), 1 deletion(-) diff --git a/api/package.json b/api/package.json index 80facafb..f0bca6d4 100644 --- a/api/package.json +++ b/api/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "@polkadot/api": "8.2.1", + "@polkadot/keyring": "^11.1.1", "apicache": "^1.6.3", "axios": "^0.21.1", "body-parser": "^1.19.0", diff --git a/yarn.lock b/yarn.lock index 3dd630b7..55f2e0cd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2301,6 +2301,11 @@ resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-1.2.0.tgz#a3150eeb09cc7ab207ebf6d7b9ad311a9bdbed12" integrity sha512-FZfhjEDbT5GRswV3C6uvLPHMiVD6lQBmpoX5+eSiPaMTXte/IKqI5dykDxzZB/WBeK/CDuQRBWarPdi3FNY2zQ== +"@noble/hashes@1.3.0": + version "1.3.0" + resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-1.3.0.tgz#085fd70f6d7d9d109671090ccae1d3bec62554a1" + integrity sha512-ilHEACi9DwqJB0pw7kv+Apvh50jiiSyR/cQ3y4W7lOR5mhvn/50FLUfsnfJz0BDZtl/RR16kXvptiv6q1msYZg== + "@noble/hashes@^0.4.5": version "0.4.5" resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-0.4.5.tgz#f69a963b0c59c1145bc5aca1f3eef58a48bf9a59" @@ -3060,6 +3065,15 @@ "@polkadot/util" "10.4.2" "@polkadot/util-crypto" "10.4.2" +"@polkadot/keyring@^11.1.1": + version "11.1.1" + resolved "https://registry.yarnpkg.com/@polkadot/keyring/-/keyring-11.1.1.tgz#5f0d4159b652ae7e127092f62ceaeae0fb7d96b8" + integrity sha512-E3b33WmhOrgAmQkm8roDy+M+7rklqeVitqwQ7HvRAos3Rn8ZOqawG9g0zgTlyP7kKqp0WRK2ccrgHXdVgFcyFg== + dependencies: + "@polkadot/util" "11.1.1" + "@polkadot/util-crypto" "11.1.1" + tslib "^2.5.0" + "@polkadot/keyring@^6.0.5": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/keyring/-/keyring-6.11.1.tgz#2510c349c965c74cc2f108f114f1048856940604" @@ -3099,6 +3113,15 @@ "@polkadot/util" "10.4.2" "@substrate/ss58-registry" "^1.38.0" +"@polkadot/networks@11.1.1": + version "11.1.1" + resolved "https://registry.yarnpkg.com/@polkadot/networks/-/networks-11.1.1.tgz#4b1c7bb7df52658d5cf4fce5ff6fb0ff02d41a42" + integrity sha512-5qjIkZKSCCW9MpvrKvT8QSeHyozIJSlTxA0lGM6sGT3KsFoOcW6ZaGBEsX7Kw4RrXCevxG60347cTzViekxF4A== + dependencies: + "@polkadot/util" "11.1.1" + "@substrate/ss58-registry" "^1.39.0" + tslib "^2.5.0" + "@polkadot/networks@6.11.1", "@polkadot/networks@^6.0.5": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/networks/-/networks-6.11.1.tgz#8fd189593f6ee4f8bf64378d0aaae09e39a37d35" @@ -3308,6 +3331,22 @@ ed2curve "^0.3.0" tweetnacl "^1.0.3" +"@polkadot/util-crypto@11.1.1": + version "11.1.1" + resolved "https://registry.yarnpkg.com/@polkadot/util-crypto/-/util-crypto-11.1.1.tgz#f4e4280b70fac38fc9a24f0dfbcdab5d01a56453" + integrity sha512-AB4z5IxBV90IEAtzs4LxVc7wrVyAQHmBSKoZ5xnOVsd0Hm10WeCiAOJa6DSFJcEs9+YfzA4H+nIWlyD7s2p9Yg== + dependencies: + "@noble/hashes" "1.3.0" + "@noble/secp256k1" "1.7.1" + "@polkadot/networks" "11.1.1" + "@polkadot/util" "11.1.1" + "@polkadot/wasm-crypto" "^7.0.3" + "@polkadot/x-bigint" "11.1.1" + "@polkadot/x-randomvalues" "11.1.1" + "@scure/base" "1.1.1" + tslib "^2.5.0" + tweetnacl "^1.0.3" + "@polkadot/util-crypto@6.11.1", "@polkadot/util-crypto@^6.0.5": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/util-crypto/-/util-crypto-6.11.1.tgz#7a36acf5c8bf52541609ec0b0b2a69af295d652e" @@ -3400,6 +3439,19 @@ "@types/bn.js" "^5.1.1" bn.js "^5.2.1" +"@polkadot/util@11.1.1": + version "11.1.1" + resolved "https://registry.yarnpkg.com/@polkadot/util/-/util-11.1.1.tgz#88db2df312e57bd6a0291d8e1e271a44e113fdd5" + integrity sha512-8vlSfJhMAck2OVdk8aep3sZP17txR+p8X3bFNP0qNJ7frfF741v/eViEC7bbVIgdT0/vYNmgS6+0Dwe06dnKuA== + dependencies: + "@polkadot/x-bigint" "11.1.1" + "@polkadot/x-global" "11.1.1" + "@polkadot/x-textdecoder" "11.1.1" + "@polkadot/x-textencoder" "11.1.1" + "@types/bn.js" "^5.1.1" + bn.js "^5.2.1" + tslib "^2.5.0" + "@polkadot/util@6.11.1", "@polkadot/util@^6.0.5": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/util/-/util-6.11.1.tgz#8950b038ba3e6ebfc0a7ff47feeb972e81b2626c" @@ -3479,6 +3531,13 @@ dependencies: "@babel/runtime" "^7.20.6" +"@polkadot/wasm-bridge@7.0.3": + version "7.0.3" + resolved "https://registry.yarnpkg.com/@polkadot/wasm-bridge/-/wasm-bridge-7.0.3.tgz#9691450830604dc4a361692a8a2a3df22fa53e96" + integrity sha512-q5qyhkGE9lHQmThNg6G5zCM4gYip2KtmR+De/URX7yWAO6snsinFqt066RFVuHvX1hZijrYSe/BGQABAUtH4pw== + dependencies: + tslib "^2.5.0" + "@polkadot/wasm-crypto-asmjs@6.3.1": version "6.3.1" resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-asmjs/-/wasm-crypto-asmjs-6.3.1.tgz#e8f469c9cf4a7709c8131a96f857291953f3e30a" @@ -3493,6 +3552,13 @@ dependencies: "@babel/runtime" "^7.20.6" +"@polkadot/wasm-crypto-asmjs@7.0.3": + version "7.0.3" + resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-asmjs/-/wasm-crypto-asmjs-7.0.3.tgz#a1bc942029979b2696a1062066d774e99a5a6b4c" + integrity sha512-ldMZjowYywn0Uj7jSr8a21rrlFFq/jWhCXVl21/KDcYGdFEfIajqbcrO5cHoT6w95sQgAwMWJwwDClXOaBjc/Q== + dependencies: + tslib "^2.5.0" + "@polkadot/wasm-crypto-asmjs@^4.2.1": version "4.2.1" resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-asmjs/-/wasm-crypto-asmjs-4.2.1.tgz#6b7eae1c011709f8042dfd30872a5fc5e9e021c0" @@ -3527,6 +3593,16 @@ "@polkadot/wasm-crypto-asmjs" "6.4.1" "@polkadot/wasm-crypto-wasm" "6.4.1" +"@polkadot/wasm-crypto-init@7.0.3": + version "7.0.3" + resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-init/-/wasm-crypto-init-7.0.3.tgz#336af713edfcd6fdd0194fee2919781893fba577" + integrity sha512-W4ClfPrzOTqiX0x4h6rXjCt8UsVsbg3zU7LJFFjeLgrguPoKTLGw4h5O1rR2H7EuMFbuqdztzJn3qTjBcR03Cg== + dependencies: + "@polkadot/wasm-bridge" "7.0.3" + "@polkadot/wasm-crypto-asmjs" "7.0.3" + "@polkadot/wasm-crypto-wasm" "7.0.3" + tslib "^2.5.0" + "@polkadot/wasm-crypto-wasm@6.3.1": version "6.3.1" resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-wasm/-/wasm-crypto-wasm-6.3.1.tgz#67f720e7f9694fef096abe9d60abbac02e032383" @@ -3543,6 +3619,14 @@ "@babel/runtime" "^7.20.6" "@polkadot/wasm-util" "6.4.1" +"@polkadot/wasm-crypto-wasm@7.0.3": + version "7.0.3" + resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-wasm/-/wasm-crypto-wasm-7.0.3.tgz#016834b1eb2564d8a13b133ee77a4612ad873d41" + integrity sha512-FRjUADiA3wMkjJqQLgB0v9rbSADcb2PY/6dJi06iza9m41HebTN3x7f5D3gWTCfgJjzWLAPchY2Hwsa0WpTQkw== + dependencies: + "@polkadot/wasm-util" "7.0.3" + tslib "^2.5.0" + "@polkadot/wasm-crypto-wasm@^4.2.1": version "4.2.1" resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-wasm/-/wasm-crypto-wasm-4.2.1.tgz#2a86f9b405e7195c3f523798c6ce4afffd19737e" @@ -3599,6 +3683,18 @@ "@polkadot/wasm-crypto-wasm" "6.4.1" "@polkadot/wasm-util" "6.4.1" +"@polkadot/wasm-crypto@^7.0.3": + version "7.0.3" + resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto/-/wasm-crypto-7.0.3.tgz#e07ddbeea0b45149d8e58be292ad423d646f1cb1" + integrity sha512-mOCLCaL9cyrU72PCc9nMNAj3zdvOzau5mOGJjLahIz+mqlHAoAmEXCAJvJ2qCo7OFl8QiDToAEGhdDWQfiHUyg== + dependencies: + "@polkadot/wasm-bridge" "7.0.3" + "@polkadot/wasm-crypto-asmjs" "7.0.3" + "@polkadot/wasm-crypto-init" "7.0.3" + "@polkadot/wasm-crypto-wasm" "7.0.3" + "@polkadot/wasm-util" "7.0.3" + tslib "^2.5.0" + "@polkadot/wasm-util@6.3.1": version "6.3.1" resolved "https://registry.yarnpkg.com/@polkadot/wasm-util/-/wasm-util-6.3.1.tgz#439ebb68a436317af388ed6438b8f879df3afcda" @@ -3613,6 +3709,13 @@ dependencies: "@babel/runtime" "^7.20.6" +"@polkadot/wasm-util@7.0.3": + version "7.0.3" + resolved "https://registry.yarnpkg.com/@polkadot/wasm-util/-/wasm-util-7.0.3.tgz#eab59f9dac0f00ca736aff8b24925108b7b2f860" + integrity sha512-L9U5nSbzr5xa2YSpveP/zZxhOB6i8ibssK+ihuG+7SICYtTC0B9wJp/UnjP/c6bEDlMV3yWiNXJPBTJMGmkmIQ== + dependencies: + tslib "^2.5.0" + "@polkadot/x-bigint@10.4.2": version "10.4.2" resolved "https://registry.yarnpkg.com/@polkadot/x-bigint/-/x-bigint-10.4.2.tgz#7eb2ec732259df48b5a00f07879a1331e05606ec" @@ -3621,6 +3724,14 @@ "@babel/runtime" "^7.20.13" "@polkadot/x-global" "10.4.2" +"@polkadot/x-bigint@11.1.1": + version "11.1.1" + resolved "https://registry.yarnpkg.com/@polkadot/x-bigint/-/x-bigint-11.1.1.tgz#f23fb17b6d1267cb7f4fbad11a04b67ec26a14c6" + integrity sha512-iLaaPSCnVuZ7LoOWZTHgs+Ebws0MdoNHmXoTriU60YLoojDJbcOInlO+1h3fNy6oPnYN3qA3Ml1mKDnP837nxg== + dependencies: + "@polkadot/x-global" "11.1.1" + tslib "^2.5.0" + "@polkadot/x-bigint@8.2.2": version "8.2.2" resolved "https://registry.yarnpkg.com/@polkadot/x-bigint/-/x-bigint-8.2.2.tgz#24f4e1b889a1ee899ac9c98fee526e44e2130c14" @@ -3664,6 +3775,13 @@ dependencies: "@babel/runtime" "^7.20.13" +"@polkadot/x-global@11.1.1": + version "11.1.1" + resolved "https://registry.yarnpkg.com/@polkadot/x-global/-/x-global-11.1.1.tgz#6862c6c679e94b0fc39dff3b6954bb508ff3d7a4" + integrity sha512-++LFUT98bi2m15w8LrgOcpE5mi9bmH65YB02xbKzU0ZHe1g5l0LwFt+QFB9tZlNqfWTgwpsFshGtvdPQqrFnKw== + dependencies: + tslib "^2.5.0" + "@polkadot/x-global@6.11.1", "@polkadot/x-global@^6.0.5": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/x-global/-/x-global-6.11.1.tgz#c292b3825fea60e9b33fff1790323fc57de1ca5d" @@ -3700,6 +3818,14 @@ "@babel/runtime" "^7.20.13" "@polkadot/x-global" "10.4.2" +"@polkadot/x-randomvalues@11.1.1": + version "11.1.1" + resolved "https://registry.yarnpkg.com/@polkadot/x-randomvalues/-/x-randomvalues-11.1.1.tgz#3d1a5f4df7cd7cd3499fd424600fc2599b90366e" + integrity sha512-t+Ag+RM/Agi8N86N73Ij1xz+87FYQLWZW+BlCdSEozTt933zloTNl4046IKj4sofZc51+ftRM3BFNmNT1UdlWQ== + dependencies: + "@polkadot/x-global" "11.1.1" + tslib "^2.5.0" + "@polkadot/x-randomvalues@6.11.1": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/x-randomvalues/-/x-randomvalues-6.11.1.tgz#f006fa250c8e82c92ccb769976a45a8e7f3df28b" @@ -3748,6 +3874,14 @@ "@babel/runtime" "^7.20.13" "@polkadot/x-global" "10.4.2" +"@polkadot/x-textdecoder@11.1.1": + version "11.1.1" + resolved "https://registry.yarnpkg.com/@polkadot/x-textdecoder/-/x-textdecoder-11.1.1.tgz#9596fe10981fa05f7629eaa198b11c8b191b1501" + integrity sha512-YoB82pr6kYkK5yg2BQgm5wVTf6Hq+01i+A6PgV1uXr7Rm3bxmQpGR2DKZq0QNjwWP0s6e91BxXvGoPjf7S9tBA== + dependencies: + "@polkadot/x-global" "11.1.1" + tslib "^2.5.0" + "@polkadot/x-textdecoder@6.11.1": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/x-textdecoder/-/x-textdecoder-6.11.1.tgz#6cc314645681cc4639085c03b65328671c7f182c" @@ -3788,6 +3922,14 @@ "@babel/runtime" "^7.20.13" "@polkadot/x-global" "10.4.2" +"@polkadot/x-textencoder@11.1.1": + version "11.1.1" + resolved "https://registry.yarnpkg.com/@polkadot/x-textencoder/-/x-textencoder-11.1.1.tgz#1c91e9aa568384595cbd7bd6ad50d71e4f92f2d0" + integrity sha512-I4IygnZeSyGUPyTmu7W2IsCHakax7QTVR9kMkCywaKEjiLzZU5B/LuDB0Gxn/3Jw2X2YfoB1TQ4mZ1bte4LX0g== + dependencies: + "@polkadot/x-global" "11.1.1" + tslib "^2.5.0" + "@polkadot/x-textencoder@6.11.1": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/x-textencoder/-/x-textencoder-6.11.1.tgz#73e89da5b91954ae380042c19314c90472f59d9e" @@ -3967,7 +4109,7 @@ resolved "https://registry.yarnpkg.com/@substrate/ss58-registry/-/ss58-registry-1.34.0.tgz#b6faed02343da7a8956444f5db23bc7246dd5fb5" integrity sha512-8Df5usnWvjnw/WRAmKOqHXRPPRfiCd1kIN8ttH4YmBrRTERjVInsdu0xvLdbyUYKyvgK6zKhHWQfYohXqllHhg== -"@substrate/ss58-registry@^1.38.0": +"@substrate/ss58-registry@^1.38.0", "@substrate/ss58-registry@^1.39.0": version "1.39.0" resolved "https://registry.yarnpkg.com/@substrate/ss58-registry/-/ss58-registry-1.39.0.tgz#eb916ff5fea7fa02e77745823fde21af979273d2" integrity sha512-qZYpuE6n+mwew+X71dOur/CbMXj6rNW27o63JeJwdQH/GvcSKm3JLNhd+bGzwUKg0D/zD30Qc6p4JykArzM+tA== @@ -17851,6 +17993,11 @@ tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.1.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.0.tgz#803b8cdab3e12ba581a4ca41c8839bbb0dacb09e" integrity sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg== +tslib@^2.5.0: + version "2.5.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.5.0.tgz#42bfed86f5787aeb41d031866c8f402429e0fddf" + integrity sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg== + tslib@~2.0.1: version "2.0.3" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.0.3.tgz#8e0741ac45fc0c226e58a17bfc3e64b9bc6ca61c" From a5e722512bdac72105228b8f173c0ef900d0b71d Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Tue, 21 Mar 2023 15:30:52 +0400 Subject: [PATCH 24/59] fix(api): set @polkadot/keyring dependency version to ^10.4.2 --- api/package.json | 2 +- yarn.lock | 149 +---------------------------------------------- 2 files changed, 2 insertions(+), 149 deletions(-) diff --git a/api/package.json b/api/package.json index f0bca6d4..90cb7e70 100644 --- a/api/package.json +++ b/api/package.json @@ -11,7 +11,7 @@ }, "dependencies": { "@polkadot/api": "8.2.1", - "@polkadot/keyring": "^11.1.1", + "@polkadot/keyring": "^10.4.2", "apicache": "^1.6.3", "axios": "^0.21.1", "body-parser": "^1.19.0", diff --git a/yarn.lock b/yarn.lock index 55f2e0cd..3dd630b7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2301,11 +2301,6 @@ resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-1.2.0.tgz#a3150eeb09cc7ab207ebf6d7b9ad311a9bdbed12" integrity sha512-FZfhjEDbT5GRswV3C6uvLPHMiVD6lQBmpoX5+eSiPaMTXte/IKqI5dykDxzZB/WBeK/CDuQRBWarPdi3FNY2zQ== -"@noble/hashes@1.3.0": - version "1.3.0" - resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-1.3.0.tgz#085fd70f6d7d9d109671090ccae1d3bec62554a1" - integrity sha512-ilHEACi9DwqJB0pw7kv+Apvh50jiiSyR/cQ3y4W7lOR5mhvn/50FLUfsnfJz0BDZtl/RR16kXvptiv6q1msYZg== - "@noble/hashes@^0.4.5": version "0.4.5" resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-0.4.5.tgz#f69a963b0c59c1145bc5aca1f3eef58a48bf9a59" @@ -3065,15 +3060,6 @@ "@polkadot/util" "10.4.2" "@polkadot/util-crypto" "10.4.2" -"@polkadot/keyring@^11.1.1": - version "11.1.1" - resolved "https://registry.yarnpkg.com/@polkadot/keyring/-/keyring-11.1.1.tgz#5f0d4159b652ae7e127092f62ceaeae0fb7d96b8" - integrity sha512-E3b33WmhOrgAmQkm8roDy+M+7rklqeVitqwQ7HvRAos3Rn8ZOqawG9g0zgTlyP7kKqp0WRK2ccrgHXdVgFcyFg== - dependencies: - "@polkadot/util" "11.1.1" - "@polkadot/util-crypto" "11.1.1" - tslib "^2.5.0" - "@polkadot/keyring@^6.0.5": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/keyring/-/keyring-6.11.1.tgz#2510c349c965c74cc2f108f114f1048856940604" @@ -3113,15 +3099,6 @@ "@polkadot/util" "10.4.2" "@substrate/ss58-registry" "^1.38.0" -"@polkadot/networks@11.1.1": - version "11.1.1" - resolved "https://registry.yarnpkg.com/@polkadot/networks/-/networks-11.1.1.tgz#4b1c7bb7df52658d5cf4fce5ff6fb0ff02d41a42" - integrity sha512-5qjIkZKSCCW9MpvrKvT8QSeHyozIJSlTxA0lGM6sGT3KsFoOcW6ZaGBEsX7Kw4RrXCevxG60347cTzViekxF4A== - dependencies: - "@polkadot/util" "11.1.1" - "@substrate/ss58-registry" "^1.39.0" - tslib "^2.5.0" - "@polkadot/networks@6.11.1", "@polkadot/networks@^6.0.5": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/networks/-/networks-6.11.1.tgz#8fd189593f6ee4f8bf64378d0aaae09e39a37d35" @@ -3331,22 +3308,6 @@ ed2curve "^0.3.0" tweetnacl "^1.0.3" -"@polkadot/util-crypto@11.1.1": - version "11.1.1" - resolved "https://registry.yarnpkg.com/@polkadot/util-crypto/-/util-crypto-11.1.1.tgz#f4e4280b70fac38fc9a24f0dfbcdab5d01a56453" - integrity sha512-AB4z5IxBV90IEAtzs4LxVc7wrVyAQHmBSKoZ5xnOVsd0Hm10WeCiAOJa6DSFJcEs9+YfzA4H+nIWlyD7s2p9Yg== - dependencies: - "@noble/hashes" "1.3.0" - "@noble/secp256k1" "1.7.1" - "@polkadot/networks" "11.1.1" - "@polkadot/util" "11.1.1" - "@polkadot/wasm-crypto" "^7.0.3" - "@polkadot/x-bigint" "11.1.1" - "@polkadot/x-randomvalues" "11.1.1" - "@scure/base" "1.1.1" - tslib "^2.5.0" - tweetnacl "^1.0.3" - "@polkadot/util-crypto@6.11.1", "@polkadot/util-crypto@^6.0.5": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/util-crypto/-/util-crypto-6.11.1.tgz#7a36acf5c8bf52541609ec0b0b2a69af295d652e" @@ -3439,19 +3400,6 @@ "@types/bn.js" "^5.1.1" bn.js "^5.2.1" -"@polkadot/util@11.1.1": - version "11.1.1" - resolved "https://registry.yarnpkg.com/@polkadot/util/-/util-11.1.1.tgz#88db2df312e57bd6a0291d8e1e271a44e113fdd5" - integrity sha512-8vlSfJhMAck2OVdk8aep3sZP17txR+p8X3bFNP0qNJ7frfF741v/eViEC7bbVIgdT0/vYNmgS6+0Dwe06dnKuA== - dependencies: - "@polkadot/x-bigint" "11.1.1" - "@polkadot/x-global" "11.1.1" - "@polkadot/x-textdecoder" "11.1.1" - "@polkadot/x-textencoder" "11.1.1" - "@types/bn.js" "^5.1.1" - bn.js "^5.2.1" - tslib "^2.5.0" - "@polkadot/util@6.11.1", "@polkadot/util@^6.0.5": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/util/-/util-6.11.1.tgz#8950b038ba3e6ebfc0a7ff47feeb972e81b2626c" @@ -3531,13 +3479,6 @@ dependencies: "@babel/runtime" "^7.20.6" -"@polkadot/wasm-bridge@7.0.3": - version "7.0.3" - resolved "https://registry.yarnpkg.com/@polkadot/wasm-bridge/-/wasm-bridge-7.0.3.tgz#9691450830604dc4a361692a8a2a3df22fa53e96" - integrity sha512-q5qyhkGE9lHQmThNg6G5zCM4gYip2KtmR+De/URX7yWAO6snsinFqt066RFVuHvX1hZijrYSe/BGQABAUtH4pw== - dependencies: - tslib "^2.5.0" - "@polkadot/wasm-crypto-asmjs@6.3.1": version "6.3.1" resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-asmjs/-/wasm-crypto-asmjs-6.3.1.tgz#e8f469c9cf4a7709c8131a96f857291953f3e30a" @@ -3552,13 +3493,6 @@ dependencies: "@babel/runtime" "^7.20.6" -"@polkadot/wasm-crypto-asmjs@7.0.3": - version "7.0.3" - resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-asmjs/-/wasm-crypto-asmjs-7.0.3.tgz#a1bc942029979b2696a1062066d774e99a5a6b4c" - integrity sha512-ldMZjowYywn0Uj7jSr8a21rrlFFq/jWhCXVl21/KDcYGdFEfIajqbcrO5cHoT6w95sQgAwMWJwwDClXOaBjc/Q== - dependencies: - tslib "^2.5.0" - "@polkadot/wasm-crypto-asmjs@^4.2.1": version "4.2.1" resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-asmjs/-/wasm-crypto-asmjs-4.2.1.tgz#6b7eae1c011709f8042dfd30872a5fc5e9e021c0" @@ -3593,16 +3527,6 @@ "@polkadot/wasm-crypto-asmjs" "6.4.1" "@polkadot/wasm-crypto-wasm" "6.4.1" -"@polkadot/wasm-crypto-init@7.0.3": - version "7.0.3" - resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-init/-/wasm-crypto-init-7.0.3.tgz#336af713edfcd6fdd0194fee2919781893fba577" - integrity sha512-W4ClfPrzOTqiX0x4h6rXjCt8UsVsbg3zU7LJFFjeLgrguPoKTLGw4h5O1rR2H7EuMFbuqdztzJn3qTjBcR03Cg== - dependencies: - "@polkadot/wasm-bridge" "7.0.3" - "@polkadot/wasm-crypto-asmjs" "7.0.3" - "@polkadot/wasm-crypto-wasm" "7.0.3" - tslib "^2.5.0" - "@polkadot/wasm-crypto-wasm@6.3.1": version "6.3.1" resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-wasm/-/wasm-crypto-wasm-6.3.1.tgz#67f720e7f9694fef096abe9d60abbac02e032383" @@ -3619,14 +3543,6 @@ "@babel/runtime" "^7.20.6" "@polkadot/wasm-util" "6.4.1" -"@polkadot/wasm-crypto-wasm@7.0.3": - version "7.0.3" - resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-wasm/-/wasm-crypto-wasm-7.0.3.tgz#016834b1eb2564d8a13b133ee77a4612ad873d41" - integrity sha512-FRjUADiA3wMkjJqQLgB0v9rbSADcb2PY/6dJi06iza9m41HebTN3x7f5D3gWTCfgJjzWLAPchY2Hwsa0WpTQkw== - dependencies: - "@polkadot/wasm-util" "7.0.3" - tslib "^2.5.0" - "@polkadot/wasm-crypto-wasm@^4.2.1": version "4.2.1" resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto-wasm/-/wasm-crypto-wasm-4.2.1.tgz#2a86f9b405e7195c3f523798c6ce4afffd19737e" @@ -3683,18 +3599,6 @@ "@polkadot/wasm-crypto-wasm" "6.4.1" "@polkadot/wasm-util" "6.4.1" -"@polkadot/wasm-crypto@^7.0.3": - version "7.0.3" - resolved "https://registry.yarnpkg.com/@polkadot/wasm-crypto/-/wasm-crypto-7.0.3.tgz#e07ddbeea0b45149d8e58be292ad423d646f1cb1" - integrity sha512-mOCLCaL9cyrU72PCc9nMNAj3zdvOzau5mOGJjLahIz+mqlHAoAmEXCAJvJ2qCo7OFl8QiDToAEGhdDWQfiHUyg== - dependencies: - "@polkadot/wasm-bridge" "7.0.3" - "@polkadot/wasm-crypto-asmjs" "7.0.3" - "@polkadot/wasm-crypto-init" "7.0.3" - "@polkadot/wasm-crypto-wasm" "7.0.3" - "@polkadot/wasm-util" "7.0.3" - tslib "^2.5.0" - "@polkadot/wasm-util@6.3.1": version "6.3.1" resolved "https://registry.yarnpkg.com/@polkadot/wasm-util/-/wasm-util-6.3.1.tgz#439ebb68a436317af388ed6438b8f879df3afcda" @@ -3709,13 +3613,6 @@ dependencies: "@babel/runtime" "^7.20.6" -"@polkadot/wasm-util@7.0.3": - version "7.0.3" - resolved "https://registry.yarnpkg.com/@polkadot/wasm-util/-/wasm-util-7.0.3.tgz#eab59f9dac0f00ca736aff8b24925108b7b2f860" - integrity sha512-L9U5nSbzr5xa2YSpveP/zZxhOB6i8ibssK+ihuG+7SICYtTC0B9wJp/UnjP/c6bEDlMV3yWiNXJPBTJMGmkmIQ== - dependencies: - tslib "^2.5.0" - "@polkadot/x-bigint@10.4.2": version "10.4.2" resolved "https://registry.yarnpkg.com/@polkadot/x-bigint/-/x-bigint-10.4.2.tgz#7eb2ec732259df48b5a00f07879a1331e05606ec" @@ -3724,14 +3621,6 @@ "@babel/runtime" "^7.20.13" "@polkadot/x-global" "10.4.2" -"@polkadot/x-bigint@11.1.1": - version "11.1.1" - resolved "https://registry.yarnpkg.com/@polkadot/x-bigint/-/x-bigint-11.1.1.tgz#f23fb17b6d1267cb7f4fbad11a04b67ec26a14c6" - integrity sha512-iLaaPSCnVuZ7LoOWZTHgs+Ebws0MdoNHmXoTriU60YLoojDJbcOInlO+1h3fNy6oPnYN3qA3Ml1mKDnP837nxg== - dependencies: - "@polkadot/x-global" "11.1.1" - tslib "^2.5.0" - "@polkadot/x-bigint@8.2.2": version "8.2.2" resolved "https://registry.yarnpkg.com/@polkadot/x-bigint/-/x-bigint-8.2.2.tgz#24f4e1b889a1ee899ac9c98fee526e44e2130c14" @@ -3775,13 +3664,6 @@ dependencies: "@babel/runtime" "^7.20.13" -"@polkadot/x-global@11.1.1": - version "11.1.1" - resolved "https://registry.yarnpkg.com/@polkadot/x-global/-/x-global-11.1.1.tgz#6862c6c679e94b0fc39dff3b6954bb508ff3d7a4" - integrity sha512-++LFUT98bi2m15w8LrgOcpE5mi9bmH65YB02xbKzU0ZHe1g5l0LwFt+QFB9tZlNqfWTgwpsFshGtvdPQqrFnKw== - dependencies: - tslib "^2.5.0" - "@polkadot/x-global@6.11.1", "@polkadot/x-global@^6.0.5": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/x-global/-/x-global-6.11.1.tgz#c292b3825fea60e9b33fff1790323fc57de1ca5d" @@ -3818,14 +3700,6 @@ "@babel/runtime" "^7.20.13" "@polkadot/x-global" "10.4.2" -"@polkadot/x-randomvalues@11.1.1": - version "11.1.1" - resolved "https://registry.yarnpkg.com/@polkadot/x-randomvalues/-/x-randomvalues-11.1.1.tgz#3d1a5f4df7cd7cd3499fd424600fc2599b90366e" - integrity sha512-t+Ag+RM/Agi8N86N73Ij1xz+87FYQLWZW+BlCdSEozTt933zloTNl4046IKj4sofZc51+ftRM3BFNmNT1UdlWQ== - dependencies: - "@polkadot/x-global" "11.1.1" - tslib "^2.5.0" - "@polkadot/x-randomvalues@6.11.1": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/x-randomvalues/-/x-randomvalues-6.11.1.tgz#f006fa250c8e82c92ccb769976a45a8e7f3df28b" @@ -3874,14 +3748,6 @@ "@babel/runtime" "^7.20.13" "@polkadot/x-global" "10.4.2" -"@polkadot/x-textdecoder@11.1.1": - version "11.1.1" - resolved "https://registry.yarnpkg.com/@polkadot/x-textdecoder/-/x-textdecoder-11.1.1.tgz#9596fe10981fa05f7629eaa198b11c8b191b1501" - integrity sha512-YoB82pr6kYkK5yg2BQgm5wVTf6Hq+01i+A6PgV1uXr7Rm3bxmQpGR2DKZq0QNjwWP0s6e91BxXvGoPjf7S9tBA== - dependencies: - "@polkadot/x-global" "11.1.1" - tslib "^2.5.0" - "@polkadot/x-textdecoder@6.11.1": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/x-textdecoder/-/x-textdecoder-6.11.1.tgz#6cc314645681cc4639085c03b65328671c7f182c" @@ -3922,14 +3788,6 @@ "@babel/runtime" "^7.20.13" "@polkadot/x-global" "10.4.2" -"@polkadot/x-textencoder@11.1.1": - version "11.1.1" - resolved "https://registry.yarnpkg.com/@polkadot/x-textencoder/-/x-textencoder-11.1.1.tgz#1c91e9aa568384595cbd7bd6ad50d71e4f92f2d0" - integrity sha512-I4IygnZeSyGUPyTmu7W2IsCHakax7QTVR9kMkCywaKEjiLzZU5B/LuDB0Gxn/3Jw2X2YfoB1TQ4mZ1bte4LX0g== - dependencies: - "@polkadot/x-global" "11.1.1" - tslib "^2.5.0" - "@polkadot/x-textencoder@6.11.1": version "6.11.1" resolved "https://registry.yarnpkg.com/@polkadot/x-textencoder/-/x-textencoder-6.11.1.tgz#73e89da5b91954ae380042c19314c90472f59d9e" @@ -4109,7 +3967,7 @@ resolved "https://registry.yarnpkg.com/@substrate/ss58-registry/-/ss58-registry-1.34.0.tgz#b6faed02343da7a8956444f5db23bc7246dd5fb5" integrity sha512-8Df5usnWvjnw/WRAmKOqHXRPPRfiCd1kIN8ttH4YmBrRTERjVInsdu0xvLdbyUYKyvgK6zKhHWQfYohXqllHhg== -"@substrate/ss58-registry@^1.38.0", "@substrate/ss58-registry@^1.39.0": +"@substrate/ss58-registry@^1.38.0": version "1.39.0" resolved "https://registry.yarnpkg.com/@substrate/ss58-registry/-/ss58-registry-1.39.0.tgz#eb916ff5fea7fa02e77745823fde21af979273d2" integrity sha512-qZYpuE6n+mwew+X71dOur/CbMXj6rNW27o63JeJwdQH/GvcSKm3JLNhd+bGzwUKg0D/zD30Qc6p4JykArzM+tA== @@ -17993,11 +17851,6 @@ tslib@^2.0.0, tslib@^2.0.1, tslib@^2.0.3, tslib@^2.1.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.3.0.tgz#803b8cdab3e12ba581a4ca41c8839bbb0dacb09e" integrity sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg== -tslib@^2.5.0: - version "2.5.0" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.5.0.tgz#42bfed86f5787aeb41d031866c8f402429e0fddf" - integrity sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg== - tslib@~2.0.1: version "2.0.3" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.0.3.tgz#8e0741ac45fc0c226e58a17bfc3e64b9bc6ca61c" From a444d8877330f036887f149ee280c96e3d42e0ed Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Tue, 21 Mar 2023 17:20:02 +0400 Subject: [PATCH 25/59] fix(docker): add package.json and yarn.lock to Dockerfile --- api/Dockerfile | 2 ++ backend/docker/crawler/Dockerfile | 2 ++ 2 files changed, 4 insertions(+) diff --git a/api/Dockerfile b/api/Dockerfile index 91a055e8..5868c953 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -2,6 +2,8 @@ FROM node:fermium WORKDIR /usr/app/api +COPY ./package.json /usr/app +COPY ./yarn.lock /usr/app COPY ./api /usr/app/api COPY ./db /usr/app/db diff --git a/backend/docker/crawler/Dockerfile b/backend/docker/crawler/Dockerfile index c53bc115..8e5eb367 100644 --- a/backend/docker/crawler/Dockerfile +++ b/backend/docker/crawler/Dockerfile @@ -2,6 +2,8 @@ FROM node:fermium WORKDIR /usr/app/crawler +COPY ./package.json /usr/app +COPY ./yarn.lock /usr/app COPY ./backend /usr/app/crawler COPY ./db /usr/app/db From 4a87560c29e358fdfbf82d1274562dc6d3b7df10 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Tue, 21 Mar 2023 17:33:07 +0400 Subject: [PATCH 26/59] chore(*): Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47bf6efc..0a509dc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ ## vNext - ... +## v0.25.1 +- [BE] Add package.json and yarn.lock to Dockerfile + ## v0.25.0 - [BE] Add ss58 prefix migration for first priority data From 83b1b3612822c625f5469331fee018e543a7f789 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Tue, 21 Mar 2023 18:01:19 +0400 Subject: [PATCH 27/59] chore(*): Update CHANGELOG.md --- CHANGELOG.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2de75702..3820554e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,6 @@ ## vNext - [BE] Add ss58 prefix migration for first priority data - -## v0.25.1 - [BE] Add package.json and yarn.lock to Dockerfile ## v0.25.0 From bde0adbfff8c2b5d14bbee6b5064b62e8cb9ccfe Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 17:57:10 +0100 Subject: [PATCH 28/59] decrease amount of logs --- db/migrations/20230227122704-ss58-prefix.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 6faf6a9a..3f6a00af 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -35,7 +35,9 @@ class Logger { if (currentRow === totalRows) { console.log(`✅ Done for ${tableName} table`); } else { - console.log(`⏳ Migrated ${Math.trunc(currentRow / totalRows * 10000) / 100}% for ${tableName} table`); + if (currentRow % 150 === 0) { + console.log(`⏳ Migrated ${Math.trunc(currentRow / totalRows * 10000) / 100}% for ${tableName} table`); + } } } } From 5e0d0ef6904b12305958ac7c6dbc69603207e638 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 18:01:12 +0100 Subject: [PATCH 29/59] removed frontend build to speed up the process --- .github/workflows/dev.yaml | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/.github/workflows/dev.yaml b/.github/workflows/dev.yaml index 42966720..11b16214 100644 --- a/.github/workflows/dev.yaml +++ b/.github/workflows/dev.yaml @@ -29,19 +29,3 @@ jobs: repository: stats-crawler file: ./backend/docker/crawler/Dockerfile secrets: inherit - - build_and_deploy_frontend: - uses: Cerebellum-Network/reusable-workflows/.github/workflows/build-and-upload-static.yaml@1.0.0 - with: - runs-on: '["self-hosted", "cere-network-large-workers"]' - build_container: 'node:14-buster-slim' - deploy_container: 'ubuntu:20.04' - install_packages_command: 'cp -rf frontend/frontend.config-cere-dev.js frontend/frontend.config.js; yarn install' - build_command: 'yarn workspace frontend generate' - path_to_static_files_to_upload: 'frontend/dist' - secrets: - NETWORK_AWS_ACCESS_KEY_ID: ${{ secrets.DEV_NETWORK_AWS_ACCESS_KEY_ID }} - NETWORK_AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_NETWORK_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: ${{ secrets.AWS_REGION }} - S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME_DEV }} - CF_DISTRIBUTION_ID: ${{ secrets.CF_DISTRIBUTION_ID_DEV }} From 4905a964f117e0dec98e5471a47cdb91b16e30d0 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 18:07:03 +0100 Subject: [PATCH 30/59] added unique log --- backend/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/index.js b/backend/index.js index 71ee76a6..9381985b 100644 --- a/backend/index.js +++ b/backend/index.js @@ -48,7 +48,7 @@ const runCrawler = async ({ crawler, name }) => { }; const runCrawlers = async () => { - logger.info('Running migrations'); + logger.info('Running migrations 1'); await DBMigrate.getInstance(true, { env: process.env.NODE_ENV || 'local', config: '../db/database.json', From 15f20d5cb83977e0926e7f3626aae69b9e3943f4 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 18:46:39 +0100 Subject: [PATCH 31/59] removed crawler job --- .github/workflows/dev.yaml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/dev.yaml b/.github/workflows/dev.yaml index 11b16214..43c28067 100644 --- a/.github/workflows/dev.yaml +++ b/.github/workflows/dev.yaml @@ -18,14 +18,4 @@ jobs: aws_account_id: ${{ vars.DEV_NETWORK_AWS_ACCOUNT_ID }} repository: stats-api file: ./api/Dockerfile - secrets: inherit - - build_backend_cerestats_crawler: - uses: Cerebellum-Network/reusable-workflows/.github/workflows/deploy-to-ecr.yaml@1.3.0 - with: - runs-on: '["self-hosted", "cere-network-large-workers"]' - environment: dev - aws_account_id: ${{ vars.DEV_NETWORK_AWS_ACCOUNT_ID }} - repository: stats-crawler - file: ./backend/docker/crawler/Dockerfile - secrets: inherit + secrets: inherit \ No newline at end of file From 87bf8426be9e882338a808c62fd5c0a11863b4a4 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 18:47:26 +0100 Subject: [PATCH 32/59] removed migration from crawler --- backend/index.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/backend/index.js b/backend/index.js index 9381985b..eae9732f 100644 --- a/backend/index.js +++ b/backend/index.js @@ -1,6 +1,5 @@ // @ts-check const express = require('express'); -const DBMigrate = require('db-migrate'); const pino = require('pino'); const { spawn } = require('child_process'); const { StatusCodes } = require('http-status-codes'); @@ -48,16 +47,6 @@ const runCrawler = async ({ crawler, name }) => { }; const runCrawlers = async () => { - logger.info('Running migrations 1'); - await DBMigrate.getInstance(true, { - env: process.env.NODE_ENV || 'local', - config: '../db/database.json', - cmdOptions: { - 'migrations-dir': '../db/migrations', - }, - }).up(); - logger.info('Migrations completed'); - logger.info('Starting backend, waiting 10s...'); await wait(10000); From 92dc0a46d1d018788c338cce5856f3ff6daf1487 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 18:49:55 +0100 Subject: [PATCH 33/59] Revert "removed crawler job" This reverts commit 15f20d5cb83977e0926e7f3626aae69b9e3943f4. --- .github/workflows/dev.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dev.yaml b/.github/workflows/dev.yaml index 43c28067..11b16214 100644 --- a/.github/workflows/dev.yaml +++ b/.github/workflows/dev.yaml @@ -18,4 +18,14 @@ jobs: aws_account_id: ${{ vars.DEV_NETWORK_AWS_ACCOUNT_ID }} repository: stats-api file: ./api/Dockerfile - secrets: inherit \ No newline at end of file + secrets: inherit + + build_backend_cerestats_crawler: + uses: Cerebellum-Network/reusable-workflows/.github/workflows/deploy-to-ecr.yaml@1.3.0 + with: + runs-on: '["self-hosted", "cere-network-large-workers"]' + environment: dev + aws_account_id: ${{ vars.DEV_NETWORK_AWS_ACCOUNT_ID }} + repository: stats-crawler + file: ./backend/docker/crawler/Dockerfile + secrets: inherit From 02b36dfe27f71997d0f985791c99e3b150719119 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 19:27:10 +0100 Subject: [PATCH 34/59] enabled only one table --- db/migrations/20230227122704-ss58-prefix.js | 51 ++++++++++++--------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 3f6a00af..2517939c 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -57,29 +57,38 @@ const decode = (address, ss58) => { const convertAccountTable = (db, ss58Format) => { const logger = new Logger('account'); - db.runSql('SELECT account_id, balances, identity from account;', (_, result) => { - logger.setTotalRows(result.rows.length); - result.rows.forEach(({account_id, balances, identity}) => { + return new Promise(function(resolve, reject) { + db.runSql('SELECT account_id, balances, identity from account;', (_, result) => { + logger.setTotalRows(result.rows.length); - const nextAddress = decode(account_id, ss58Format); + for (let i = 0; i < result.rows.length; i++) { + const {account_id, balances, identity} = result.rows[i]; - const nextBalances = balances.replace(account_id, nextAddress); + const nextAddress = decode(account_id, ss58Format); - let queryString = `UPDATE account SET account_id='${nextAddress}', balances='${nextBalances}'` + const nextBalances = balances.replace(account_id, nextAddress); - if (identity?.includes("parent")) { - const currentParent = JSON.parse(identity).parent; + let queryString = `UPDATE account SET account_id='${nextAddress}', balances='${nextBalances}'` - const nextParent = decode(currentParent, ss58Format); + if (identity?.includes("parent")) { + const currentParent = JSON.parse(identity).parent; - const nextIdentity = identity.replace(currentParent, nextParent); + const nextParent = decode(currentParent, ss58Format); - queryString += `, identity='${nextIdentity}'` - } + const nextIdentity = identity.replace(currentParent, nextParent); - queryString += ` WHERE account_id='${account_id}';` + queryString += `, identity='${nextIdentity}'` + } - db.runSql(queryString, logger.log); + queryString += ` WHERE account_id='${account_id}';` + + db.runSql(queryString, () => { + logger.log(); + if (i === result.rows.length - 1) { + resolve(); + } + }); + } }); }); } @@ -197,21 +206,21 @@ const convertTransfers = (db, ss58Format) => { }; const convertTables = (db, prefix) => { - convertAccountTable(db, prefix); - convertEraTables(db, prefix); - convertRankingTable(db, prefix); - convertFaucetTable(db, prefix); - convertTransfers(db, prefix); + return convertAccountTable(db, prefix); + // convertEraTables(db, prefix); + // convertRankingTable(db, prefix); + // convertFaucetTable(db, prefix); + // convertTransfers(db, prefix); } exports.up = async function (db) { const newSs58Prefix = 54; - convertTables(db, newSs58Prefix); + return convertTables(db, newSs58Prefix); }; exports.down = async function (db) { const oldSs58Prefix = 42; - convertTables(db, oldSs58Prefix); + return convertTables(db, oldSs58Prefix); }; exports._meta = { From cd40e744cc9a1854805cc431a97ef55643056b48 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 22:14:32 +0100 Subject: [PATCH 35/59] added error log --- db/migrations/20230227122704-ss58-prefix.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 2517939c..2b26b9e1 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -82,8 +82,11 @@ const convertAccountTable = (db, ss58Format) => { queryString += ` WHERE account_id='${account_id}';` - db.runSql(queryString, () => { + db.runSql(queryString, (e, r) => { logger.log(); + if (e !== null) { + console.log(`${e} + ${JSON.stringify(r)}`); + } if (i === result.rows.length - 1) { resolve(); } From c896c398c642939d923db704ca9a03195fcf159a Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 22:34:34 +0100 Subject: [PATCH 36/59] added empty migration --- db/migrations/20230227122704-ss58-prefix.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 2b26b9e1..461f0c37 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -209,7 +209,7 @@ const convertTransfers = (db, ss58Format) => { }; const convertTables = (db, prefix) => { - return convertAccountTable(db, prefix); + // return convertAccountTable(db, prefix); // convertEraTables(db, prefix); // convertRankingTable(db, prefix); // convertFaucetTable(db, prefix); From 73e5a8070ef40cff5e6fcdcbfdf524d218a14e4f Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 22:53:09 +0100 Subject: [PATCH 37/59] migrate the first row only --- db/migrations/20230227122704-ss58-prefix.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index 461f0c37..b55f0c61 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -62,6 +62,7 @@ const convertAccountTable = (db, ss58Format) => { logger.setTotalRows(result.rows.length); for (let i = 0; i < result.rows.length; i++) { + for (let i = 0; i < 1; i++) { const {account_id, balances, identity} = result.rows[i]; const nextAddress = decode(account_id, ss58Format); @@ -82,6 +83,8 @@ const convertAccountTable = (db, ss58Format) => { queryString += ` WHERE account_id='${account_id}';` + console.log(`Query is ${queryString}`); + db.runSql(queryString, (e, r) => { logger.log(); if (e !== null) { @@ -209,7 +212,7 @@ const convertTransfers = (db, ss58Format) => { }; const convertTables = (db, prefix) => { - // return convertAccountTable(db, prefix); + return convertAccountTable(db, prefix); // convertEraTables(db, prefix); // convertRankingTable(db, prefix); // convertFaucetTable(db, prefix); From 29fb1b4e355a9ef1a09c557451d3749e2fd7b795 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 23:06:01 +0100 Subject: [PATCH 38/59] migrate the first row only --- db/migrations/20230227122704-ss58-prefix.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-ss58-prefix.js index b55f0c61..e10d8e7a 100644 --- a/db/migrations/20230227122704-ss58-prefix.js +++ b/db/migrations/20230227122704-ss58-prefix.js @@ -61,7 +61,7 @@ const convertAccountTable = (db, ss58Format) => { db.runSql('SELECT account_id, balances, identity from account;', (_, result) => { logger.setTotalRows(result.rows.length); - for (let i = 0; i < result.rows.length; i++) { + // for (let i = 0; i < result.rows.length; i++) { for (let i = 0; i < 1; i++) { const {account_id, balances, identity} = result.rows[i]; @@ -90,6 +90,7 @@ const convertAccountTable = (db, ss58Format) => { if (e !== null) { console.log(`${e} + ${JSON.stringify(r)}`); } + resolve(); if (i === result.rows.length - 1) { resolve(); } From 8104e69b303976ca25494b21b877f8a425b881e2 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 23:06:26 +0100 Subject: [PATCH 39/59] renaming --- ...30227122704-ss58-prefix.js => 20230227122704-1-ss58-prefix.js} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename db/migrations/{20230227122704-ss58-prefix.js => 20230227122704-1-ss58-prefix.js} (100%) diff --git a/db/migrations/20230227122704-ss58-prefix.js b/db/migrations/20230227122704-1-ss58-prefix.js similarity index 100% rename from db/migrations/20230227122704-ss58-prefix.js rename to db/migrations/20230227122704-1-ss58-prefix.js From 450a2deb92a0bae3f5d019c34eb042c275236846 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 23:08:38 +0100 Subject: [PATCH 40/59] added error handling --- db/migrations/20230227122704-1-ss58-prefix.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/db/migrations/20230227122704-1-ss58-prefix.js b/db/migrations/20230227122704-1-ss58-prefix.js index e10d8e7a..265b40a2 100644 --- a/db/migrations/20230227122704-1-ss58-prefix.js +++ b/db/migrations/20230227122704-1-ss58-prefix.js @@ -89,10 +89,12 @@ const convertAccountTable = (db, ss58Format) => { logger.log(); if (e !== null) { console.log(`${e} + ${JSON.stringify(r)}`); - } - resolve(); - if (i === result.rows.length - 1) { + reject(e); + } else { resolve(); + if (i === result.rows.length - 1) { + resolve(); + } } }); } From 015b87f1c221b2c6fe1d400baf1c38d6c9fee03d Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 23:25:45 +0100 Subject: [PATCH 41/59] make async --- db/migrations/20230227122704-1-ss58-prefix.js | 71 ++++++++++--------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/db/migrations/20230227122704-1-ss58-prefix.js b/db/migrations/20230227122704-1-ss58-prefix.js index 265b40a2..843bfe7f 100644 --- a/db/migrations/20230227122704-1-ss58-prefix.js +++ b/db/migrations/20230227122704-1-ss58-prefix.js @@ -54,52 +54,55 @@ const decode = (address, ss58) => { }; // Migration script for Account Table -const convertAccountTable = (db, ss58Format) => { +const convertAccountTable = async (db, ss58Format) => { const logger = new Logger('account'); - return new Promise(function(resolve, reject) { - db.runSql('SELECT account_id, balances, identity from account;', (_, result) => { - logger.setTotalRows(result.rows.length); + const rows = await new Promise(function(resolve, reject) { + db.runSql('SELECT account_id, balances, identity from account;', (error, result) => { + if (error) { + reject(error); + } + resolve(result.rows); + })}); + logger.setTotalRows(rows.length); - // for (let i = 0; i < result.rows.length; i++) { - for (let i = 0; i < 1; i++) { - const {account_id, balances, identity} = result.rows[i]; + for (let i = 0; i < rows.length; i++) { + console.log(i); - const nextAddress = decode(account_id, ss58Format); + const {account_id, balances, identity} = rows[i]; - const nextBalances = balances.replace(account_id, nextAddress); + const nextAddress = decode(account_id, ss58Format); - let queryString = `UPDATE account SET account_id='${nextAddress}', balances='${nextBalances}'` + const nextBalances = balances.replace(account_id, nextAddress); - if (identity?.includes("parent")) { - const currentParent = JSON.parse(identity).parent; + let queryString = `UPDATE account SET account_id='${nextAddress}', balances='${nextBalances}'` - const nextParent = decode(currentParent, ss58Format); + if (identity?.includes("parent")) { + const currentParent = JSON.parse(identity).parent; - const nextIdentity = identity.replace(currentParent, nextParent); + const nextParent = decode(currentParent, ss58Format); - queryString += `, identity='${nextIdentity}'` - } + const nextIdentity = identity.replace(currentParent, nextParent); - queryString += ` WHERE account_id='${account_id}';` - - console.log(`Query is ${queryString}`); - - db.runSql(queryString, (e, r) => { - logger.log(); - if (e !== null) { - console.log(`${e} + ${JSON.stringify(r)}`); - reject(e); - } else { - resolve(); - if (i === result.rows.length - 1) { - resolve(); - } - } - }); - } + queryString += `, identity='${nextIdentity}'` + } + + queryString += ` WHERE account_id='${account_id}';` + + console.log(`Query is ${queryString}`); + + const updateResult = await new Promise(function (resolve, reject) { + db.runSql(queryString, (e, r) => { + if (e) { + console.log(`${e} + ${JSON.stringify(r)}`); + reject(e); + } else { + resolve(r); + } + }); }); - }); + logger.log(); + } } // Migration script for featured, era_commission, era_points, era_relative_performance, era_self_stake, era_vrc_score tables From 36ab7b403881f5898d3d79f6b0f251691eda7084 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 23:27:05 +0100 Subject: [PATCH 42/59] renaming --- ...227122704-1-ss58-prefix.js => 20230227122704-2-ss58-prefix.js} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename db/migrations/{20230227122704-1-ss58-prefix.js => 20230227122704-2-ss58-prefix.js} (100%) diff --git a/db/migrations/20230227122704-1-ss58-prefix.js b/db/migrations/20230227122704-2-ss58-prefix.js similarity index 100% rename from db/migrations/20230227122704-1-ss58-prefix.js rename to db/migrations/20230227122704-2-ss58-prefix.js From a15c071f3c5a6e93de8b0bb578d98da82311421b Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Tue, 21 Mar 2023 23:59:39 +0100 Subject: [PATCH 43/59] refactoring --- db/migrations/20230227122704-2-ss58-prefix.js | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/db/migrations/20230227122704-2-ss58-prefix.js b/db/migrations/20230227122704-2-ss58-prefix.js index 843bfe7f..7b0b817d 100644 --- a/db/migrations/20230227122704-2-ss58-prefix.js +++ b/db/migrations/20230227122704-2-ss58-prefix.js @@ -57,18 +57,10 @@ const decode = (address, ss58) => { const convertAccountTable = async (db, ss58Format) => { const logger = new Logger('account'); - const rows = await new Promise(function(resolve, reject) { - db.runSql('SELECT account_id, balances, identity from account;', (error, result) => { - if (error) { - reject(error); - } - resolve(result.rows); - })}); + const rows = await executeDbRunSqlAsPromise(db, 'SELECT account_id, balances, identity from account;'); logger.setTotalRows(rows.length); for (let i = 0; i < rows.length; i++) { - console.log(i); - const {account_id, balances, identity} = rows[i]; const nextAddress = decode(account_id, ss58Format); @@ -89,18 +81,8 @@ const convertAccountTable = async (db, ss58Format) => { queryString += ` WHERE account_id='${account_id}';` - console.log(`Query is ${queryString}`); + await executeDbRunSqlAsPromise(db, queryString); - const updateResult = await new Promise(function (resolve, reject) { - db.runSql(queryString, (e, r) => { - if (e) { - console.log(`${e} + ${JSON.stringify(r)}`); - reject(e); - } else { - resolve(r); - } - }); - }); logger.log(); } } @@ -225,6 +207,17 @@ const convertTables = (db, prefix) => { // convertTransfers(db, prefix); } +const executeDbRunSqlAsPromise = (db, sqlQuery) => { + return new Promise(function (resolve, reject) { + db.runSql(sqlQuery, (error, result) => { + if (error) { + reject(error); + } + resolve(result); + }) + }); +} + exports.up = async function (db) { const newSs58Prefix = 54; return convertTables(db, newSs58Prefix); From d04f63759b948207c8cc17884f3f09cb41b2e0eb Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Wed, 22 Mar 2023 00:04:11 +0100 Subject: [PATCH 44/59] added clean up --- db/migrations/20230227122704-2-ss58-prefix.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db/migrations/20230227122704-2-ss58-prefix.js b/db/migrations/20230227122704-2-ss58-prefix.js index 7b0b817d..fbedc6c1 100644 --- a/db/migrations/20230227122704-2-ss58-prefix.js +++ b/db/migrations/20230227122704-2-ss58-prefix.js @@ -55,6 +55,8 @@ const decode = (address, ss58) => { // Migration script for Account Table const convertAccountTable = async (db, ss58Format) => { + const deleteResult = await executeDbRunSqlAsPromise(db, "DELETE FROM account WHERE account_id LIKE '6%'"); + const logger = new Logger('account'); const rows = await executeDbRunSqlAsPromise(db, 'SELECT account_id, balances, identity from account;'); From fd2a8a186f9282ff0ffdb6ed9a3d88225ef91e7c Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Wed, 22 Mar 2023 00:17:02 +0100 Subject: [PATCH 45/59] refactoring --- db/migrations/20230227122704-2-ss58-prefix.js | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/db/migrations/20230227122704-2-ss58-prefix.js b/db/migrations/20230227122704-2-ss58-prefix.js index fbedc6c1..32917603 100644 --- a/db/migrations/20230227122704-2-ss58-prefix.js +++ b/db/migrations/20230227122704-2-ss58-prefix.js @@ -90,7 +90,7 @@ const convertAccountTable = async (db, ss58Format) => { } // Migration script for featured, era_commission, era_points, era_relative_performance, era_self_stake, era_vrc_score tables -const convertEraTables = (db, ss58Format) => { +const convertEraTables = async (db, ss58Format) => { const tables = [ 'featured', 'era_commission', @@ -99,24 +99,27 @@ const convertEraTables = (db, ss58Format) => { 'era_self_stake', 'era_vrc_score', ]; - tables.forEach((table) => { - const logger = new Logger(table); - db.runSql(`SELECT DISTINCT stash_address from ${table}`, (_, result) => { - logger.setTotalRows(result.rows.length); + for (let i = 0; i < tables.length; i++) { + const table = tables[i]; - result.rows.forEach(({stash_address}) => { + const logger = new Logger(table); - const nextAddress = decode(stash_address, ss58Format) + const rows = await executeDbRunSqlAsPromise(db, `SELECT DISTINCT stash_address from ${table}`); + logger.setTotalRows(rows.length); - const queryString = `UPDATE ${table} SET stash_address='${nextAddress}' WHERE stash_address='${stash_address}';` + for (let j = 0; j < rows.length; j++) { + const {stash_address} = rows[j]; - db.runSql(queryString, logger.log); + const nextAddress = decode(stash_address, ss58Format) - }); + const queryString = `UPDATE ${table} SET stash_address='${nextAddress}' WHERE stash_address='${stash_address}';` - }); - }); + await executeDbRunSqlAsPromise(db, queryString); + + logger.log(); + } + } }; const convertRankingTable = (db, ss58Format) => { From 6a8ce7144339967cac4b7b501c3c21d632a97cda Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Wed, 22 Mar 2023 00:25:53 +0100 Subject: [PATCH 46/59] refactoring --- ...fix.js => 20230227122704-3-ss58-prefix.js} | 178 +++++++++--------- 1 file changed, 91 insertions(+), 87 deletions(-) rename db/migrations/{20230227122704-2-ss58-prefix.js => 20230227122704-3-ss58-prefix.js} (58%) diff --git a/db/migrations/20230227122704-2-ss58-prefix.js b/db/migrations/20230227122704-3-ss58-prefix.js similarity index 58% rename from db/migrations/20230227122704-2-ss58-prefix.js rename to db/migrations/20230227122704-3-ss58-prefix.js index 32917603..e8281b99 100644 --- a/db/migrations/20230227122704-2-ss58-prefix.js +++ b/db/migrations/20230227122704-3-ss58-prefix.js @@ -17,42 +17,6 @@ exports.setup = function (options, seedLink) { Promise = options.Promise; }; -class Logger { - constructor(tableName) { - this.tableName = tableName; - this.totalRows = 0; - this.currentRow = 0; - } - - setTotalRows = (totalRows) => { - this.totalRows = totalRows; - } - - log = () => { - this.currentRow++; - const {currentRow, totalRows, tableName} = this; - - if (currentRow === totalRows) { - console.log(`✅ Done for ${tableName} table`); - } else { - if (currentRow % 150 === 0) { - console.log(`⏳ Migrated ${Math.trunc(currentRow / totalRows * 10000) / 100}% for ${tableName} table`); - } - } - } -} - -const decode = (address, ss58) => { - if (address.startsWith('0x')) { - console.log('Address starts with 0x', address); - return address; - } else { - return keyring.encodeAddress( - keyring.decodeAddress(address), - ss58); - } -}; - // Migration script for Account Table const convertAccountTable = async (db, ss58Format) => { const deleteResult = await executeDbRunSqlAsPromise(db, "DELETE FROM account WHERE account_id LIKE '6%'"); @@ -122,96 +86,136 @@ const convertEraTables = async (db, ss58Format) => { } }; -const convertRankingTable = (db, ss58Format) => { +const convertRankingTable = async (db, ss58Format) => { const logger = new Logger('ranking'); - db.runSql('SELECT identity, stash_address, controller_address, rank from ranking;', (_, result) => { - logger.setTotalRows(result.rows.length); + const rows = await executeDbRunSqlAsPromise(db, 'SELECT identity, stash_address, controller_address, rank from ranking;'); + logger.setTotalRows(rows.length); - result.rows.forEach(({identity, stash_address, controller_address, rank}) => { + for (let i = 0; i < rows.length; i++) { + const {identity, stash_address, controller_address, rank} = rows[i]; - const nextStashAddress = decode(stash_address, ss58Format); + const nextStashAddress = decode(stash_address, ss58Format); - const nextControllerAddress = decode(controller_address, ss58Format); + const nextControllerAddress = decode(controller_address, ss58Format); - let queryString = `UPDATE ranking SET stash_address='${nextStashAddress}', controller_address='${nextControllerAddress}'` + let queryString = `UPDATE ranking SET stash_address='${nextStashAddress}', controller_address='${nextControllerAddress}'` - if (identity?.includes("parent")) { - const currentParent = JSON.parse(identity).parent; + if (identity?.includes("parent")) { + const currentParent = JSON.parse(identity).parent; - const nextParent = decode(currentParent, ss58Format); + const nextParent = decode(currentParent, ss58Format); - const nextIdentity = identity.replace(currentParent, nextParent); + const nextIdentity = identity.replace(currentParent, nextParent); - queryString += `, identity='${nextIdentity}'` - } + queryString += `, identity='${nextIdentity}'` + } - queryString += ` WHERE rank='${rank}'`; + queryString += ` WHERE rank='${rank}'`; - db.runSql(queryString, logger.log); - }); + await executeDbRunSqlAsPromise(db, queryString); - }); + logger.log(); + } } -const convertFaucetTable = (db, ss58Format) => { +const convertFaucetTable = async (db, ss58Format) => { const logger = new Logger('faucet'); - db.runSql('SELECT id, sender, destination from faucet;', (_, result) => { - logger.setTotalRows(result.rows.length); + const rows = await executeDbRunSqlAsPromise(db, 'SELECT id, sender, destination from faucet;'); + logger.setTotalRows(rows.length); - result.rows.forEach(({id, sender, destination}) => { + for (let i = 0; i < rows.length; i++) { + const {id, sender, destination} = rows[i]; - const nextSender = decode(sender, ss58Format); + const nextSender = decode(sender, ss58Format); - const nextDestination = decode(destination, ss58Format); + const nextDestination = decode(destination, ss58Format); - const queryString = `UPDATE faucet SET sender='${nextSender}', destination='${nextDestination}' WHERE id='${id}';` + const queryString = `UPDATE faucet SET sender='${nextSender}', destination='${nextDestination}' WHERE id='${id}';` - db.runSql(queryString, logger.log); - }); + await executeDbRunSqlAsPromise(db, queryString); - }); + logger.log(); + } } -// Transfers are a part of extrinsict table -const convertTransfers = (db, ss58Format) => { +// Transfers are a part of extrinsics table +const convertTransfers = async (db, ss58Format) => { const logger = new Logger('transfer'); + const rows = await executeDbRunSqlAsPromise(db, `select block_number, signer, args, method from extrinsic where method like 'transfer%'`); - db.runSql(`select block_number, signer, args, method from extrinsic where method like 'transfer%'`, (_, result) => { - logger.setTotalRows(result.rows.length); + logger.setTotalRows(rows.length); - result.rows.forEach(({block_number, signer, args, method}) => { - const nextSigner = decode(signer, ss58Format); + for (let i = 0; i < rows.length; i++) { + const {block_number, signer, args, method} = rows[i]; - let queryString = `UPDATE extrinsic SET signer='${nextSigner}'` + const nextSigner = decode(signer, ss58Format); - if (method === 'transfer' || method === 'transferKeepAlive' || method === 'transferAll') { - const [account] = JSON.parse(args); - const accountId = typeof account === 'string' ? account : account.id ? account.id : account.address20; + let queryString = `UPDATE extrinsic SET signer='${nextSigner}'` - const nextAccountId = decode(accountId, ss58Format); + if (method === 'transfer' || method === 'transferKeepAlive' || method === 'transferAll') { + const [account] = JSON.parse(args); + const accountId = typeof account === 'string' ? account : account.id ? account.id : account.address20; - const nextArgs = args.replace(accountId, nextAccountId); - queryString += `, args='${nextArgs}'`; - } + const nextAccountId = decode(accountId, ss58Format); - queryString += ` WHERE block_number='${block_number}';` + const nextArgs = args.replace(accountId, nextAccountId); + queryString += `, args='${nextArgs}'`; + } - db.runSql(queryString, logger.log); - }); + queryString += ` WHERE block_number='${block_number}';` - }); + await executeDbRunSqlAsPromise(db, queryString); + + logger.log(); + } }; -const convertTables = (db, prefix) => { - return convertAccountTable(db, prefix); - // convertEraTables(db, prefix); - // convertRankingTable(db, prefix); - // convertFaucetTable(db, prefix); - // convertTransfers(db, prefix); +const convertTables = async (db, prefix) => { + await convertAccountTable(db, prefix); + await convertEraTables(db, prefix); + await convertRankingTable(db, prefix); + await convertFaucetTable(db, prefix); + await convertTransfers(db, prefix); +} + +class Logger { + constructor(tableName) { + this.tableName = tableName; + this.totalRows = 0; + this.currentRow = 0; + } + + setTotalRows = (totalRows) => { + this.totalRows = totalRows; + } + + log = () => { + this.currentRow++; + const {currentRow, totalRows, tableName} = this; + + if (currentRow === totalRows) { + console.log(`✅ Done for ${tableName} table`); + } else { + if (currentRow % 150 === 0) { + console.log(`⏳ Migrated ${Math.trunc(currentRow / totalRows * 10000) / 100}% for ${tableName} table`); + } + } + } } +const decode = (address, ss58) => { + if (address.startsWith('0x')) { + console.log('Address starts with 0x', address); + return address; + } else { + return keyring.encodeAddress( + keyring.decodeAddress(address), + ss58); + } +}; + const executeDbRunSqlAsPromise = (db, sqlQuery) => { return new Promise(function (resolve, reject) { db.runSql(sqlQuery, (error, result) => { From 20ca60d94211bddf54a35b3bcfe5a1d139856651 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Wed, 22 Mar 2023 00:35:11 +0100 Subject: [PATCH 47/59] refactoring --- db/migrations/20230227122704-3-ss58-prefix.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/db/migrations/20230227122704-3-ss58-prefix.js b/db/migrations/20230227122704-3-ss58-prefix.js index e8281b99..1761d431 100644 --- a/db/migrations/20230227122704-3-ss58-prefix.js +++ b/db/migrations/20230227122704-3-ss58-prefix.js @@ -23,7 +23,8 @@ const convertAccountTable = async (db, ss58Format) => { const logger = new Logger('account'); - const rows = await executeDbRunSqlAsPromise(db, 'SELECT account_id, balances, identity from account;'); + const result = await executeDbRunSqlAsPromise(db, 'SELECT account_id, balances, identity from account;'); + const rows = result.rows; logger.setTotalRows(rows.length); for (let i = 0; i < rows.length; i++) { @@ -69,7 +70,8 @@ const convertEraTables = async (db, ss58Format) => { const logger = new Logger(table); - const rows = await executeDbRunSqlAsPromise(db, `SELECT DISTINCT stash_address from ${table}`); + const result = await executeDbRunSqlAsPromise(db, `SELECT DISTINCT stash_address from ${table}`); + const rows = result.rows; logger.setTotalRows(rows.length); for (let j = 0; j < rows.length; j++) { @@ -89,7 +91,8 @@ const convertEraTables = async (db, ss58Format) => { const convertRankingTable = async (db, ss58Format) => { const logger = new Logger('ranking'); - const rows = await executeDbRunSqlAsPromise(db, 'SELECT identity, stash_address, controller_address, rank from ranking;'); + const result = await executeDbRunSqlAsPromise(db, 'SELECT identity, stash_address, controller_address, rank from ranking;'); + const rows = result.rows; logger.setTotalRows(rows.length); for (let i = 0; i < rows.length; i++) { @@ -122,7 +125,8 @@ const convertRankingTable = async (db, ss58Format) => { const convertFaucetTable = async (db, ss58Format) => { const logger = new Logger('faucet'); - const rows = await executeDbRunSqlAsPromise(db, 'SELECT id, sender, destination from faucet;'); + const result = await executeDbRunSqlAsPromise(db, 'SELECT id, sender, destination from faucet;'); + const rows = result.rows; logger.setTotalRows(rows.length); for (let i = 0; i < rows.length; i++) { @@ -143,8 +147,9 @@ const convertFaucetTable = async (db, ss58Format) => { // Transfers are a part of extrinsics table const convertTransfers = async (db, ss58Format) => { const logger = new Logger('transfer'); - const rows = await executeDbRunSqlAsPromise(db, `select block_number, signer, args, method from extrinsic where method like 'transfer%'`); + const result = await executeDbRunSqlAsPromise(db, `select block_number, signer, args, method from extrinsic where method like 'transfer%'`); + const rows = result.rows; logger.setTotalRows(rows.length); for (let i = 0; i < rows.length; i++) { @@ -185,10 +190,12 @@ class Logger { this.tableName = tableName; this.totalRows = 0; this.currentRow = 0; + console.log(`Migrating ${this.tableName}`); } setTotalRows = (totalRows) => { this.totalRows = totalRows; + console.log(`Total Rows are ${this.tableName}`); } log = () => { From 9bb8c954fc0d2d25514a87447f09f3d478aab9f7 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Wed, 22 Mar 2023 13:11:49 +0400 Subject: [PATCH 48/59] feat(ss85-migration): add deleting new accounts for ranking table --- db/migrations/20230227122704-3-ss58-prefix.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db/migrations/20230227122704-3-ss58-prefix.js b/db/migrations/20230227122704-3-ss58-prefix.js index 1761d431..0688a0ef 100644 --- a/db/migrations/20230227122704-3-ss58-prefix.js +++ b/db/migrations/20230227122704-3-ss58-prefix.js @@ -89,6 +89,8 @@ const convertEraTables = async (db, ss58Format) => { }; const convertRankingTable = async (db, ss58Format) => { + const deleteResult = await executeDbRunSqlAsPromise(db, "DELETE FROM ranking WHERE stash_address LIKE '6%' OR controller_address LIKE '6%'"); + const logger = new Logger('ranking'); const result = await executeDbRunSqlAsPromise(db, 'SELECT identity, stash_address, controller_address, rank from ranking;'); From e779c0ed1f96b9991eed75115dea371f62e1f621 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Wed, 22 Mar 2023 13:16:14 +0400 Subject: [PATCH 49/59] feat(ss85-migration): add deleting new accounts for featured and era_* table --- db/migrations/20230227122704-3-ss58-prefix.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db/migrations/20230227122704-3-ss58-prefix.js b/db/migrations/20230227122704-3-ss58-prefix.js index 0688a0ef..6cb6cbec 100644 --- a/db/migrations/20230227122704-3-ss58-prefix.js +++ b/db/migrations/20230227122704-3-ss58-prefix.js @@ -68,6 +68,8 @@ const convertEraTables = async (db, ss58Format) => { for (let i = 0; i < tables.length; i++) { const table = tables[i]; + const deleteResult = await executeDbRunSqlAsPromise(db, `DELETE FROM ${table} WHERE stash_address LIKE '6%'`); + const logger = new Logger(table); const result = await executeDbRunSqlAsPromise(db, `SELECT DISTINCT stash_address from ${table}`); From 4537bdb5bb5b4b38c40a5567900019563691e598 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Wed, 22 Mar 2023 13:54:37 +0400 Subject: [PATCH 50/59] feat(ss85-migration): remove deleting accounts starts with 6 --- db/migrations/20230227122704-3-ss58-prefix.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/db/migrations/20230227122704-3-ss58-prefix.js b/db/migrations/20230227122704-3-ss58-prefix.js index 6cb6cbec..0f04cdfd 100644 --- a/db/migrations/20230227122704-3-ss58-prefix.js +++ b/db/migrations/20230227122704-3-ss58-prefix.js @@ -19,8 +19,6 @@ exports.setup = function (options, seedLink) { // Migration script for Account Table const convertAccountTable = async (db, ss58Format) => { - const deleteResult = await executeDbRunSqlAsPromise(db, "DELETE FROM account WHERE account_id LIKE '6%'"); - const logger = new Logger('account'); const result = await executeDbRunSqlAsPromise(db, 'SELECT account_id, balances, identity from account;'); @@ -68,8 +66,6 @@ const convertEraTables = async (db, ss58Format) => { for (let i = 0; i < tables.length; i++) { const table = tables[i]; - const deleteResult = await executeDbRunSqlAsPromise(db, `DELETE FROM ${table} WHERE stash_address LIKE '6%'`); - const logger = new Logger(table); const result = await executeDbRunSqlAsPromise(db, `SELECT DISTINCT stash_address from ${table}`); @@ -91,8 +87,6 @@ const convertEraTables = async (db, ss58Format) => { }; const convertRankingTable = async (db, ss58Format) => { - const deleteResult = await executeDbRunSqlAsPromise(db, "DELETE FROM ranking WHERE stash_address LIKE '6%' OR controller_address LIKE '6%'"); - const logger = new Logger('ranking'); const result = await executeDbRunSqlAsPromise(db, 'SELECT identity, stash_address, controller_address, rank from ranking;'); From 72bd390c794ddfa8c50e698ee73d88427b3cb1d2 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Wed, 22 Mar 2023 13:57:14 +0400 Subject: [PATCH 51/59] feat(ss85-migration): add logging sql errros --- db/migrations/20230227122704-3-ss58-prefix.js | 1 + 1 file changed, 1 insertion(+) diff --git a/db/migrations/20230227122704-3-ss58-prefix.js b/db/migrations/20230227122704-3-ss58-prefix.js index 0f04cdfd..f02d2aff 100644 --- a/db/migrations/20230227122704-3-ss58-prefix.js +++ b/db/migrations/20230227122704-3-ss58-prefix.js @@ -225,6 +225,7 @@ const executeDbRunSqlAsPromise = (db, sqlQuery) => { return new Promise(function (resolve, reject) { db.runSql(sqlQuery, (error, result) => { if (error) { + console.log(error); reject(error); } resolve(result); From a6af46aa991aec847bae2256cbce78e28de7ad5d Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Wed, 22 Mar 2023 13:58:21 +0400 Subject: [PATCH 52/59] Revert "removed migration from crawler" This reverts commit 87bf8426be9e882338a808c62fd5c0a11863b4a4. --- backend/index.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/backend/index.js b/backend/index.js index eae9732f..9381985b 100644 --- a/backend/index.js +++ b/backend/index.js @@ -1,5 +1,6 @@ // @ts-check const express = require('express'); +const DBMigrate = require('db-migrate'); const pino = require('pino'); const { spawn } = require('child_process'); const { StatusCodes } = require('http-status-codes'); @@ -47,6 +48,16 @@ const runCrawler = async ({ crawler, name }) => { }; const runCrawlers = async () => { + logger.info('Running migrations 1'); + await DBMigrate.getInstance(true, { + env: process.env.NODE_ENV || 'local', + config: '../db/database.json', + cmdOptions: { + 'migrations-dir': '../db/migrations', + }, + }).up(); + logger.info('Migrations completed'); + logger.info('Starting backend, waiting 10s...'); await wait(10000); From 035e1f4204466b7dc781022543ddb03706236b0b Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Wed, 22 Mar 2023 14:00:15 +0400 Subject: [PATCH 53/59] Revert "added unique log" This reverts commit 4905a964f117e0dec98e5471a47cdb91b16e30d0. --- backend/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/index.js b/backend/index.js index 9381985b..71ee76a6 100644 --- a/backend/index.js +++ b/backend/index.js @@ -48,7 +48,7 @@ const runCrawler = async ({ crawler, name }) => { }; const runCrawlers = async () => { - logger.info('Running migrations 1'); + logger.info('Running migrations'); await DBMigrate.getInstance(true, { env: process.env.NODE_ENV || 'local', config: '../db/database.json', From afa53f0ab24041be285179e05138a8771ef6b2e3 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Wed, 22 Mar 2023 14:00:52 +0400 Subject: [PATCH 54/59] Revert "removed frontend build to speed up the process" This reverts commit 5e0d0ef6904b12305958ac7c6dbc69603207e638. --- .github/workflows/dev.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/dev.yaml b/.github/workflows/dev.yaml index 11b16214..42966720 100644 --- a/.github/workflows/dev.yaml +++ b/.github/workflows/dev.yaml @@ -29,3 +29,19 @@ jobs: repository: stats-crawler file: ./backend/docker/crawler/Dockerfile secrets: inherit + + build_and_deploy_frontend: + uses: Cerebellum-Network/reusable-workflows/.github/workflows/build-and-upload-static.yaml@1.0.0 + with: + runs-on: '["self-hosted", "cere-network-large-workers"]' + build_container: 'node:14-buster-slim' + deploy_container: 'ubuntu:20.04' + install_packages_command: 'cp -rf frontend/frontend.config-cere-dev.js frontend/frontend.config.js; yarn install' + build_command: 'yarn workspace frontend generate' + path_to_static_files_to_upload: 'frontend/dist' + secrets: + NETWORK_AWS_ACCESS_KEY_ID: ${{ secrets.DEV_NETWORK_AWS_ACCESS_KEY_ID }} + NETWORK_AWS_SECRET_ACCESS_KEY: ${{ secrets.DEV_NETWORK_AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ secrets.AWS_REGION }} + S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME_DEV }} + CF_DISTRIBUTION_ID: ${{ secrets.CF_DISTRIBUTION_ID_DEV }} From 6c323e172fb1b35c121f51ebab9a4133e99c8b22 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Wed, 22 Mar 2023 14:15:04 +0400 Subject: [PATCH 55/59] feat(ss85-migration): improve logger --- db/migrations/20230227122704-3-ss58-prefix.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/db/migrations/20230227122704-3-ss58-prefix.js b/db/migrations/20230227122704-3-ss58-prefix.js index f02d2aff..3e49d299 100644 --- a/db/migrations/20230227122704-3-ss58-prefix.js +++ b/db/migrations/20230227122704-3-ss58-prefix.js @@ -188,6 +188,7 @@ class Logger { this.tableName = tableName; this.totalRows = 0; this.currentRow = 0; + this.migratedPercent = 0; console.log(`Migrating ${this.tableName}`); } @@ -203,8 +204,11 @@ class Logger { if (currentRow === totalRows) { console.log(`✅ Done for ${tableName} table`); } else { - if (currentRow % 150 === 0) { - console.log(`⏳ Migrated ${Math.trunc(currentRow / totalRows * 10000) / 100}% for ${tableName} table`); + const migratedPercent = Math.trunc(100 * currentRow / totalRows); + + if (migratedPercent % 20 === 0 && migratedPercent > this.migratedPercent) { + this.migratedPercent = migratedPercent; + console.log(`⏳ Migrated ${migratedPercent}% for ${tableName} table`); } } } From 707a95ddb7660f6cb58286483f81480a40602891 Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Wed, 22 Mar 2023 14:47:40 +0400 Subject: [PATCH 56/59] feat(ss85-migration): add logging sql-query in case of error --- db/migrations/20230227122704-3-ss58-prefix.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/db/migrations/20230227122704-3-ss58-prefix.js b/db/migrations/20230227122704-3-ss58-prefix.js index 3e49d299..d556ecf6 100644 --- a/db/migrations/20230227122704-3-ss58-prefix.js +++ b/db/migrations/20230227122704-3-ss58-prefix.js @@ -229,7 +229,8 @@ const executeDbRunSqlAsPromise = (db, sqlQuery) => { return new Promise(function (resolve, reject) { db.runSql(sqlQuery, (error, result) => { if (error) { - console.log(error); + console.log('Error', error); + console.log('sqlQuery', sqlQuery); reject(error); } resolve(result); From 130c75e682a762fc9d2dfcbb09dfe4a7376a1aea Mon Sep 17 00:00:00 2001 From: Shamilkhan Akhmetzyanov Date: Wed, 22 Mar 2023 16:33:55 +0400 Subject: [PATCH 57/59] feat(transfer-page): add address20 format support --- CHANGELOG.md | 1 + frontend/mixins/commonMixin.js | 3 +++ 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3820554e..6b25dbe9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## vNext - [BE] Add ss58 prefix migration for first priority data - [BE] Add package.json and yarn.lock to Dockerfile +- [BE] Handle addresses in address20 format in Transfer Page ## v0.25.0 - [BE] Fix crawlers status restore after fail diff --git a/frontend/mixins/commonMixin.js b/frontend/mixins/commonMixin.js index b77045ac..a6e1c624 100644 --- a/frontend/mixins/commonMixin.js +++ b/frontend/mixins/commonMixin.js @@ -10,6 +10,9 @@ const base = new BigNumber(10).pow(network.tokenDecimals) export default { methods: { shortAddress(address) { + if (address.address20) { + return this.shortAddress(address.address20) + } return ( address.substring(0, 5) + '…' + address.substring(address.length - 5) ) From 2e7f09d6d3bee855631e65a992080c1107614101 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Wed, 22 Mar 2023 18:46:14 +0100 Subject: [PATCH 58/59] set version to 0.26.0 --- CHANGELOG.md | 3 +++ api/package.json | 2 +- backend/package.json | 2 +- frontend/package.json | 2 +- package.json | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b25dbe9..789cbbd0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Cere Stats ## vNext +- ... + +## v0.26.0 - [BE] Add ss58 prefix migration for first priority data - [BE] Add package.json and yarn.lock to Dockerfile - [BE] Handle addresses in address20 format in Transfer Page diff --git a/api/package.json b/api/package.json index e907e296..a7377072 100644 --- a/api/package.json +++ b/api/package.json @@ -1,6 +1,6 @@ { "name": "api", - "version": "0.25.0", + "version": "0.26.0", "description": "PolkaStats API", "author": "Mario Pino Uceda", "license": "Apache-2.0", diff --git a/backend/package.json b/backend/package.json index 6d786fe0..58b63097 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "backend", - "version": "0.25.0", + "version": "0.26.0", "description": "PolkaStats NG Backend", "author": "Mario Pino Uceda", "license": "Apache-2.0", diff --git a/frontend/package.json b/frontend/package.json index 80166bd9..13e73495 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "frontend", - "version": "0.25.0", + "version": "0.26.0", "description": "PolkaStats NG frontend", "author": "Mario Pino Uceda", "license": "Apache-2.0", diff --git a/package.json b/package.json index 270fc671..f20e34db 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "cerestats", - "version": "0.25.0", + "version": "0.26.0", "description": "Cere Stats mono repo", "repository": { "type": "git", From 9353c56c0d282030fe779f8b13053b440310eaf3 Mon Sep 17 00:00:00 2001 From: Andrei Navoichyk Date: Thu, 23 Mar 2023 03:07:25 +0100 Subject: [PATCH 59/59] turn off migration from crawler --- backend/index.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/backend/index.js b/backend/index.js index 71ee76a6..1b0c5fb9 100644 --- a/backend/index.js +++ b/backend/index.js @@ -1,6 +1,5 @@ // @ts-check const express = require('express'); -const DBMigrate = require('db-migrate'); const pino = require('pino'); const { spawn } = require('child_process'); const { StatusCodes } = require('http-status-codes'); @@ -48,14 +47,6 @@ const runCrawler = async ({ crawler, name }) => { }; const runCrawlers = async () => { - logger.info('Running migrations'); - await DBMigrate.getInstance(true, { - env: process.env.NODE_ENV || 'local', - config: '../db/database.json', - cmdOptions: { - 'migrations-dir': '../db/migrations', - }, - }).up(); logger.info('Migrations completed'); logger.info('Starting backend, waiting 10s...');