From e90a606bd977e6d1ede4c6666a3cae75dfdbfc1a Mon Sep 17 00:00:00 2001
From: JSKitty <mail@jskitty.cat>
Date: Mon, 21 Oct 2024 14:09:26 +0100
Subject: [PATCH] Add ability for MPW to run explorer-less (#427)

* Add Shield Sync-via-Node fallback

* Prettier

* Fix unawaited Raw Tx map

* Add ability to send transactions via RPC

* Optimise `getblock` with filters

* Add "background" transparent sync

* Use Nodes for faster block synchronisation

* Remove coinbase slicing from `getBlock`

* Remove artificial Blockbook height sync delay

* Fix and simplify sync tests

* nit: test typo fix

* cleanup: remove unused `skipCoinstake` param

* nit: rename `message` to `warning`

* review: remove `transparentSync` checks and `force` param
---
 scripts/dashboard/WalletBalance.vue   |  26 ++++---
 scripts/network.js                    | 104 ++++++++++++++------------
 scripts/wallet.js                     |  50 +++++++++----
 tests/integration/wallet/sync.spec.js |  19 +++--
 4 files changed, 115 insertions(+), 84 deletions(-)

diff --git a/scripts/dashboard/WalletBalance.vue b/scripts/dashboard/WalletBalance.vue
index 07c1fa3c6..4e9445a80 100644
--- a/scripts/dashboard/WalletBalance.vue
+++ b/scripts/dashboard/WalletBalance.vue
@@ -1,6 +1,6 @@
 <script setup>
 import { cChainParams, COIN } from '../chain_params.js';
-import { translation } from '../i18n';
+import { translation, tr } from '../i18n';
 import { ref, computed, toRefs } from 'vue';
 import { beautifyNumber } from '../misc';
 import { getEventEmitter } from '../event_bus';
@@ -130,11 +130,13 @@ const emit = defineEmits([
 
 getEventEmitter().on(
     'transparent-sync-status-update',
-    (i, totalPages, finished) => {
-        const str = tr(translation.syncStatusHistoryProgress, [
-            { current: totalPages - i + 1 },
-            { total: totalPages },
-        ]);
+    (i, totalPages, finished, warning) => {
+        const str =
+            warning ||
+            tr(translation.syncStatusHistoryProgress, [
+                { current: totalPages - i + 1 },
+                { total: totalPages },
+            ]);
         const progress = ((totalPages - i) / totalPages) * 100;
         syncTStr.value = str;
         transparentProgressSyncing.value = progress;
@@ -510,16 +512,16 @@ function restoreWallet() {
                 </div>
                 <div style="width: 100%">
                     {{
-                        transparentSyncing
-                            ? syncTStr
-                            : `Syncing ${shieldBlockRemainingSyncing} Blocks...`
+                        shieldSyncing
+                            ? `Syncing ${shieldBlockRemainingSyncing} Blocks...`
+                            : syncTStr
                     }}
                     <LoadingBar
                         :show="true"
                         :percentage="
-                            transparentSyncing
-                                ? transparentProgressSyncing
-                                : shieldPercentageSyncing
+                            shieldSyncing
+                                ? shieldPercentageSyncing
+                                : transparentProgressSyncing
                         "
                         style="
                             border: 1px solid #932ecd;
diff --git a/scripts/network.js b/scripts/network.js
index f9e8fecd1..68d21e463 100644
--- a/scripts/network.js
+++ b/scripts/network.js
@@ -92,58 +92,47 @@ export class ExplorerNetwork extends Network {
     }
 
     /**
-     * Fetch a block from the explorer given the height
+     * Fetch a block from the current node given the height
      * @param {number} blockHeight
-     * @param {boolean} skipCoinstake - if true coinstake tx will be skipped
-     * @returns {Promise<Object>} the block fetched from explorer
+     * @returns {Promise<Object>} the block
      */
-    async getBlock(blockHeight, skipCoinstake = false) {
-        const block = await this.safeFetchFromExplorer(
-            `/api/v2/block/${blockHeight}`
-        );
-        const newTxs = [];
-        // This is bad. We're making so many requests
-        // This is a quick fix to try to be compliant with the blockbook
-        // API, and not the PIVX extension.
-        // In the Blockbook API /block doesn't have any chain specific information
-        // Like hex, shield info or what not.
-        // We could change /getshieldblocks to /getshieldtxs?
-        // In addition, always skip the coinbase transaction and in case the coinstake one
-        // TODO: once v6.0 and shield stake is activated we might need to change this optimization
-        for (const tx of block.txs.slice(skipCoinstake ? 2 : 1)) {
-            const r = await fetch(
-                `${this.strUrl}/api/v2/tx-specific/${tx.txid}`
-            );
-            if (!r.ok) throw new Error('failed');
-            const newTx = await r.json();
-            newTxs.push(newTx);
-        }
-        block.txs = newTxs;
-        return block;
+    async getBlock(blockHeight) {
+        // First we fetch the blockhash (and strip RPC's quotes)
+        const strHash = (
+            await this.callRPC(`/getblockhash?params=${blockHeight}`, true)
+        ).replace(/"/g, '');
+        // Craft a filter to retrieve only raw Tx hex and txid, also change "tx" to "txs"
+        const strFilter =
+            '&filter=' +
+            encodeURI(`. | .txs = [.tx[] | { hex: .hex, txid: .txid}]`);
+        // Fetch the full block (verbose)
+        return await this.callRPC(`/getblock?params=${strHash},2${strFilter}`);
     }
 
     /**
-     * Fetch the block height of the current explorer
+     * Fetch the block height of the current node
      * @returns {Promise<number>} - Block height
      */
     async getBlockCount() {
-        const { backend } = await (
-            await retryWrapper(fetchBlockbook, true, `/api/v2/api`)
-        ).json();
-
-        return backend.blocks;
+        return parseInt(await this.callRPC('/getblockcount', true));
     }
 
     /**
-     * Fetch the latest block hash of the current explorer
+     * Fetch the latest block hash of the current explorer or fallback node
      * @returns {Promise<string>} - Block hash
      */
     async getBestBlockHash() {
-        const { backend } = await (
-            await retryWrapper(fetchBlockbook, true, `/api/v2/api`)
-        ).json();
+        try {
+            // Attempt via Explorer first
+            const { backend } = await (
+                await retryWrapper(fetchBlockbook, true, `/api/v2/api`)
+            ).json();
 
-        return backend.bestBlockHash;
+            return backend.bestBlockHash;
+        } catch {
+            // Use Nodes as a fallback
+            return await this.callRPC('/getbestblockhash', true);
+        }
     }
 
     /**
@@ -272,19 +261,38 @@ export class ExplorerNetwork extends Network {
 
     async sendTransaction(hex) {
         try {
-            const data = await (
-                await retryWrapper(fetchBlockbook, true, '/api/v2/sendtx/', {
-                    method: 'post',
-                    body: hex,
-                })
-            ).json();
+            // Attempt via Explorer first
+            let strTXID = '';
+            try {
+                const cData = await (
+                    await retryWrapper(
+                        fetchBlockbook,
+                        true,
+                        '/api/v2/sendtx/',
+                        {
+                            method: 'post',
+                            body: hex,
+                        }
+                    )
+                ).json();
+                // If there's no TXID, we throw any potential Blockbook errors
+                if (!cData.result || cData.result.length !== 64) throw cData;
+                strTXID = cData.result;
+            } catch {
+                // Use Nodes as a fallback
+                strTXID = await this.callRPC(
+                    '/sendrawtransaction?params=' + hex,
+                    true
+                );
+                strTXID = strTXID.replace(/"/g, '');
+            }
 
-            // Throw and catch if the data is not a TXID
-            if (!data.result || data.result.length !== 64) throw data;
+            // Throw and catch if there's no TXID
+            if (!strTXID || strTXID.length !== 64) throw strTXID;
 
-            debugLog(DebugTopics.NET, 'Transaction sent! ' + data.result);
-            getEventEmitter().emit('transaction-sent', true, data.result);
-            return data.result;
+            debugLog(DebugTopics.NET, 'Transaction sent! ' + strTXID);
+            getEventEmitter().emit('transaction-sent', true, strTXID);
+            return strTXID;
         } catch (e) {
             getEventEmitter().emit('transaction-sent', false, e);
             return false;
diff --git a/scripts/wallet.js b/scripts/wallet.js
index 66fe675ee..89fad4113 100644
--- a/scripts/wallet.js
+++ b/scripts/wallet.js
@@ -289,8 +289,8 @@ export class Wallet {
     }
 
     /**
-     * Derive xpub (given nReceiving and nIndex)
-     * @return {boolean} Return true if a masterKey has been loaded in the wallet
+     * Check if the wallet (masterKey) is loaded in memory
+     * @return {boolean} Return `true` if a masterKey has been loaded in the wallet
      */
     isLoaded() {
         return !!this.#masterKey;
@@ -690,20 +690,42 @@ export class Wallet {
         return histTXs;
     }
     sync = lockableFunction(async () => {
+        if (!this.isLoaded()) {
+            throw new Error('Attempting to sync without a wallet loaded');
+        }
         if (this.#isSynced) {
             throw new Error('Attempting to sync when already synced');
         }
-        // While syncing the wallet ( DB read + network sync) disable the event balance-update
+        // While syncing the wallet (DB read + network sync) disable the event balance-update
         // This is done to avoid a huge spam of event.
         getEventEmitter().disableEvent('balance-update');
 
         await this.loadFromDisk();
         await this.loadShieldFromDisk();
-        // Let's set the last processed block 5 blocks behind the actual chain tip
-        // This is just to be sure since blockbook (as we know)
-        // usually does not return txs of the actual last block.
-        this.#lastProcessedBlock = blockCount - 5;
-        await this.#transparentSync();
+        this.#lastProcessedBlock = blockCount;
+        // Transparent sync is inherently less stable than Shield since it requires heavy
+        // explorer indexing, so we'll attempt once asynchronously, and if it fails, set a
+        // recurring "background" sync interval until it's finally successful.
+        try {
+            await this.#transparentSync();
+        } catch {
+            // We'll set a 5s interval sync until it's finally successful, then nuke the 'thread'.
+            const cThread = new AsyncInterval(async () => {
+                try {
+                    await this.#transparentSync();
+                    cThread.clearInterval();
+                } catch {
+                    // Emit a transparent sync warning
+                    getEventEmitter().emit(
+                        'transparent-sync-status-update',
+                        0,
+                        0,
+                        false,
+                        'Explorers are unreachable, your wallet may not be fully synced!'
+                    );
+                }
+            }, 5000);
+        }
         if (this.hasShield()) {
             await this.#syncShield();
         }
@@ -714,8 +736,10 @@ export class Wallet {
         getEventEmitter().emit('new-tx');
     });
 
+    /**
+     * Synchronise UTXOs via xpub/address from the current explorer.
+     */
     async #transparentSync() {
-        if (!this.isLoaded() || this.#isSynced) return;
         const cNet = getNetwork();
         const addr = this.getKeyToExport();
         let nStartHeight = Math.max(
@@ -763,7 +787,7 @@ export class Wallet {
             await startBatch(
                 async (i) => {
                     let block;
-                    block = await cNet.getBlock(blockHeights[i], true);
+                    block = await cNet.getBlock(blockHeights[i]);
                     downloaded++;
                     blocks[i] = block;
                     // We need to process blocks monotically
@@ -852,11 +876,9 @@ export class Wallet {
         async (blockCount) => {
             const cNet = getNetwork();
             let block;
-            // Don't ask for the exact last block that arrived,
-            // since it takes around 1 minute for blockbook to make it API available
             for (
-                let blockHeight = this.#lastProcessedBlock + 1;
-                blockHeight < blockCount;
+                let blockHeight = this.#lastProcessedBlock;
+                blockHeight <= blockCount;
                 blockHeight++
             ) {
                 try {
diff --git a/tests/integration/wallet/sync.spec.js b/tests/integration/wallet/sync.spec.js
index be86ce6c4..ab712889d 100644
--- a/tests/integration/wallet/sync.spec.js
+++ b/tests/integration/wallet/sync.spec.js
@@ -22,7 +22,7 @@ vi.mock('../../../scripts/network.js');
  * @param{number} value - amounts to transfer
  * @returns {Promise<void>}
  */
-async function crateAndSendTransaction(wallet, address, value) {
+async function createAndSendTransaction(wallet, address, value) {
     const tx = wallet.createTransaction(address, value);
     await wallet.sign(tx);
     expect(getNetwork().sendTransaction(tx.serialize())).toBeTruthy();
@@ -65,23 +65,19 @@ describe('Wallet sync tests', () => {
     it('Basic 2 wallets sync test', async () => {
         // --- Verify that funds are received after sending a transaction ---
         // The legacy wallet sends the HD wallet 0.05 PIVs
-        await crateAndSendTransaction(
+        await createAndSendTransaction(
             walletLegacy,
             walletHD.getCurrentAddress(),
             0.05 * 10 ** 8
         );
+
         // Mint the block with the transaction
         await mineAndSync();
-        // getLatestBlocks sync up until chain tip - 1 block,
-        // so at this point walletHD doesn't still know about the UTXO he received
-        expect(walletHD.balance).toBe(1 * 10 ** 8);
-        // mine an empty block and verify that the tx arrived
-        await mineAndSync();
         expect(walletHD.balance).toBe((1 + 0.05) * 10 ** 8);
 
         // Sends funds back to the legacy wallet and verify that he also correctly receives funds
         const legacyBalance = walletLegacy.balance;
-        await crateAndSendTransaction(
+        await createAndSendTransaction(
             walletHD,
             walletLegacy.getCurrentAddress(),
             1 * 10 ** 8
@@ -103,13 +99,16 @@ describe('Wallet sync tests', () => {
             let newAddress = walletHD.getAddressFromPath(
                 path.slice(0, -1) + String(nAddress)
             );
-            await crateAndSendTransaction(
+            // Create a Tx to the new account address
+            await createAndSendTransaction(
                 walletLegacy,
                 newAddress,
                 0.01 * 10 ** 8
             );
-            await mineAndSync();
+            // Validate the balance of the HD wallet pre-tx-confirm
             expect(walletHD.balance).toBe((1 + 0.01 * i) * 10 ** 8);
+            // Mine a block with the Tx
+            await mineAndSync();
         }
     });
     it('recognizes immature balance', async () => {