Skip to content

Commit

Permalink
Add ability for MPW to run explorer-less (#427)
Browse files Browse the repository at this point in the history 
* Add Shield Sync-via-Node fallback

* Prettier

* Fix unawaited Raw Tx map

* Add ability to send transactions via RPC

* Optimise `getblock` with filters

* Add "background" transparent sync

* Use Nodes for faster block synchronisation

* Remove coinbase slicing from `getBlock`

* Remove artificial Blockbook height sync delay

* Fix and simplify sync tests

* nit: test typo fix

* cleanup: remove unused `skipCoinstake` param

* nit: rename `message` to `warning`

* review: remove `transparentSync` checks and `force` param
  • Loading branch information
@JSKitty
JSKitty authored Oct 21, 2024
1 parent 388b63e commit e90a606
Show file tree
Hide file tree
Showing 4 changed files with 115 additions and 84 deletions.
26 changes: 14 additions & 12 deletions scripts/dashboard/WalletBalance.vue
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<script setup>
import { cChainParams, COIN } from '../chain_params.js';
import { translation } from '../i18n';
import { translation, tr } from '../i18n';
import { ref, computed, toRefs } from 'vue';
import { beautifyNumber } from '../misc';
import { getEventEmitter } from '../event_bus';
Expand Down Expand Up @@ -130,11 +130,13 @@ const emit = defineEmits([
getEventEmitter().on(
'transparent-sync-status-update',
(i, totalPages, finished) => {
const str = tr(translation.syncStatusHistoryProgress, [
{ current: totalPages - i + 1 },
{ total: totalPages },
]);
(i, totalPages, finished, warning) => {
const str =
warning ||
tr(translation.syncStatusHistoryProgress, [
{ current: totalPages - i + 1 },
{ total: totalPages },
]);
const progress = ((totalPages - i) / totalPages) * 100;
syncTStr.value = str;
transparentProgressSyncing.value = progress;
Expand Down Expand Up @@ -510,16 +512,16 @@ function restoreWallet() {
</div>
<div style="width: 100%">
{{
transparentSyncing
? syncTStr
: `Syncing ${shieldBlockRemainingSyncing} Blocks...`
shieldSyncing
? `Syncing ${shieldBlockRemainingSyncing} Blocks...`
: syncTStr
}}
<LoadingBar
:show="true"
:percentage="
transparentSyncing
? transparentProgressSyncing
: shieldPercentageSyncing
shieldSyncing
? shieldPercentageSyncing
: transparentProgressSyncing
"
style="
border: 1px solid #932ecd;
Expand Down
104 changes: 56 additions & 48 deletions scripts/network.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,58 +92,47 @@ export class ExplorerNetwork extends Network {
}

/**
* Fetch a block from the explorer given the height
* Fetch a block from the current node given the height
* @param {number} blockHeight
* @param {boolean} skipCoinstake - if true coinstake tx will be skipped
* @returns {Promise<Object>} the block fetched from explorer
* @returns {Promise<Object>} the block
*/
async getBlock(blockHeight, skipCoinstake = false) {
const block = await this.safeFetchFromExplorer(
`/api/v2/block/${blockHeight}`
);
const newTxs = [];
// This is bad. We're making so many requests
// This is a quick fix to try to be compliant with the blockbook
// API, and not the PIVX extension.
// In the Blockbook API /block doesn't have any chain specific information
// Like hex, shield info or what not.
// We could change /getshieldblocks to /getshieldtxs?
// In addition, always skip the coinbase transaction and in case the coinstake one
// TODO: once v6.0 and shield stake is activated we might need to change this optimization
for (const tx of block.txs.slice(skipCoinstake ? 2 : 1)) {
const r = await fetch(
`${this.strUrl}/api/v2/tx-specific/${tx.txid}`
);
if (!r.ok) throw new Error('failed');
const newTx = await r.json();
newTxs.push(newTx);
}
block.txs = newTxs;
return block;
async getBlock(blockHeight) {
// First we fetch the blockhash (and strip RPC's quotes)
const strHash = (
await this.callRPC(`/getblockhash?params=${blockHeight}`, true)
).replace(/"/g, '');
// Craft a filter to retrieve only raw Tx hex and txid, also change "tx" to "txs"
const strFilter =
'&filter=' +
encodeURI(`. | .txs = [.tx[] | { hex: .hex, txid: .txid}]`);
// Fetch the full block (verbose)
return await this.callRPC(`/getblock?params=${strHash},2${strFilter}`);
}

/**
* Fetch the block height of the current explorer
* Fetch the block height of the current node
* @returns {Promise<number>} - Block height
*/
async getBlockCount() {
const { backend } = await (
await retryWrapper(fetchBlockbook, true, `/api/v2/api`)
).json();

return backend.blocks;
return parseInt(await this.callRPC('/getblockcount', true));
}

/**
* Fetch the latest block hash of the current explorer
* Fetch the latest block hash of the current explorer or fallback node
* @returns {Promise<string>} - Block hash
*/
async getBestBlockHash() {
const { backend } = await (
await retryWrapper(fetchBlockbook, true, `/api/v2/api`)
).json();
try {
// Attempt via Explorer first
const { backend } = await (
await retryWrapper(fetchBlockbook, true, `/api/v2/api`)
).json();

return backend.bestBlockHash;
return backend.bestBlockHash;
} catch {
// Use Nodes as a fallback
return await this.callRPC('/getbestblockhash', true);
}
}

/**
Expand Down Expand Up @@ -272,19 +261,38 @@ export class ExplorerNetwork extends Network {

async sendTransaction(hex) {
try {
const data = await (
await retryWrapper(fetchBlockbook, true, '/api/v2/sendtx/', {
method: 'post',
body: hex,
})
).json();
// Attempt via Explorer first
let strTXID = '';
try {
const cData = await (
await retryWrapper(
fetchBlockbook,
true,
'/api/v2/sendtx/',
{
method: 'post',
body: hex,
}
)
).json();
// If there's no TXID, we throw any potential Blockbook errors
if (!cData.result || cData.result.length !== 64) throw cData;
strTXID = cData.result;
} catch {
// Use Nodes as a fallback
strTXID = await this.callRPC(
'/sendrawtransaction?params=' + hex,
true
);
strTXID = strTXID.replace(/"/g, '');
}

// Throw and catch if the data is not a TXID
if (!data.result || data.result.length !== 64) throw data;
// Throw and catch if there's no TXID
if (!strTXID || strTXID.length !== 64) throw strTXID;

debugLog(DebugTopics.NET, 'Transaction sent! ' + data.result);
getEventEmitter().emit('transaction-sent', true, data.result);
return data.result;
debugLog(DebugTopics.NET, 'Transaction sent! ' + strTXID);
getEventEmitter().emit('transaction-sent', true, strTXID);
return strTXID;
} catch (e) {
getEventEmitter().emit('transaction-sent', false, e);
return false;
Expand Down
50 changes: 36 additions & 14 deletions scripts/wallet.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,8 +289,8 @@ export class Wallet {
}

/**
* Derive xpub (given nReceiving and nIndex)
* @return {boolean} Return true if a masterKey has been loaded in the wallet
* Check if the wallet (masterKey) is loaded in memory
* @return {boolean} Return `true` if a masterKey has been loaded in the wallet
*/
isLoaded() {
return !!this.#masterKey;
Expand Down Expand Up @@ -690,20 +690,42 @@ export class Wallet {
return histTXs;
}
sync = lockableFunction(async () => {
if (!this.isLoaded()) {
throw new Error('Attempting to sync without a wallet loaded');
}
if (this.#isSynced) {
throw new Error('Attempting to sync when already synced');
}
// While syncing the wallet ( DB read + network sync) disable the event balance-update
// While syncing the wallet (DB read + network sync) disable the event balance-update
// This is done to avoid a huge spam of event.
getEventEmitter().disableEvent('balance-update');

await this.loadFromDisk();
await this.loadShieldFromDisk();
// Let's set the last processed block 5 blocks behind the actual chain tip
// This is just to be sure since blockbook (as we know)
// usually does not return txs of the actual last block.
this.#lastProcessedBlock = blockCount - 5;
await this.#transparentSync();
this.#lastProcessedBlock = blockCount;
// Transparent sync is inherently less stable than Shield since it requires heavy
// explorer indexing, so we'll attempt once asynchronously, and if it fails, set a
// recurring "background" sync interval until it's finally successful.
try {
await this.#transparentSync();
} catch {
// We'll set a 5s interval sync until it's finally successful, then nuke the 'thread'.
const cThread = new AsyncInterval(async () => {
try {
await this.#transparentSync();
cThread.clearInterval();
} catch {
// Emit a transparent sync warning
getEventEmitter().emit(
'transparent-sync-status-update',
0,
0,
false,
'Explorers are unreachable, your wallet may not be fully synced!'
);
}
}, 5000);
}
if (this.hasShield()) {
await this.#syncShield();
}
Expand All @@ -714,8 +736,10 @@ export class Wallet {
getEventEmitter().emit('new-tx');
});

/**
* Synchronise UTXOs via xpub/address from the current explorer.
*/
async #transparentSync() {
if (!this.isLoaded() || this.#isSynced) return;
const cNet = getNetwork();
const addr = this.getKeyToExport();
let nStartHeight = Math.max(
Expand Down Expand Up @@ -763,7 +787,7 @@ export class Wallet {
await startBatch(
async (i) => {
let block;
block = await cNet.getBlock(blockHeights[i], true);
block = await cNet.getBlock(blockHeights[i]);
downloaded++;
blocks[i] = block;
// We need to process blocks monotically
Expand Down Expand Up @@ -852,11 +876,9 @@ export class Wallet {
async (blockCount) => {
const cNet = getNetwork();
let block;
// Don't ask for the exact last block that arrived,
// since it takes around 1 minute for blockbook to make it API available
for (
let blockHeight = this.#lastProcessedBlock + 1;
blockHeight < blockCount;
let blockHeight = this.#lastProcessedBlock;
blockHeight <= blockCount;
blockHeight++
) {
try {
Expand Down
19 changes: 9 additions & 10 deletions tests/integration/wallet/sync.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ vi.mock('../../../scripts/network.js');
* @param{number} value - amounts to transfer
* @returns {Promise<void>}
*/
async function crateAndSendTransaction(wallet, address, value) {
async function createAndSendTransaction(wallet, address, value) {
const tx = wallet.createTransaction(address, value);
await wallet.sign(tx);
expect(getNetwork().sendTransaction(tx.serialize())).toBeTruthy();
Expand Down Expand Up @@ -65,23 +65,19 @@ describe('Wallet sync tests', () => {
it('Basic 2 wallets sync test', async () => {
// --- Verify that funds are received after sending a transaction ---
// The legacy wallet sends the HD wallet 0.05 PIVs
await crateAndSendTransaction(
await createAndSendTransaction(
walletLegacy,
walletHD.getCurrentAddress(),
0.05 * 10 ** 8
);

// Mint the block with the transaction
await mineAndSync();
// getLatestBlocks sync up until chain tip - 1 block,
// so at this point walletHD doesn't still know about the UTXO he received
expect(walletHD.balance).toBe(1 * 10 ** 8);
// mine an empty block and verify that the tx arrived
await mineAndSync();
expect(walletHD.balance).toBe((1 + 0.05) * 10 ** 8);

// Sends funds back to the legacy wallet and verify that he also correctly receives funds
const legacyBalance = walletLegacy.balance;
await crateAndSendTransaction(
await createAndSendTransaction(
walletHD,
walletLegacy.getCurrentAddress(),
1 * 10 ** 8
Expand All @@ -103,13 +99,16 @@ describe('Wallet sync tests', () => {
let newAddress = walletHD.getAddressFromPath(
path.slice(0, -1) + String(nAddress)
);
await crateAndSendTransaction(
// Create a Tx to the new account address
await createAndSendTransaction(
walletLegacy,
newAddress,
0.01 * 10 ** 8
);
await mineAndSync();
// Validate the balance of the HD wallet pre-tx-confirm
expect(walletHD.balance).toBe((1 + 0.01 * i) * 10 ** 8);
// Mine a block with the Tx
await mineAndSync();
}
});
it('recognizes immature balance', async () => {
Expand Down

0 comments on commit e90a606

Please sign in to comment.