From 40b320f95eddc04011fba676c506c12339dbc292 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?The=20Lone=20R=C5=8Dnin?= <68780068+TheLoneRonin@users.noreply.github.com> Date: Thu, 13 May 2021 14:55:36 -0400 Subject: [PATCH] ANS Caching and Verification (#72) --- package.json | 3 ++- src/caching/ans.caching.ts | 5 +++- src/caching/ans.entry.caching.ts | 17 ++++++++++++++ src/caching/ans.verify.ts | 40 ++++++++++++++++++++++++++++++++ src/database/sync.database.ts | 2 ++ src/query/node.query.ts | 2 +- src/route/data.route.ts | 24 +++++++++++-------- 7 files changed, 80 insertions(+), 13 deletions(-) create mode 100644 src/caching/ans.entry.caching.ts create mode 100644 src/caching/ans.verify.ts diff --git a/package.json b/package.json index a5ba942..4de7b5f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@arweave/gateway", - "version": "0.9.1", + "version": "0.10.1", "main": "dist/src/Gateway.js", "repository": "git@github.com:ArweaveTeam/gateway.git", "author": "Arweave ", @@ -15,6 +15,7 @@ "dev:restart": "npm run migrate:down && npm run migrate:latest && npm run dev:start", "import:transaction": "npm run dev:build && node dist/src/import/transaction.import.js", "import:tags": "npm run dev:build && node dist/src/import/tags.import.ts", + "verify:ans": "npm run dev:build && node dist/src/caching/ans.verify.js", "verify:prune": "npm run dev:build && node dist/src/verify/prune.verify.js", "verify:transaction": "npm run dev:build && node dist/src/verify/transaction.verify.js", "migrate:down": "knex migrate:down", diff --git a/src/caching/ans.caching.ts b/src/caching/ans.caching.ts index 95fbe4d..693accb 100644 --- a/src/caching/ans.caching.ts +++ b/src/caching/ans.caching.ts @@ -3,6 +3,7 @@ import {DataItemJson} from 'arweave-bundles'; import {ansBundles} from '../utility/ans.utility'; import {getDataFromChunks} from '../query/node.query'; import {tagToUTF8} from '../query/transaction.query'; +import {cacheANSEntries} from './ans.entry.caching'; export async function streamAndCacheAns(id: string): Promise { try { @@ -24,12 +25,14 @@ export async function streamAndCacheAns(id: string): Promise { signature: ansTx.signature, tags: tagToUTF8(ansTx.tags), }; - + ansTxsConverted.push(newAnsTx); } write(`${process.cwd()}/cache/tx/${id}`, JSON.stringify(ansTxsConverted, null, 2)); + await cacheANSEntries(ansTxs); + return true; } catch (error) { remove(`${process.cwd()}/cache/tx/${id}`); diff --git a/src/caching/ans.entry.caching.ts b/src/caching/ans.entry.caching.ts new file mode 100644 index 0000000..e5ca899 --- /dev/null +++ b/src/caching/ans.entry.caching.ts @@ -0,0 +1,17 @@ +import {exists, write} from 'fs-jetpack'; +import {DataItemJson} from 'arweave-bundles'; +import {b64UrlToBuffer} from '../utility/encoding.utility'; + +export async function cacheANSEntries(entries: Array) { + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]; + const id = entry.id; + const data = entry.data; + + const bufferData = Buffer.from(b64UrlToBuffer(data)); + + if (exists(`${process.cwd()}/cache/tx/${id}`) === false) { + write(`${process.cwd()}/cache/tx/${id}`, bufferData.toString('utf-8')); + } + } +} diff --git a/src/caching/ans.verify.ts b/src/caching/ans.verify.ts new file mode 100644 index 0000000..39073c3 --- /dev/null +++ b/src/caching/ans.verify.ts @@ -0,0 +1,40 @@ +import 'colors'; +import {connection} from '../database/connection.database'; +import {toB64url} from '../query/transaction.query'; +import {cacheAnsFile} from './file.caching'; + +const name = toB64url('Bundle-Type'); +const value = toB64url('ANS-102'); + +export async function ansVerify(offset: number = 0) { + const query = await connection + .queryBuilder() + .select('*') + .from('tags') + .where('name', name) + .where('value', value) + .orderByRaw('created_at ASC') + .limit(10) + .offset(offset); + + for (let i = 0; i < query.length; i++) { + const item = query[i]; + + try { + await cacheAnsFile(item.tx_id); + } catch (error) { + console.log(`Could not cache ${item.tx_id}`.red); + } + } + + console.log(`Successfully cached ANS bundles at offset ${offset}`.green); + + if (query.length === 0) { + console.log('Successfully cached all ANS bundles. Good work!'.green.bold); + process.exit(); + } else { + ansVerify(offset + 10); + } +} + +(async () => await ansVerify())(); diff --git a/src/database/sync.database.ts b/src/database/sync.database.ts index e232904..386fbbd 100644 --- a/src/database/sync.database.ts +++ b/src/database/sync.database.ts @@ -15,6 +15,7 @@ import {transaction, tagValue, Tag} from '../query/transaction.query'; import {getDataFromChunks} from '../query/node.query'; import {importBlocks, importTransactions, importTags} from './import.database'; import {DatabaseTag} from './transaction.database'; +import {cacheANSEntries} from '../caching/ans.entry.caching'; config(); mkdir('snapshot'); @@ -202,6 +203,7 @@ export async function processAns(id: string, height: number, retry: boolean = tr const ansPayload = await getDataFromChunks(id); const ansTxs = await ansBundles.unbundleData(ansPayload.toString('utf-8')); + await cacheANSEntries(ansTxs); await processANSTransaction(ansTxs, height); } catch (error) { if (retry) { diff --git a/src/query/node.query.ts b/src/query/node.query.ts index 8f780ed..c4b7afa 100644 --- a/src/query/node.query.ts +++ b/src/query/node.query.ts @@ -62,7 +62,7 @@ export async function getDataFromChunks(id: string, retry: boolean = true): Prom } catch (error) { if (retry) { console.error(`error retrieving data from ${id}, please note that this may be a cancelled transaction`.red.bold); - return await getDataFromChunks(id, true); + return await getDataFromChunks(id, false); } else { throw error; } diff --git a/src/route/data.route.ts b/src/route/data.route.ts index 5a5c762..e97db06 100644 --- a/src/route/data.route.ts +++ b/src/route/data.route.ts @@ -32,23 +32,27 @@ export async function dataRoute(req: Request, res: Response) { } } - const metadata = await getTransaction(transaction); - const contentType = tagValue(metadata.tags, 'Content-Type'); - const ans102 = tagValue(metadata.tags, 'Bundle-Type') === 'ANS-102'; + try { + const metadata = await getTransaction(transaction); + const contentType = tagValue(metadata.tags, 'Content-Type'); + const ans102 = tagValue(metadata.tags, 'Bundle-Type') === 'ANS-102'; - res.setHeader('content-type', contentType); + res.setHeader('content-type', contentType); - if (ans102) { - await cacheAnsFile(transaction); - } else { - await cacheFile(transaction); - } + if (ans102) { + await cacheAnsFile(transaction); + } else { + await cacheFile(transaction); + } + } catch (error) { + + } if (exists(`${process.cwd()}/cache/tx/${transaction}`)) { res.status(200); res.sendFile(`${process.cwd()}/cache/tx/${transaction}`); } else { res.status(500); - res.json({status: 'ERROR', message: 'Could not retrieve tx, please retry'}); + res.json({status: 'ERROR', message: 'Could not retrieve transaction'}); } }