V0.8.2 (#66)

* Optimized tag query and peer route

* removing old koi, adding new koi logs :) (#65)

* removing old koi, adding new koi logs :)

* Delete yarn.lock

* fixing order of express operations

Co-authored-by: Al <al@openkoi.com>

* Formatting

* Sub query limit and offset

* Update migrations

Co-authored-by: Al Morris <37120777+alexander-morris@users.noreply.github.com>
Co-authored-by: Al <al@openkoi.com>

bin/index.create.sh

@@ -13,22 +13,47 @@ SHOW statement_timeout;
 SHOW maintenance_work_mem;
 SHOW max_parallel_maintenance_workers;
 
+--- Block Indices
+--- Block Height Index
 CREATE INDEX "blocks_height" ON blocks USING BTREE ("height");
+--- Blocks Mined At Index
 CREATE INDEX "blocks_mined_at" ON blocks USING BTREE ("mined_at");
 
+--- Transaction Indices
+--- Transaction Height Index
 CREATE INDEX "transactions_height" ON transactions USING BTREE ("height");
+--- Transaction Target Index
 CREATE INDEX "transactions_target" ON transactions USING BTREE ("target");
+--- Transaction Owner Address Index
 CREATE INDEX "transactions_owner_address" ON transactions USING BTREE ("owner_address");
+--- Transaction Namespace Index
 CREATE INDEX "index_namespace_transactions" ON transactions USING BTREE ("namespace");
+--- Transaction Domain Index
 CREATE INDEX "index_domain_transactions" ON transactions USING BTREE ("domain");
+--- Transaction App Index
 CREATE INDEX "index_app_transactions" ON transactions USING BTREE ("app");
+--- Transaction App-Name Index
 CREATE INDEX "index_App-Name_transactions" ON transactions USING BTREE ("App-Name");
 
+--- Tag Indices
+--- Tag Transaction Id Index
+CREATE INDEX "tags_tx_id" ON tags USING BTREE ("tx_id");
+--- Tag Name Index (under 64 characters)
 CREATE INDEX "tags_name" ON tags USING BTREE ("name") WHERE LENGTH("name") < 64;
+--- Tag Value Index (under 64 characters)
 CREATE INDEX "tags_value" ON tags USING BTREE ("value") WHERE LENGTH("value") < 64;
+--- Tag Name, Value Index (under 64 characters)
 CREATE INDEX "tags_name_value" ON tags USING BTREE ("name", "value") WHERE LENGTH("name") < 64 AND LENGTH("value") < 64;
+--- Tag Transaction Id, Name Index (under 64 characters)
 CREATE INDEX "tags_tx_id_name" ON tags USING BTREE ("tx_id", "name") WHERE LENGTH("name") < 64;
-CREATE INDEX "tags_tx_id" ON tags USING BTREE ("tx_id");
+--- Tag Name Index (under 128 chracters)
+CREATE INDEX "tags_name_128" ON tags USING BTREE ("name") WHERE LENGTH("name") > 64 AND LENGTH("name") < 128;
+--- Tag Value Index (under 128 chracters)
+CREATE INDEX "tags_value_128" ON tags USING BTREE ("value") WHERE LENGTH("value") > 64; AND LENGTH("value") < 128;
+--- Tag Name, Value Index (under 128 chracters)
+CREATE INDEX "tags_name_value_128" ON tags USING BTREE ("name", "value") WHERE LENGTH("name") > 64 AND LENGTH("name") < 128 AND LENGTH("value") > 64 AND LENGTH("value") < 128;
+--- Tag Transaction Id, Name Index (under 128 chracters)
+CREATE INDEX "tags_tx_id_name_128" ON tags USING BTREE ("tx_id", "name") WHERE LENGTH("name") > 64 AND LENGTH("name") < 128;
 
 EOF
 

migrations/20200404025828_initialize.ts

@@ -4,8 +4,6 @@ import {config} from 'dotenv';
 config();
 
 export async function up(knex: Knex) {
-  const indices = JSON.parse(process.env.INDICES || '[]');
-
   return knex.schema
       .withSchema(process.env.ENVIRONMENT || 'public')
       .createTable('transactions', (table) => {
@@ -26,16 +24,7 @@ export async function up(knex: Knex) {
         table.string('parent', 64);
         table.timestamp('created_at').defaultTo(knex.fn.now());
 
-        for (let i = 0; i < indices.length; i++) {
-          const index = indices[i];
-          table.string(index, 64);
-          table.index(index, `index_${index}_transactions`, 'BTREE');
-        }
-
         table.primary(['id'], 'pkey_transactions');
-        table.index(['height'], 'transactions_height', 'BTREE');
-        table.index(['owner_address'], 'transactions_owner_address', 'BTREE');
-        table.index(['target'], 'transactions_target', 'BTREE');
       })
       .createTable('blocks', (table) => {
         table.string('id', 64).notNullable();
@@ -47,8 +36,6 @@ export async function up(knex: Knex) {
         table.timestamp('created_at').defaultTo(knex.fn.now());
 
         table.primary(['id'], 'pkey_blocks');
-        table.index(['height'], 'blocks_height', 'BTREE');
-        table.index(['mined_at'], 'blocks_mined_at', 'BTREE');
       })
       .createTable('tags', (table) => {
         table.string('tx_id', 64).notNullable();
@@ -58,11 +45,6 @@ export async function up(knex: Knex) {
         table.timestamp('created_at').defaultTo(knex.fn.now());
 
         table.primary(['tx_id', 'index'], 'pkey_tags');
-        table.index(['tx_id', 'name'], 'tags_tx_id_name', 'BTREE');
-        table.index(['name'], 'tags_name', 'BTREE');
-        table.index(['value'], 'tags_value', 'BTREE');
-        table.index(['name', 'value'], 'tags_name_value', 'BTREE');
-        table.index(['tx_id'], 'tags_tx_id', 'BTREE');
       });
 }
 

package.json

@@ -45,6 +45,7 @@
     "graphql-fields": "^2.0.3",
     "js-sha256": "^0.9.0",
     "knex": "^0.21.17",
+    "koi-logs": "git+https://github.com/open-koi/logs.git",
     "lodash": "^4.17.20",
     "moment": "^2.29.1",
     "morgan": "^1.10.0",

src/Gateway.ts

@@ -1,7 +1,6 @@
 import 'colors';
 import express, {Express} from 'express';
 import {config} from 'dotenv';
-import cron from 'node-cron';
 import {corsMiddleware} from './middleware/cors.middleware';
 import {jsonMiddleware} from './middleware/json.middleware';
 import {logMiddleware} from './middleware/log.middleware';
@@ -10,8 +9,9 @@ import {graphServer} from './graphql/server.graphql';
 import {statusRoute} from './route/status.route';
 import {proxyRoute} from './route/proxy.route';
 import {dataRouteRegex, dataHeadRoute, dataRoute} from './route/data.route';
+import {peerRoute} from './route/peer.route';
+import {koiLogger, koiLogsRoute, koiLogsRawRoute} from './route/koi.route';
 import {startSync} from './database/sync.database';
-import {logsHelper, logsTask} from './utility/log.helper';
 
 config();
 
@@ -22,21 +22,18 @@ export function start() {
   app.use(corsMiddleware);
   app.use(jsonMiddleware);
   app.use(logMiddleware);
-
-  cron.schedule('0 0 * * *', async function() {
-    console.log('running the log cleanup task once per day on ', new Date() );
-    const result = await logsTask();
-    console.log('daily log task returned ', result);
-  });
+  app.use(koiLogger.logger);
 
   app.get('/', statusRoute);
+
   app.head(dataRouteRegex, dataHeadRoute);
   app.get(dataRouteRegex, dataRoute);
 
   graphServer({introspection: true, playground: true}).applyMiddleware({app, path: '/graphql'});
 
-  app.get(dataRouteRegex, dataRoute);
-  app.get('/logs', logsHelper);
+  app.get('/peers', peerRoute);
+  app.get('/logs', koiLogsRoute);
+  app.get('/logs/raw', koiLogsRawRoute);
 
   app.all('*', proxyRoute);
 

src/graphql/query.graphql.ts

@@ -4,7 +4,7 @@ import {indices} from '../utility/order.utility';
 import {connection} from '../database/connection.database';
 import {ISO8601DateTimeString} from '../utility/encoding.utility';
 import {TagFilter} from './types';
-import {tagToB64} from '../query/transaction.query';
+import {tagToB64, toB64url} from '../query/transaction.query';
 
 config();
 
@@ -72,13 +72,19 @@ export async function generateQuery(params: QueryParams): Promise<QueryBuilder>
   if (tags) {
     const tagsConverted = tagToB64(tags);
 
+    const subQuery = connection
+        .queryBuilder()
+        .select('*')
+        .from('tags');
+
+    let runSubQuery = false;
+
     for (let i = 0; i < tagsConverted.length; i++) {
       const tag = tagsConverted[i];
-      const tagAlias = `${i}_${i}`;
       let indexed = false;
 
       for (let ii = 0; ii < indices.length; ii++) {
-        const index = indices[ii];
+        const index = toB64url(indices[ii]);
 
         if (tag.name === index) {
           indexed = true;
@@ -87,13 +93,22 @@ export async function generateQuery(params: QueryParams): Promise<QueryBuilder>
       }
 
       if (indexed === false) {
-        query.join(`tags as ${tagAlias}`, (join) => {
-          join.on('transactions.id', `${tagAlias}.tx_id`);
+        subQuery.where('name', tag.name);
+        subQuery.whereIn('value', tag.values);
+        runSubQuery = true;
+      }
+    }
+
+    if (runSubQuery) {
+      const results = await subQuery.limit(limit).offset(offset);
+      const tx_ids = [];
 
-          join.andOnIn(`${tagAlias}.name`, [tag.name]);
-          join.andOnIn(`${tagAlias}.value`, tag.values);
-        });
+      for (let i = 0; i < results.length; i++) {
+        const {tx_id} = results[i];
+        tx_ids.push(tx_id);
       }
+
+      query.whereIn('transactions.id', tx_ids);
     }
   }
 

src/route/koi.route.ts

@@ -0,0 +1,12 @@
+import KoiLogs from 'koi-logs';
+import {Request, Response} from 'express';
+
+export const koiLogger = new KoiLogs('./');
+
+export async function koiLogsRoute(req: Request, res: Response) {
+  return koiLogger.koiLogsHelper(req, res);
+}
+
+export async function koiLogsRawRoute(req: Request, res: Response) {
+  return koiLogger.koiRawLogsHelper(req, res);
+}

src/route/peer.route.ts

@@ -0,0 +1,11 @@
+import {Request, Response} from 'express';
+
+export async function peerRoute(req: Request, res: Response) {
+  return res.status(200).send({
+    status: 'OK',
+    nodes: [
+      'https://gateway-n1.amplify.host',
+      'https://gateway-n2.amplify.host',
+    ],
+  });
+}