From 68d44ba268d977a27fc3bcc11de4f3e8b21f85b4 Mon Sep 17 00:00:00 2001 From: Shubham Padia Date: Sat, 6 Apr 2024 20:02:42 +0530 Subject: [PATCH] Databases: Support postgres schema in dev environment. Fixes #1818. This commit adds postgres schema support to the app logic. The dev environment uses synchronize function to create tables, and does not run the explicit migrations. We will add schema support for production in the next commit. --- .env.example | 3 +++ lib/env.ts | 3 +++ npm/src/db/constants.ts | 1 + npm/src/db/defaultDb.ts | 3 +++ npm/src/db/sql/sql.ts | 24 +++++++++++++++++++++--- npm/src/typings.ts | 3 +++ npm/test/db/db.test.ts | 37 +++++++++++++++++++++++++++++++++++-- 7 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 npm/src/db/constants.ts diff --git a/.env.example b/.env.example index ef95f7528..b8122675c 100644 --- a/.env.example +++ b/.env.example @@ -20,6 +20,9 @@ DB_PAGE_LIMIT=50 DB_ENCRYPTION_KEY= # Uncomment below if you wish to run DB migrations manually. #DB_MANUAL_MIGRATION=true +# Specify postgres schema to use. In production, you will need to create the schema first +# to use this option. Jackson will not create it for you. +POSTGRES_SCHEMA= # Admin Portal settings # SMTP details for Magic Links diff --git a/lib/env.ts b/lib/env.ts index 80aa50aec..af2c3786f 100644 --- a/lib/env.ts +++ b/lib/env.ts @@ -50,6 +50,9 @@ const db: DatabaseOption = { writeCapacityUnits: process.env.DB_DYNAMODB_RCUS ? Number(process.env.DB_DYNAMODB_WCUS) : undefined, }, manualMigration: process.env.DB_MANUAL_MIGRATION === 'true', + postgres: { + schema: process.env.POSTGRES_SCHEMA, + }, }; /** Indicates if the Jackson instance is hosted (i.e. not self-hosted) */ diff --git a/npm/src/db/constants.ts b/npm/src/db/constants.ts new file mode 100644 index 000000000..93bed1e31 --- /dev/null +++ b/npm/src/db/constants.ts @@ -0,0 +1 @@ +export const DEFAULT_POSTGRES_SCHEMA = 'public'; diff --git a/npm/src/db/defaultDb.ts b/npm/src/db/defaultDb.ts index 6b935d726..aab4d1e2c 100644 --- a/npm/src/db/defaultDb.ts +++ b/npm/src/db/defaultDb.ts @@ -1,4 +1,5 @@ import { JacksonOption } from '../typings'; +import { DEFAULT_POSTGRES_SCHEMA } from './constants'; export default function defaultDb(opts: JacksonOption) { opts.db = opts.db || {}; @@ -12,6 +13,8 @@ export default function defaultDb(opts: JacksonOption) { opts.db.dynamodb.readCapacityUnits = opts.db.dynamodb.readCapacityUnits || 5; opts.db.dynamodb.writeCapacityUnits = opts.db.dynamodb.writeCapacityUnits || 5; opts.db.manualMigration = opts.db.manualMigration || false; + opts.db.postgres = opts.db.postgres || {}; + opts.db.postgres.schema = opts.db.postgres.schema || DEFAULT_POSTGRES_SCHEMA; return opts; } diff --git a/npm/src/db/sql/sql.ts b/npm/src/db/sql/sql.ts index eee5a494e..bc5990ae2 100644 --- a/npm/src/db/sql/sql.ts +++ b/npm/src/db/sql/sql.ts @@ -6,8 +6,9 @@ import { DatabaseDriver, DatabaseOption, Index, Encrypted, Records, SortOrder } import { DataSource, DataSourceOptions, In, IsNull } from 'typeorm'; import * as dbutils from '../utils'; import * as mssql from './mssql'; +import { DEFAULT_POSTGRES_SCHEMA } from '../constants'; -class Sql implements DatabaseDriver { +export class Sql implements DatabaseDriver { private options: DatabaseOption; private dataSource!: DataSource; private storeRepository; @@ -26,6 +27,7 @@ class Sql implements DatabaseDriver { async init({ JacksonStore, JacksonIndex, JacksonTTL }): Promise { const sqlType = this.options.engine === 'planetscale' ? 'mysql' : this.options.type!; + const postgresSchema = this.options.postgres?.schema || DEFAULT_POSTGRES_SCHEMA; // Synchronize by default for non-planetscale engines only if migrations are not set to run let synchronize = !this.options.manualMigration; if (this.options.engine === 'planetscale') { @@ -53,14 +55,30 @@ class Sql implements DatabaseDriver { ...baseOpts, }); } else { - this.dataSource = new DataSource({ + const dataSourceOptions = { url: this.options.url, ssl: this.options.ssl, ...baseOpts, - }); + }; + + if (sqlType === 'postgres') { + dataSourceOptions['synchronize'] = false; + dataSourceOptions['schema'] = postgresSchema; + } + this.dataSource = new DataSource(dataSourceOptions); } + await this.dataSource.initialize(); + if (sqlType === 'postgres' && synchronize) { + // We skip synchronization for postgres databases because TypeORM + // does not create schemas if they don't exist, we manually run + // synchronize here if it is set to true. + const queryRunner = this.dataSource.createQueryRunner(); + await queryRunner.query(`CREATE SCHEMA IF NOT EXISTS ${postgresSchema}`); + this.dataSource.synchronize(); + } + break; } catch (err) { console.error(`error connecting to engine: ${this.options.engine}, type: ${sqlType} db: ${err}`); diff --git a/npm/src/typings.ts b/npm/src/typings.ts index 4835d23f0..469c7f638 100644 --- a/npm/src/typings.ts +++ b/npm/src/typings.ts @@ -418,6 +418,9 @@ export interface DatabaseOption { writeCapacityUnits?: number; }; manualMigration?: boolean; + postgres?: { + schema?: string; + }; } export interface JacksonOption { diff --git a/npm/test/db/db.test.ts b/npm/test/db/db.test.ts index fe376b463..008cc6de3 100644 --- a/npm/test/db/db.test.ts +++ b/npm/test/db/db.test.ts @@ -9,6 +9,7 @@ const dbObjs: { [key: string]: DatabaseDriver } = {}; const connectionStores: Storable[] = []; const ttlStores: Storable[] = []; const ttl = 2; +const non_default_schema = 'non_default'; const record1 = { id: '1', @@ -130,6 +131,12 @@ const dbs = [ ...postgresDbConfig, encryptionKey, }, + { + ...postgresDbConfig, + postgres: { + schema: non_default_schema, + }, + }, { ...mongoDbConfig, }, @@ -188,7 +195,11 @@ tap.before(async () => { for (const idx in dbs) { const opts = dbs[idx]; const db = await DB.new(opts, true); - dbObjs[opts.engine! + (opts.type ? opts.type : '')] = db; + if (opts.type === 'postgres' && opts['schema'] === non_default_schema) { + dbObjs[opts['schema'] + opts.engine! + (opts.type ? opts.type : '')] = db; + } else { + dbObjs[opts.engine! + (opts.type ? opts.type : '')] = db; + } const randomSession = Date.now(); connectionStores.push(db.store('saml:config:' + randomSession + randomBytes(4).toString('hex'))); @@ -201,15 +212,32 @@ tap.teardown(async () => { }); tap.test('dbs', async () => { + // We need this to ensure that the test runs atleast once. + // It is quite easy to skip the test by mistake in the future + // if one of the conditions change and it goes unnoticed. + let has_non_default_postgres_schema_test_ran = false; for (const idx in connectionStores) { const connectionStore = connectionStores[idx]; const ttlStore = ttlStores[idx]; const dbEngine = dbs[idx].engine!; - let dbType = dbEngine; + let dbType = dbEngine.toString(); if (dbs[idx].type) { dbType += ': ' + dbs[idx].type; } + tap.test('Test non default postgres schema', (t) => { + if (dbType === 'sql: postgres' && dbs[idx].postgres?.schema === non_default_schema) { + t.same( + connectionStore['db']['db']['dataSource']['createQueryBuilder']()['connection']['options'][ + 'schema' + ], + non_default_schema + ); + } + has_non_default_postgres_schema_test_ran = true; + t.end(); + }); + tap.test('put(): ' + dbType, async () => { await connectionStore.put( record1.id, @@ -527,4 +555,9 @@ tap.test('dbs', async () => { await value.close(); } }); + + tap.test('Ensure that the test for non default postgres schema has ran atleast once', (t) => { + t.same(has_non_default_postgres_schema_test_ran, true); + t.end(); + }); });