From 05214d8f9a9c0d1becd19c08e72ef4c4abc5caf6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaros=C5=82aw=20Sadzi=C5=84ski?=
 <jaroslaw.sadzinski@gmail.com>
Date: Mon, 24 Jun 2024 16:52:51 +0200
Subject: [PATCH 01/16] (core) Port allocation fix in TestServer

Summary:
- Fixing port allocation in TestServer
- Extending logging in the Billing test
- Fixing negative rowIds support for add/remove actions
- Making FormulaEditor and CardView tests less flacky

Test Plan: Existing

Reviewers: paulfitz

Reviewed By: paulfitz

Subscribers: paulfitz, dsagal

Differential Revision: https://phab.getgrist.com/D4280
---
 sandbox/grist/useractions.py          | 8 +++++++-
 test/server/lib/helpers/TestServer.ts | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/sandbox/grist/useractions.py b/sandbox/grist/useractions.py
index ac9261ef1a..95f5a49c31 100644
--- a/sandbox/grist/useractions.py
+++ b/sandbox/grist/useractions.py
@@ -395,7 +395,7 @@ def doBulkAddOrReplace(self, table_id, row_ids, column_values, replace=False):
 
     # Whenever we add new rows, remember the mapping from any negative row_ids to their final
     # values. This allows the negative_row_ids to be used as Reference values in subsequent
-    # actions in the same bundle.
+    # actions in the same bundle, and in UpdateRecord/RemoveRecord actions.
     self._engine.out_actions.summary.update_new_rows_map(table_id, row_ids, filled_row_ids)
 
     # Convert entered values to the correct types.
@@ -446,6 +446,9 @@ def _addACLRules(self, table_id, row_ids, col_values):
   # ----------------------------------------
 
   def doBulkUpdateRecord(self, table_id, row_ids, columns):
+    # Replace negative ids that may refer to rows just added to this table in this bundle.
+    row_ids = self._engine.out_actions.summary.translate_new_row_ids(table_id, row_ids)
+
     # Convert passed-in values to the column's correct types (or alttext, or errors) and trim any
     # unchanged values.
     action, extra_actions = self._engine.convert_action_values(
@@ -1071,6 +1074,9 @@ def doBulkRemoveRecord(self, table_id, row_ids_or_records):
     assert all(isinstance(r, (int, table.Record)) for r in row_ids_or_records)
     row_ids = [int(r) for r in row_ids_or_records]
 
+    # Replace negative ids that may refer to rows just added to this table in this bundle.
+    row_ids = self._engine.out_actions.summary.translate_new_row_ids(table_id, row_ids)
+
     self._do_doc_action(actions.BulkRemoveRecord(table_id, row_ids))
 
     # Also remove any references to this row from other tables.
diff --git a/test/server/lib/helpers/TestServer.ts b/test/server/lib/helpers/TestServer.ts
index 959468598f..080abb1e9c 100644
--- a/test/server/lib/helpers/TestServer.ts
+++ b/test/server/lib/helpers/TestServer.ts
@@ -79,7 +79,7 @@ export class TestServer {
       throw new Error(`Path of testingSocket too long: ${this.testingSocket.length} (${this.testingSocket})`);
     }
 
-    const port = await getAvailablePort();
+    const port = await getAvailablePort(Number(process.env.GET_AVAILABLE_PORT_START || '8000'));
     this._serverUrl = `http://localhost:${port}`;
     const homeUrl = _homeUrl ?? (this._serverTypes.includes('home') ? this._serverUrl : undefined);
 

From 184be9387fd2153299df0f30e67a1d94338b99dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jaros=C5=82aw=20Sadzi=C5=84ski?=
 <jaroslaw.sadzinski@gmail.com>
Date: Wed, 26 Jun 2024 15:49:13 +0200
Subject: [PATCH 02/16] (core) Enabling telemetry on /api/version endpoint

Summary:
Version API endpoint wasn't logging telemetry from POST requests. The issue was in registration
order, this endpoint was registered before `expressJson` and it couldn't read json body in the handler.

Test Plan: Added new test

Reviewers: paulfitz

Reviewed By: paulfitz

Subscribers: paulfitz

Differential Revision: https://phab.getgrist.com/D4277
---
 app/server/lib/FlexServer.ts        |  2 +-
 app/server/lib/UpdateManager.ts     |  5 +---
 app/server/mergedServerMain.ts      |  2 +-
 test/client/lib/UrlState.ts         |  5 ----
 test/client/models/gristUrlState.ts |  2 --
 test/gen-server/UpdateChecks.ts     | 40 ++++++++++++++++++++++++++---
 test/server/Comm.ts                 | 10 --------
 test/server/lib/DocApi.ts           |  2 +-
 8 files changed, 41 insertions(+), 27 deletions(-)

diff --git a/app/server/lib/FlexServer.ts b/app/server/lib/FlexServer.ts
index 46e4a50847..015b3c11f4 100644
--- a/app/server/lib/FlexServer.ts
+++ b/app/server/lib/FlexServer.ts
@@ -1971,7 +1971,7 @@ export class FlexServer implements GristServer {
   }
 
   public addUpdatesCheck() {
-    if (this._check('update')) { return; }
+    if (this._check('update', 'json')) { return; }
 
     // For now we only are active for sass deployments.
     if (this._deploymentType !== 'saas') { return; }
diff --git a/app/server/lib/UpdateManager.ts b/app/server/lib/UpdateManager.ts
index c7ac9f6765..a79c5b2c98 100644
--- a/app/server/lib/UpdateManager.ts
+++ b/app/server/lib/UpdateManager.ts
@@ -85,15 +85,12 @@ export class UpdateManager {
       const payload = (name: string) => req.body?.[name] ?? req.query[name];
 
       // This is the most interesting part for us, to track installation ids and match them
-      // with the version of the client. Won't be send without telemetry opt in.
+      // with the version of the client.
       const deploymentId = optStringParam(
         payload("installationId"),
         "installationId"
       );
 
-      // Current version of grist-core part of the client. Currently not used and not
-      // passed from the client.
-
       // Deployment type of the client (we expect this to be 'core' for most of the cases).
       const deploymentType = optStringParam(
         payload("deploymentType"),
diff --git a/app/server/mergedServerMain.ts b/app/server/mergedServerMain.ts
index 8405cd8def..987343f6a0 100644
--- a/app/server/mergedServerMain.ts
+++ b/app/server/mergedServerMain.ts
@@ -106,7 +106,6 @@ export async function main(port: number, serverTypes: ServerType[],
   server.addHealthCheck();
   if (includeHome || includeApp) {
     server.addBootPage();
-    server.addUpdatesCheck();
   }
   server.denyRequestsIfNotReady();
 
@@ -148,6 +147,7 @@ export async function main(port: number, serverTypes: ServerType[],
         server.addDocApiForwarder();
       }
       server.addJsonSupport();
+      server.addUpdatesCheck();
       await server.addLandingPages();
       // todo: add support for home api to standalone app
       server.addHomeApi();
diff --git a/test/client/lib/UrlState.ts b/test/client/lib/UrlState.ts
index 5de50c2ec9..356c3c0ed5 100644
--- a/test/client/lib/UrlState.ts
+++ b/test/client/lib/UrlState.ts
@@ -1,14 +1,11 @@
-import * as log from 'app/client/lib/log';
 import {HistWindow, UrlState} from 'app/client/lib/UrlState';
 import {assert} from 'chai';
 import {dom} from 'grainjs';
 import {popGlobals, pushGlobals} from 'grainjs/dist/cjs/lib/browserGlobals';
 import {JSDOM} from 'jsdom';
 import fromPairs = require('lodash/fromPairs');
-import * as sinon from 'sinon';
 
 describe('UrlState', function() {
-  const sandbox = sinon.createSandbox();
   let mockWindow: HistWindow;
 
   function pushState(state: any, title: any, href: string) {
@@ -26,12 +23,10 @@ describe('UrlState', function() {
     // These grainjs browserGlobals are needed for using dom() in tests.
     const jsdomDoc = new JSDOM("<!doctype html><html><body></body></html>");
     pushGlobals(jsdomDoc.window);
-    sandbox.stub(log, 'debug');
   });
 
   afterEach(function() {
     popGlobals();
-    sandbox.restore();
   });
 
   interface State {
diff --git a/test/client/models/gristUrlState.ts b/test/client/models/gristUrlState.ts
index c7065fef5d..764b00fdf9 100644
--- a/test/client/models/gristUrlState.ts
+++ b/test/client/models/gristUrlState.ts
@@ -1,4 +1,3 @@
-import * as log from 'app/client/lib/log';
 import {HistWindow, UrlState} from 'app/client/lib/UrlState';
 import {getLoginUrl, UrlStateImpl} from 'app/client/models/gristUrlState';
 import {IGristUrlState} from 'app/common/gristUrls';
@@ -42,7 +41,6 @@ describe('gristUrlState', function() {
     // These grainjs browserGlobals are needed for using dom() in tests.
     const jsdomDoc = new JSDOM("<!doctype html><html><body></body></html>");
     pushGlobals(jsdomDoc.window);
-    sandbox.stub(log, 'debug');
   });
 
   afterEach(function() {
diff --git a/test/gen-server/UpdateChecks.ts b/test/gen-server/UpdateChecks.ts
index cdd9426fa9..53441f8cd8 100644
--- a/test/gen-server/UpdateChecks.ts
+++ b/test/gen-server/UpdateChecks.ts
@@ -5,10 +5,12 @@ import * as sinon from 'sinon';
 import { configForUser } from "test/gen-server/testUtils";
 import * as testUtils from "test/server/testUtils";
 import { Defer, serveSomething, Serving } from "test/server/customUtil";
+import { Telemetry } from 'app/server/lib/Telemetry';
 import { Deps } from "app/server/lib/UpdateManager";
 import { TestServer } from "test/gen-server/apiUtils";
 import { delay } from "app/common/delay";
 import { LatestVersion } from 'app/common/InstallAPI';
+import { TelemetryEvent, TelemetryMetadataByLevel } from 'app/common/Telemetry';
 
 const assert = chai.assert;
 
@@ -21,8 +23,13 @@ const stop = async () => {
 
 let homeUrl: string;
 let dockerHub: Serving & { signal: () => Defer };
+let sandbox: sinon.SinonSandbox;
+const logMessages: [TelemetryEvent, TelemetryMetadataByLevel?][] = [];
 
 const chimpy = configForUser("Chimpy");
+const headers = {
+  headers: {'Content-Type': 'application/json'}
+};
 
 // Tests specific complex scenarios that may have previously resulted in wrong behavior.
 describe("UpdateChecks", function () {
@@ -30,8 +37,6 @@ describe("UpdateChecks", function () {
 
   this.timeout("20s");
 
-  const sandbox = sinon.createSandbox();
-
   before(async function () {
     testUtils.EnvironmentSnapshot.push();
     dockerHub = await dummyDockerHub();
@@ -41,11 +46,19 @@ describe("UpdateChecks", function () {
     Object.assign(process.env, {
       GRIST_TEST_SERVER_DEPLOYMENT_TYPE: "saas",
     });
+    sandbox = sinon.createSandbox();
     sandbox.stub(Deps, "REQUEST_TIMEOUT").value(300);
     sandbox.stub(Deps, "RETRY_TIMEOUT").value(400);
     sandbox.stub(Deps, "GOOD_RESULT_TTL").value(500);
     sandbox.stub(Deps, "BAD_RESULT_TTL").value(200);
     sandbox.stub(Deps, "DOCKER_ENDPOINT").value(dockerHub.url + "/tags");
+    sandbox.stub(Telemetry.prototype, 'logEvent').callsFake((_, name, meta) => {
+      if (name !== 'checkedUpdateAPI') {
+        return Promise.resolve();
+      }
+      logMessages.push([name, meta]);
+      return Promise.resolve();
+    });
 
     await startInProcess(this);
   });
@@ -69,7 +82,7 @@ describe("UpdateChecks", function () {
     assert.equal(result.latestVersion, "10");
 
     // Also works in post method.
-    const resp2 = await axios.post(`${homeUrl}/api/version`);
+    const resp2 = await axios.post(`${homeUrl}/api/version`, {}, headers);
     assert.equal(resp2.status, 200);
     assert.deepEqual(resp2.data, result);
   });
@@ -197,6 +210,27 @@ describe("UpdateChecks", function () {
     assert.equal(resp.status, 500);
     assert.match(resp.data.error, /timeout/);
   });
+
+  it("logs deploymentId and deploymentType", async function () {
+    logMessages.length = 0;
+    setEndpoint(dockerHub.url + "/tags");
+    const installationId = "randomInstallationId";
+    const deploymentType = "test";
+    const resp = await axios.post(`${homeUrl}/api/version`, {
+      installationId,
+      deploymentType
+    }, chimpy);
+    assert.equal(resp.status, 200);
+    assert.equal(logMessages.length, 1);
+    const [name, meta] = logMessages[0];
+    assert.equal(name, "checkedUpdateAPI");
+    assert.deepEqual(meta, {
+      full: {
+        deploymentId: installationId,
+        deploymentType,
+      },
+    });
+  });
 });
 
 async function dummyDockerHub() {
diff --git a/test/server/Comm.ts b/test/server/Comm.ts
index 51e21a0e68..eb9794869e 100644
--- a/test/server/Comm.ts
+++ b/test/server/Comm.ts
@@ -102,16 +102,6 @@ describe('Comm', function() {
     }
   };
 
-  beforeEach(function() {
-    // Silence console messages from client-side Comm.ts.
-    if (!process.env.VERBOSE) {
-      // TODO: This no longer works, now that 'log' is a more proper "module" object rather than
-      // an arbitrary JS object. Also used in a couple other tests where logs are no longer
-      // silenced.
-      sandbox.stub(log, 'debug');
-    }
-  });
-
   afterEach(async function() {
     // Run the cleanup callbacks registered in cleanup().
     await Promise.all(cleanup.splice(0).map(callback => callback()));
diff --git a/test/server/lib/DocApi.ts b/test/server/lib/DocApi.ts
index b816dc3fea..86b515d3b8 100644
--- a/test/server/lib/DocApi.ts
+++ b/test/server/lib/DocApi.ts
@@ -4029,7 +4029,7 @@ function testDocApi() {
           ...pick(options, 'name', 'memo', 'enabled', 'watchedColIds'),
         }, chimpy
       );
-      assert.equal(status, 200);
+      assert.equal(status, 200, `Error during subscription: ` + JSON.stringify(data));
       return data as WebhookSubscription;
     }
 

From 3769c5791551da4ec4eda906492455bc106ddf11 Mon Sep 17 00:00:00 2001
From: Paul Janzen <pj@paulgjanzen.com>
Date: Sat, 29 Jun 2024 19:53:47 +0000
Subject: [PATCH 03/16] Translated using Weblate (Portuguese (Brazil))

Currently translated at 100.0% (1340 of 1340 strings)

Translation: Grist/client
Translate-URL: https://hosted.weblate.org/projects/grist/client/pt_BR/
---
 static/locales/pt_BR.client.json | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/static/locales/pt_BR.client.json b/static/locales/pt_BR.client.json
index 06457ef547..74685c7561 100644
--- a/static/locales/pt_BR.client.json
+++ b/static/locales/pt_BR.client.json
@@ -1617,7 +1617,11 @@
         "Check failed.": "A verificação falhou.",
         "Current authentication method": "Método de autenticação atual",
         "Grist allows different types of authentication to be configured, including SAML and OIDC.     We recommend enabling one of these if Grist is accessible over the network or being made available     to multiple people.": "O Grist permite a configuração de diferentes tipos de autenticação, incluindo SAML e OIDC.     Recomendamos ativar um desses tipos se o Grist for acessível pela rede ou estiver sendo disponibilizado para várias pessoas.",
-        "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "O Grist permite a configuração de diferentes tipos de autenticação, incluindo SAML e OIDC. Recomendamos ativar um desses tipos se o Grist for acessível pela rede ou estiver sendo disponibilizado para várias pessoas."
+        "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "O Grist permite a configuração de diferentes tipos de autenticação, incluindo SAML e OIDC. Recomendamos ativar um desses tipos se o Grist for acessível pela rede ou estiver sendo disponibilizado para várias pessoas.",
+        "Key to sign sessions with": "Chave para assinar sessões com",
+        "Session Secret": "Segredo da sessão",
+        "Grist signs user session cookies with a secret key. Please set this key via the environment variable GRIST_SESSION_SECRET. Grist falls back to a hard-coded default when it is not set. We may remove this notice in the future since session IDs have been updated to be inherently cryptographically secure.": "O Grist assina os cookies de sessão do usuário com uma chave secreta. Defina essa chave por meio da variável de ambiente GRIST_SESSION_SECRET. O Grist retorna a um padrão codificado quando ele não está definido. Poderemos remover esse aviso no futuro, pois os IDs de sessão gerados desde a versão 1.1.16 são inerentemente seguros em termos de criptografia.",
+        "Grist signs user session cookies with a secret key. Please set this key via the environment variable GRIST_SESSION_SECRET. Grist falls back to a hard-coded default when it is not set. We may remove this notice in the future as session IDs generated since v1.1.16 are inherently cryptographically secure.": "O Grist assina os cookies de sessão do usuário com uma chave secreta. Defina essa chave por meio da variável de ambiente GRIST_SESSION_SECRET. O Grist retorna a um padrão codificado quando ele não está definido. Poderemos remover esse aviso no futuro, pois os IDs de sessão gerados desde a versão 1.1.16 são inerentemente seguros em termos de criptografia."
     },
     "Field": {
         "No choices configured": "Nenhuma opção configurada",

From d6d9d1c52e63aa04571c6d3eab4a7d7c50f05772 Mon Sep 17 00:00:00 2001
From: Paul Janzen <pj@paulgjanzen.com>
Date: Sat, 29 Jun 2024 19:54:59 +0000
Subject: [PATCH 04/16] Translated using Weblate (Spanish)

Currently translated at 100.0% (1340 of 1340 strings)

Translation: Grist/client
Translate-URL: https://hosted.weblate.org/projects/grist/client/es/
---
 static/locales/es.client.json | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/static/locales/es.client.json b/static/locales/es.client.json
index 9a54b025fa..b52afd761a 100644
--- a/static/locales/es.client.json
+++ b/static/locales/es.client.json
@@ -1607,7 +1607,11 @@
         "Or, as a fallback, you can set: {{bootKey}} in the environment and visit: {{url}}": "O, como alternativa, puedes configurar: {{bootKey}} en el entorno y visita: {{url}}",
         "You do not have access to the administrator panel.\nPlease log in as an administrator.": "No tienes acceso al panel de administrador.\nInicia sesión como administrador.",
         "Self Checks": "Controles automáticos",
-        "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "Grist permite configurar diferentes tipos de autenticación, incluidos SAML y OIDC. Recomendamos habilitar uno de estos si se puede acceder a Grist a través de la red o si está disponible para varias personas."
+        "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "Grist permite configurar diferentes tipos de autenticación, incluidos SAML y OIDC. Recomendamos habilitar uno de estos si se puede acceder a Grist a través de la red o si está disponible para varias personas.",
+        "Session Secret": "Secreto de sesión",
+        "Grist signs user session cookies with a secret key. Please set this key via the environment variable GRIST_SESSION_SECRET. Grist falls back to a hard-coded default when it is not set. We may remove this notice in the future since session IDs have been updated to be inherently cryptographically secure.": "Grist firma las cookies de sesión de usuario con una clave secreta. Establezca esta clave mediante la variable de entorno GRIST_SESSION_SECRET. Si no se establece, Grist vuelve a un valor predeterminado. Es posible que quitemos este aviso en el futuro, ya que los identificadores de sesión generados desde la versión 1.1.16 son intrínsecamente seguros desde el punto de vista criptográfico.",
+        "Key to sign sessions with": "Clave para firmar sesiones con",
+        "Grist signs user session cookies with a secret key. Please set this key via the environment variable GRIST_SESSION_SECRET. Grist falls back to a hard-coded default when it is not set. We may remove this notice in the future as session IDs generated since v1.1.16 are inherently cryptographically secure.": "Grist firma las cookies de sesión de usuario con una clave secreta. Establezca esta clave mediante la variable de entorno GRIST_SESSION_SECRET. Si no se establece, Grist vuelve a un valor predeterminado. Es posible que quitemos este aviso en el futuro, ya que los identificadores de sesión generados desde la versión 1.1.16 son intrínsecamente seguros desde el punto de vista criptográfico."
     },
     "CreateTeamModal": {
         "Cancel": "Cancelar",

From 0e777b1fcf3e9d68e2817b8b2247467cf34c724c Mon Sep 17 00:00:00 2001
From: Paul Janzen <pj@paulgjanzen.com>
Date: Sat, 29 Jun 2024 19:55:29 +0000
Subject: [PATCH 05/16] Translated using Weblate (German)

Currently translated at 100.0% (1340 of 1340 strings)

Translation: Grist/client
Translate-URL: https://hosted.weblate.org/projects/grist/client/de/
---
 static/locales/de.client.json | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/static/locales/de.client.json b/static/locales/de.client.json
index f7ed1c6d4b..8bb45bba12 100644
--- a/static/locales/de.client.json
+++ b/static/locales/de.client.json
@@ -1613,7 +1613,11 @@
         "No fault detected.": "Kein Fehler erkannt.",
         "Grist allows different types of authentication to be configured, including SAML and OIDC.     We recommend enabling one of these if Grist is accessible over the network or being made available     to multiple people.": "In Grist können verschiedene Arten der Authentifizierung konfiguriert werden, darunter SAML und OIDC.     Wir empfehlen, eine davon zu aktivieren, wenn Grist über das Netzwerk zugänglich ist oder mehreren Personen zur Verfügung gestellt wird.",
         "Or, as a fallback, you can set: {{bootKey}} in the environment and visit: {{url}}": "Als Ausweichmöglichkeit können Sie auch {{bootKey}} in der Umgebung einstellen und {{url}} besuchen",
-        "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "In Grist können verschiedene Arten der Authentifizierung konfiguriert werden, darunter SAML und OIDC. Wir empfehlen, eine davon zu aktivieren, wenn Grist über das Netzwerk zugänglich ist oder mehreren Personen zur Verfügung gestellt wird."
+        "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "In Grist können verschiedene Arten der Authentifizierung konfiguriert werden, darunter SAML und OIDC. Wir empfehlen, eine davon zu aktivieren, wenn Grist über das Netzwerk zugänglich ist oder mehreren Personen zur Verfügung gestellt wird.",
+        "Grist signs user session cookies with a secret key. Please set this key via the environment variable GRIST_SESSION_SECRET. Grist falls back to a hard-coded default when it is not set. We may remove this notice in the future since session IDs have been updated to be inherently cryptographically secure.": "Grist signiert die Sitzungscookies der Benutzer mit einem geheimen Schlüssel. Bitte setzen Sie diesen Schlüssel über die Umgebungsvariable GRIST_SESSION_SECRET. Grist greift auf eine hart kodierte Voreinstellung zurück, wenn sie nicht gesetzt ist. Wir werden diesen Hinweis möglicherweise in Zukunft entfernen, da Sitzungs-IDs, die seit v1.1.16 erzeugt werden, von Natur aus kryptographisch sicher sind.",
+        "Grist signs user session cookies with a secret key. Please set this key via the environment variable GRIST_SESSION_SECRET. Grist falls back to a hard-coded default when it is not set. We may remove this notice in the future as session IDs generated since v1.1.16 are inherently cryptographically secure.": "Grist signiert die Sitzungscookies der Benutzer mit einem geheimen Schlüssel. Bitte setzen Sie diesen Schlüssel über die Umgebungsvariable GRIST_SESSION_SECRET. Grist greift auf eine hart kodierte Voreinstellung zurück, wenn sie nicht gesetzt ist. Wir werden diesen Hinweis möglicherweise in Zukunft entfernen, da Sitzungs-IDs, die seit v1.1.16 erzeugt werden, von Natur aus kryptographisch sicher sind.",
+        "Key to sign sessions with": "Schlüssel zum Anmelden von Sitzungen mit",
+        "Session Secret": "Sitzungsgeheimnis"
     },
     "Section": {
         "Insert section above": "Abschnitt oben einfügen",

From 61421e82510f924010e356ab6b36a8b7bad3afa7 Mon Sep 17 00:00:00 2001
From: CamilleLegeron <camille@telescoop.fr>
Date: Mon, 1 Jul 2024 15:13:39 +0200
Subject: [PATCH 06/16] Create user last connection datetime (#935)

Each time the a Grist page is reload the `last_connection_at` of the user is updated

resolve [#924](https://github.com/gristlabs/grist-core/issues/924)
---
 app/gen-server/entity/User.ts                 |  3 ++
 app/gen-server/lib/homedb/UsersManager.ts     | 27 +++++++++++-----
 .../migration/1663851423064-UserUUID.ts       | 17 +++++++---
 .../migration/1664528376930-UserRefUnique.ts  | 21 ++++++++----
 .../1713186031023-UserLastConnection.ts       | 18 +++++++++++
 app/server/lib/requestUtils.ts                |  4 +--
 test/gen-server/migrations.ts                 | 32 ++++++++++++++++++-
 7 files changed, 101 insertions(+), 21 deletions(-)
 create mode 100644 app/gen-server/migration/1713186031023-UserLastConnection.ts

diff --git a/app/gen-server/entity/User.ts b/app/gen-server/entity/User.ts
index 2ed1016910..c93837cbf2 100644
--- a/app/gen-server/entity/User.ts
+++ b/app/gen-server/entity/User.ts
@@ -29,6 +29,9 @@ export class User extends BaseEntity {
   @Column({name: 'first_login_at', type: Date, nullable: true})
   public firstLoginAt: Date | null;
 
+  @Column({name: 'last_connection_at', type: Date, nullable: true})
+  public lastConnectionAt: Date | null;
+
   @OneToOne(type => Organization, organization => organization.owner)
   public personalOrg: Organization;
 
diff --git a/app/gen-server/lib/homedb/UsersManager.ts b/app/gen-server/lib/homedb/UsersManager.ts
index 8c0a5dcae1..168665f354 100644
--- a/app/gen-server/lib/homedb/UsersManager.ts
+++ b/app/gen-server/lib/homedb/UsersManager.ts
@@ -395,14 +395,6 @@ export class UsersManager {
         user.name = (profile && (profile.name || email.split('@')[0])) || '';
         needUpdate = true;
       }
-      if (profile && !user.firstLoginAt) {
-        // set first login time to now (remove milliseconds for compatibility with other
-        // timestamps in db set by typeorm, and since second level precision is fine)
-        const nowish = new Date();
-        nowish.setMilliseconds(0);
-        user.firstLoginAt = nowish;
-        needUpdate = true;
-      }
       if (!user.picture && profile && profile.picture) {
         // Set the user's profile picture if our provider knows it.
         user.picture = profile.picture;
@@ -432,6 +424,25 @@ export class UsersManager {
         user.options = {...(user.options ?? {}), authSubject: userOptions.authSubject};
         needUpdate = true;
       }
+
+      // get date of now (remove milliseconds for compatibility with other
+      // timestamps in db set by typeorm, and since second level precision is fine)
+      const nowish = new Date();
+      nowish.setMilliseconds(0);
+      if (profile && !user.firstLoginAt) {
+        // set first login time to now
+        user.firstLoginAt = nowish;
+        needUpdate = true;
+      }
+      const getTimestampStartOfDay = (date: Date) => {
+        const timestamp = Math.floor(date.getTime() / 1000); // unix timestamp seconds from epoc
+        const startOfDay = timestamp - (timestamp % 86400 /*24h*/); // start of a day in seconds since epoc
+        return startOfDay;
+      };
+      if (!user.lastConnectionAt || getTimestampStartOfDay(user.lastConnectionAt) !== getTimestampStartOfDay(nowish)) {
+        user.lastConnectionAt = nowish;
+        needUpdate = true;
+      }
       if (needUpdate) {
         login.user = user;
         await manager.save([user, login]);
diff --git a/app/gen-server/migration/1663851423064-UserUUID.ts b/app/gen-server/migration/1663851423064-UserUUID.ts
index ba0e71b1f7..60c8666829 100644
--- a/app/gen-server/migration/1663851423064-UserUUID.ts
+++ b/app/gen-server/migration/1663851423064-UserUUID.ts
@@ -1,5 +1,5 @@
-import {User} from 'app/gen-server/entity/User';
 import {makeId} from 'app/server/lib/idUtils';
+import {chunk} from 'lodash';
 import {MigrationInterface, QueryRunner, TableColumn} from "typeorm";
 
 export class UserUUID1663851423064 implements MigrationInterface {
@@ -16,11 +16,20 @@ export class UserUUID1663851423064 implements MigrationInterface {
     // Updating so many rows in a multiple queries is not ideal. We will send updates in chunks.
     // 300 seems to be a good number, for 24k rows we have 80 queries.
     const userList = await queryRunner.manager.createQueryBuilder()
-      .select("users")
-      .from(User, "users")
+      .select(["users.id", "users.ref"])
+      .from("users", "users")
       .getMany();
     userList.forEach(u => u.ref = makeId());
-    await queryRunner.manager.save(userList, { chunk: 300 });
+
+    const userChunks = chunk(userList, 300);
+    for (const users of userChunks) {
+      await queryRunner.connection.transaction(async manager => {
+        const queries = users.map((user: any, _index: number, _array: any[]) => {
+          return queryRunner.manager.update("users", user.id, user);
+        });
+        await Promise.all(queries);
+      });
+    }
 
     // We are not making this column unique yet, because it can fail
     // if there are some old workers still running, and any new user
diff --git a/app/gen-server/migration/1664528376930-UserRefUnique.ts b/app/gen-server/migration/1664528376930-UserRefUnique.ts
index 2753604250..149be01ee5 100644
--- a/app/gen-server/migration/1664528376930-UserRefUnique.ts
+++ b/app/gen-server/migration/1664528376930-UserRefUnique.ts
@@ -1,5 +1,5 @@
-import {User} from 'app/gen-server/entity/User';
 import {makeId} from 'app/server/lib/idUtils';
+import {chunk} from 'lodash';
 import {MigrationInterface, QueryRunner} from "typeorm";
 
 export class UserRefUnique1664528376930 implements MigrationInterface {
@@ -9,12 +9,21 @@ export class UserRefUnique1664528376930 implements MigrationInterface {
 
     // Update users that don't have unique ref set.
     const userList = await queryRunner.manager.createQueryBuilder()
-      .select("users")
-      .from(User, "users")
-      .where("ref is null")
-      .getMany();
+    .select(["users.id", "users.ref"])
+    .from("users", "users")
+    .where("users.ref is null")
+    .getMany();
     userList.forEach(u => u.ref = makeId());
-    await queryRunner.manager.save(userList, {chunk: 300});
+
+    const userChunks = chunk(userList, 300);
+    for (const users of userChunks) {
+      await queryRunner.connection.transaction(async manager => {
+        const queries = users.map((user: any, _index: number, _array: any[]) => {
+          return queryRunner.manager.update("users", user.id, user);
+        });
+        await Promise.all(queries);
+      });
+    }
 
     // Mark column as unique and non-nullable.
     const users = (await queryRunner.getTable('users'))!;
diff --git a/app/gen-server/migration/1713186031023-UserLastConnection.ts b/app/gen-server/migration/1713186031023-UserLastConnection.ts
new file mode 100644
index 0000000000..52310a3898
--- /dev/null
+++ b/app/gen-server/migration/1713186031023-UserLastConnection.ts
@@ -0,0 +1,18 @@
+import {MigrationInterface, QueryRunner, TableColumn} from 'typeorm';
+
+export class UserLastConnection1713186031023 implements MigrationInterface {
+
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    const sqlite = queryRunner.connection.driver.options.type === 'sqlite';
+    const datetime = sqlite ? "datetime" : "timestamp with time zone";
+    await queryRunner.addColumn('users', new TableColumn({
+      name: 'last_connection_at',
+      type: datetime,
+      isNullable: true
+    }));
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropColumn('users', 'last_connection_at');
+  }
+}
diff --git a/app/server/lib/requestUtils.ts b/app/server/lib/requestUtils.ts
index a6f291065f..de0326d478 100644
--- a/app/server/lib/requestUtils.ts
+++ b/app/server/lib/requestUtils.ts
@@ -21,8 +21,8 @@ export const TEST_HTTPS_OFFSET = process.env.GRIST_TEST_HTTPS_OFFSET ?
 
 // Database fields that we permit in entities but don't want to cross the api.
 const INTERNAL_FIELDS = new Set([
-  'apiKey', 'billingAccountId', 'firstLoginAt', 'filteredOut', 'ownerId', 'gracePeriodStart', 'stripeCustomerId',
-  'stripeSubscriptionId', 'stripeProductId', 'userId', 'isFirstTimeUser', 'allowGoogleLogin',
+  'apiKey', 'billingAccountId', 'firstLoginAt', 'lastConnectionAt', 'filteredOut', 'ownerId', 'gracePeriodStart',
+  'stripeCustomerId', 'stripeSubscriptionId', 'stripeProductId', 'userId', 'isFirstTimeUser', 'allowGoogleLogin',
   'authSubject', 'usage', 'createdBy'
 ]);
 
diff --git a/test/gen-server/migrations.ts b/test/gen-server/migrations.ts
index 9b3e31e5cf..e6a45b9862 100644
--- a/test/gen-server/migrations.ts
+++ b/test/gen-server/migrations.ts
@@ -42,6 +42,8 @@ import {ActivationPrefs1682636695021 as ActivationPrefs} from 'app/gen-server/mi
 import {AssistantLimit1685343047786 as AssistantLimit} from 'app/gen-server/migration/1685343047786-AssistantLimit';
 import {Shares1701557445716 as Shares} from 'app/gen-server/migration/1701557445716-Shares';
 import {Billing1711557445716 as BillingFeatures} from 'app/gen-server/migration/1711557445716-Billing';
+import {UserLastConnection1713186031023
+        as UserLastConnection} from 'app/gen-server/migration/1713186031023-UserLastConnection';
 
 const home: HomeDBManager = new HomeDBManager();
 
@@ -50,7 +52,8 @@ const migrations = [Initial, Login, PinDocs, UserPicture, DisplayEmail, DisplayE
                     CustomerIndex, ExtraIndexes, OrgHost, DocRemovedAt, Prefs,
                     ExternalBilling, DocOptions, Secret, UserOptions, GracePeriodStart,
                     DocumentUsage, Activations, UserConnectId, UserUUID, UserUniqueRefUUID,
-                    Forks, ForkIndexes, ActivationPrefs, AssistantLimit, Shares, BillingFeatures];
+                    Forks, ForkIndexes, ActivationPrefs, AssistantLimit, Shares, BillingFeatures,
+                    UserLastConnection];
 
 // Assert that the "members" acl rule and group exist (or not).
 function assertMembersGroup(org: Organization, exists: boolean) {
@@ -113,6 +116,33 @@ describe('migrations', function() {
     // be doing something.
   });
 
+  it('can migrate UserUUID and UserUniqueRefUUID with user in table', async function() {
+    this.timeout(60000);
+    const runner = home.connection.createQueryRunner();
+
+    // Create 400 users to test the chunk (each chunk is 300 users)
+    const nbUsersToCreate = 400;
+    for (const migration of migrations) {
+      if (migration === UserUUID) {
+        for (let i = 0; i < nbUsersToCreate; i++) {
+          await runner.query(`INSERT INTO users (id, name, is_first_time_user) VALUES (${i}, 'name${i}', true)`);
+        }
+      }
+
+      await (new migration()).up(runner);
+    }
+
+    // Check that all refs are unique
+    const userList = await runner.manager.createQueryBuilder()
+      .select(["users.id", "users.ref"])
+      .from("users", "users")
+      .getMany();
+    const setOfUserRefs = new Set(userList.map(u => u.ref));
+    assert.equal(nbUsersToCreate, userList.length);
+    assert.equal(setOfUserRefs.size, userList.length);
+    await addSeedData(home.connection);
+  });
+
   it('can correctly switch display_email column to non-null with data', async function() {
     this.timeout(60000);
     const sqlite = home.connection.driver.options.type === 'sqlite';

From 6e11e497bc80591739ed49d2ccc9286bcc2652f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jordi=20Guti=C3=A9rrez=20Hermoso?= <jordigh@octave.org>
Date: Fri, 28 Jun 2024 20:15:59 -0400
Subject: [PATCH 07/16] workflows: Do not use `ext/` director to run tests

We need this directory for building the image, but not for running the
tests outside of it.
---
 .github/workflows/docker_latest.yml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/docker_latest.yml b/.github/workflows/docker_latest.yml
index 4a6f2c1c9e..01abfd84e3 100644
--- a/.github/workflows/docker_latest.yml
+++ b/.github/workflows/docker_latest.yml
@@ -82,14 +82,16 @@ jobs:
 
       - name: Build Node.js code
         run: |
-          pushd ext && \
-          { if [ -e package.json ] ; then yarn install --frozen-lockfile --modules-folder=../../node_modules; fi } && \
-          popd
+          rm -rf ext
           yarn run build:prod
 
       - name: Run tests
         run: TEST_IMAGE=${{ github.repository_owner }}/${{ matrix.image.name }}:experimental VERBOSE=1 DEBUG=1 MOCHA_WEBDRIVER_HEADLESS=1 yarn run test:docker
 
+      - name: Restore the ext/ directory
+        if: matrix.image.name != 'grist-oss'
+        run: buildtools/checkout-ext-directory.sh ${{ matrix.image.repo }}
+
       - name: Log in to Docker Hub
         uses: docker/login-action@v1 
         with:

From 6888f9bceeb7d96db524be46ff414bf664c7fb6d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jordi=20Guti=C3=A9rrez=20Hermoso?= <jordigh@octave.org>
Date: Fri, 28 Jun 2024 20:16:50 -0400
Subject: [PATCH 08/16] tsconfig-ext: revert
 bc52f65b2648ff7987383537fc06c6a388d29ce2

While the intent was to run tests with it, we don't need it. Instead,
this caused problems because the stubs overrode the intended `ext`
directory and therefore disabled the ext features.
---
 tsconfig-ext.json | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tsconfig-ext.json b/tsconfig-ext.json
index cfa355ab7e..814d5c361d 100644
--- a/tsconfig-ext.json
+++ b/tsconfig-ext.json
@@ -3,9 +3,6 @@
   "files": [],
   "include": [],
   "references": [
-    { "path": "./app" },
-    { "path": "./stubs/app" },
-    { "path": "./test" },
     { "path": "./ext/app" }
   ],
 }

From 6801732c29030540464acc1c44b939fd1aed398f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Gr=C3=A9goire=20Cutzach?= <gregoire@cutzach.com>
Date: Mon, 1 Jul 2024 09:15:53 +0000
Subject: [PATCH 09/16] Translated using Weblate (French)

Currently translated at 99.1% (1329 of 1340 strings)

Translation: Grist/client
Translate-URL: https://hosted.weblate.org/projects/grist/client/fr/
---
 static/locales/fr.client.json | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/static/locales/fr.client.json b/static/locales/fr.client.json
index 1c64e75c48..6f5a9536ab 100644
--- a/static/locales/fr.client.json
+++ b/static/locales/fr.client.json
@@ -335,7 +335,9 @@
         "Formula timer": "Chronomètre de formule",
         "Timing is on": "Le chronomètre tourne",
         "You can make changes to the document, then stop timing to see the results.": "Vous pouvez apporter des modifications au document, puis arrêter le chronométrage pour voir les résultats.",
-        "Formula times": "Minuteur de formule"
+        "Formula times": "Minuteur de formule",
+        "Only available to document editors": "Seulement disponible aux éditeurs du document",
+        "Only available to document owners": "Seulement disponible aux propriétaires du document"
     },
     "DocumentUsage": {
         "Usage statistics are only available to users with full access to the document data.": "Les statistiques d'utilisation ne sont disponibles qu'aux utilisateurs ayant un accès complet aux données du document.",
@@ -1120,7 +1122,8 @@
         "min": "min",
         "Text": "Texte",
         "max": "max",
-        "Field Format": "Format du champ"
+        "Field Format": "Format du champ",
+        "Spinner": "Roue"
     },
     "LanguageMenu": {
         "Language": "Langue"
@@ -1547,7 +1550,15 @@
         "Details": "Détails",
         "Grist allows different types of authentication to be configured, including SAML and OIDC.     We recommend enabling one of these if Grist is accessible over the network or being made available     to multiple people.": "Grist permet de configurer différents types d'authentification, notamment SAML et OIDC. Nous recommandons d'activer l'un de ces types d'authentification si Grist est accessible via le réseau ou s'il est mis à la disposition de plusieurs personnes.",
         "You do not have access to the administrator panel.\nPlease log in as an administrator.": "Vous n'avez pas accès au panneau d'administrateur.\nVeuillez vous connecter en tant qu'administrateur.",
-        "Results": "Résultats"
+        "Results": "Résultats",
+        "Grist signs user session cookies with a secret key. Please set this key via the environment variable GRIST_SESSION_SECRET. Grist falls back to a hard-coded default when it is not set. We may remove this notice in the future since session IDs have been updated to be inherently cryptographically secure.": "Grist signe les cookies des sessions utilisateurs avec une clé secrète. Merci de renseigner cette clé via la variable d'environnement GRIST_SESSION_SECRET. Grist se replie sur une clé codée en dur par défaut si la variable n'est pas renseignée. La présente remarque sera peut-être retirée dans le futur comme les identifiants de session générés depuis la version 1.1.16 sont cryptographiquement sûrs.",
+        "Key to sign sessions with": "Clé de signature des sessions",
+        "Grist allows different types of authentication to be configured, including SAML and OIDC. We recommend enabling one of these if Grist is accessible over the network or being made available to multiple people.": "Grist autorise la configuration de différents types d'authentifications, parmi lesquels SAML et OIDC. Nous recommandons d'activer l'une d'entre elles si Grist est accessible sur le réseau ou est rendu accessible à plusieurs personnes.",
+        "Grist signs user session cookies with a secret key. Please set this key via the environment variable GRIST_SESSION_SECRET. Grist falls back to a hard-coded default when it is not set. We may remove this notice in the future as session IDs generated since v1.1.16 are inherently cryptographically secure.": "Grist signe les cookies des sessions utilisateurs avec une clé secrète. Merci de renseigner cette clé via la variable d'environnement GRIST_SESSION_SECRET. Grist utilise par défaut une clé codée en dur si la variable n'est pas renseignée. Nous retirerons peut-être cet avertissement à l'avenir comme les identifiants de session générés depuis la version 1.1.16 sont intrinsèquement cryptographiquement sûrs.",
+        "Sandboxing": "Bac à sable",
+        "Self Checks": "Auto contrôles",
+        "Session Secret": "Secret de session",
+        "Or, as a fallback, you can set: {{bootKey}} in the environment and visit: {{url}}": "Ou, comme plan B., vous pouvez renseigner : {{bootKey}} dans l'environnement et visiter : {{url}}"
     },
     "Field": {
         "No choices configured": "Aucun choix configuré",

From 3082fe0f01c79cc2bbb979d1a271a91f0aa5572d Mon Sep 17 00:00:00 2001
From: Roman Holinec <3ko@pixeon.sk>
Date: Mon, 1 Jul 2024 11:39:29 +0000
Subject: [PATCH 10/16] Translated using Weblate (Slovak)

Currently translated at 100.0% (1340 of 1340 strings)

Translation: Grist/client
Translate-URL: https://hosted.weblate.org/projects/grist/client/sk/
---
 static/locales/sk.client.json | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/static/locales/sk.client.json b/static/locales/sk.client.json
index 7bc059b39c..8dbd9f36c8 100644
--- a/static/locales/sk.client.json
+++ b/static/locales/sk.client.json
@@ -245,7 +245,7 @@
         "Permanently Delete \"{{name}}\"?": "Natrvalo Odstrániť „{{name}}“?",
         "Pin Document": "Pripnúť Dokument",
         "Pinned Documents": "Pripnuté Dokumenty",
-        "Remove": "Odstrániť",
+        "Remove": "Odobrať",
         "Rename": "Premenovať",
         "Requires edit permissions": "Vyžaduje povolenia na úpravy",
         "To restore this document, restore the workspace first.": "Ak chcete tento dokument obnoviť, najskôr obnovte pracovný priestor.",
@@ -278,7 +278,7 @@
         "Add Empty Table": "Pridať Prázdnu Tabuľku",
         "Add Page": "Pridať Stránku",
         "Add Widget to Page": "Pridať Miniaplikáciu na Stránku",
-        "Document owners can attempt to recover the document. [{{error}}]": "Vlastníci dokumentu sa môžu pokúsiť dokument obnoviť. [{{chyba}}]",
+        "Document owners can attempt to recover the document. [{{error}}]": "Vlastníci dokumentu sa môžu pokúsiť dokument obnoviť. {{chyba}}",
         "Sorry, access to this document has been denied. [{{error}}]": "Ľutujeme, prístup k tomuto dokumentu bol odmietnutý. [{{error}}]",
         "You can try reloading the document, or using recovery mode. Recovery mode opens the document to be fully accessible to owners, and inaccessible to others. It also disables formulas. [{{error}}]": "Môžete skúsiť znova načítať dokument alebo použiť režim obnovenia. Režim obnovenia otvorí dokument tak, aby bol plne prístupný pre vlastníkov a neprístupný pre ostatných. Zakáže tiež vzorce. [{{error}}]",
         "You do not have edit access to this document": "Nemáte prístup k úpravám tohto dokumentu"
@@ -516,7 +516,7 @@
         "To use Grist, please either sign up or sign in.": "Ak chcete používať Grist, zaregistrujte sa alebo sa prihláste.",
         "Visit our {{link}} to learn more about Grist.": "Navštíviť náš {{link}} a dozvedieť sa viac o Grist.",
         "Learn more in our {{helpCenterLink}}, or find an expert via our {{sproutsProgram}}.": "Viac informácií nájdete v našom {{helpCenterLink}} alebo nájdite odborníka prostredníctvom nášho {{sproutsProgram}}.",
-        "Interested in using Grist outside of your team? Visit your free ": "Máte záujem používať Grist mimo váš tím?  Navštívte svoje bezplatné ",
+        "Interested in using Grist outside of your team? Visit your free ": "Máte záujem používať Grist mimo váš tím? Navštívte svoje bezplatné ",
         "Sign up": "Prihlásiť sa"
     },
     "Importer": {
@@ -1304,7 +1304,10 @@
         "Security Settings": "Bezpečnostné nastavenia",
         "Updates": "Aktualizácie",
         "unconfigured": "nekonfigurované",
-        "unknown": "neznáme"
+        "unknown": "neznáme",
+        "Grist signs user session cookies with a secret key. Please set this key via the environment variable GRIST_SESSION_SECRET. Grist falls back to a hard-coded default when it is not set. We may remove this notice in the future as session IDs generated since v1.1.16 are inherently cryptographically secure.": "Grist podpisuje súbory cookie relácie používateľa tajným kľúčom. Nastavte tento kľúč prostredníctvom premennej prostredia GRIST_SESSION_SECRET. Grist sa vráti späť na pevne zakódované predvolené nastavenie, keď nie je nastavené. Toto upozornenie môžeme v budúcnosti odstrániť, pretože ID relácie generované od verzie 1.1.16 sú vo svojej podstate kryptograficky bezpečné.",
+        "Key to sign sessions with": "Kľúč na podpisovanie relácií",
+        "Session Secret": "Tajomstvo Relácie"
     },
     "TimingPage": {
         "Table ID": "ID Tabuľky",

From 4815a007edf46823b78b96d4a354fcbcc8ee3607 Mon Sep 17 00:00:00 2001
From: Paul Fitzpatrick <paulfitz@alum.mit.edu>
Date: Mon, 1 Jul 2024 10:24:16 -0400
Subject: [PATCH 11/16] log periodic per-document statistics about snapshot
 generation

This is to facilitate alerting to detect if snapshot generation were to
stall for a document.
---
 app/common/normalizedDateTimeString.ts | 27 ++++++++++++++++
 app/gen-server/lib/Housekeeper.ts      | 28 +----------------
 app/server/lib/ActiveDoc.ts            | 21 +++++++++++++
 app/server/lib/DocStorageManager.ts    |  6 +++-
 app/server/lib/HostedStorageManager.ts | 43 ++++++++++++++++++++++++--
 app/server/lib/IDocStorageManager.ts   | 42 +++++++++++++++++++++++++
 6 files changed, 137 insertions(+), 30 deletions(-)
 create mode 100644 app/common/normalizedDateTimeString.ts

diff --git a/app/common/normalizedDateTimeString.ts b/app/common/normalizedDateTimeString.ts
new file mode 100644
index 0000000000..ad07c3bfcc
--- /dev/null
+++ b/app/common/normalizedDateTimeString.ts
@@ -0,0 +1,27 @@
+import moment from 'moment';
+
+/**
+ * Output an ISO8601 format datetime string, with timezone.
+ * Any string fed in without timezone is expected to be in UTC.
+ *
+ * When connected to postgres, dates will be extracted as Date objects,
+ * with timezone information. The normalization done here is not
+ * really needed in this case.
+ *
+ * Timestamps in SQLite are stored as UTC, and read as strings
+ * (without timezone information). The normalization here is
+ * pretty important in this case.
+ */
+export function normalizedDateTimeString(dateTime: any): string {
+  if (!dateTime) { return dateTime; }
+  if (dateTime instanceof Date) {
+    return moment(dateTime).toISOString();
+  }
+  if (typeof dateTime === 'string' || typeof dateTime === 'number') {
+    // When SQLite returns a string, it will be in UTC.
+    // Need to make sure it actually have timezone info in it
+    // (will not by default).
+    return moment.utc(dateTime).toISOString();
+  }
+  throw new Error(`normalizedDateTimeString cannot handle ${dateTime}`);
+}
diff --git a/app/gen-server/lib/Housekeeper.ts b/app/gen-server/lib/Housekeeper.ts
index 116a3c508c..c40379fe15 100644
--- a/app/gen-server/lib/Housekeeper.ts
+++ b/app/gen-server/lib/Housekeeper.ts
@@ -1,6 +1,7 @@
 import { ApiError } from 'app/common/ApiError';
 import { delay } from 'app/common/delay';
 import { buildUrlId } from 'app/common/gristUrls';
+import { normalizedDateTimeString } from 'app/common/normalizedDateTimeString';
 import { BillingAccount } from 'app/gen-server/entity/BillingAccount';
 import { Document } from 'app/gen-server/entity/Document';
 import { Organization } from 'app/gen-server/entity/Organization';
@@ -16,7 +17,6 @@ import log from 'app/server/lib/log';
 import { IPermitStore } from 'app/server/lib/Permit';
 import { optStringParam, stringParam } from 'app/server/lib/requestUtils';
 import * as express from 'express';
-import moment from 'moment';
 import fetch from 'node-fetch';
 import * as Fetch from 'node-fetch';
 import { EntityManager } from 'typeorm';
@@ -416,32 +416,6 @@ export class Housekeeper {
   }
 }
 
-/**
- * Output an ISO8601 format datetime string, with timezone.
- * Any string fed in without timezone is expected to be in UTC.
- *
- * When connected to postgres, dates will be extracted as Date objects,
- * with timezone information. The normalization done here is not
- * really needed in this case.
- *
- * Timestamps in SQLite are stored as UTC, and read as strings
- * (without timezone information). The normalization here is
- * pretty important in this case.
- */
-function normalizedDateTimeString(dateTime: any): string {
-  if (!dateTime) { return dateTime; }
-  if (dateTime instanceof Date) {
-    return moment(dateTime).toISOString();
-  }
-  if (typeof dateTime === 'string') {
-    // When SQLite returns a string, it will be in UTC.
-    // Need to make sure it actually have timezone info in it
-    // (will not by default).
-    return moment.utc(dateTime).toISOString();
-  }
-  throw new Error(`normalizedDateTimeString cannot handle ${dateTime}`);
-}
-
 /**
  * Call callback(item) for each item on the list, sleeping periodically to allow other works to
  * happen. Any time work takes more than SYNC_WORK_LIMIT_MS, will sleep for SYNC_WORK_BREAK_MS.
diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts
index addf127820..c3fb5be024 100644
--- a/app/server/lib/ActiveDoc.ts
+++ b/app/server/lib/ActiveDoc.ts
@@ -69,6 +69,7 @@ import {commonUrls, parseUrlId} from 'app/common/gristUrls';
 import {byteString, countIf, retryOnce, safeJsonParse, timeoutReached} from 'app/common/gutil';
 import {InactivityTimer} from 'app/common/InactivityTimer';
 import {Interval} from 'app/common/Interval';
+import {normalizedDateTimeString} from 'app/common/normalizedDateTimeString';
 import {
   compilePredicateFormula,
   getPredicateFormulaProperties,
@@ -2496,6 +2497,23 @@ export class ActiveDoc extends EventEmitter {
     }
   }
 
+  private _logSnapshotProgress(docSession: OptDocSession) {
+    const snapshotProgress = this._docManager.storageManager.getSnapshotProgress(this.docName);
+    const lastWindowTime = (snapshotProgress.lastWindowStartedAt &&
+        snapshotProgress.lastWindowDoneAt &&
+        snapshotProgress.lastWindowDoneAt > snapshotProgress.lastWindowStartedAt) ?
+        snapshotProgress.lastWindowDoneAt : Date.now();
+    const delay = snapshotProgress.lastWindowStartedAt ?
+        lastWindowTime - snapshotProgress.lastWindowStartedAt : null;
+    this._log.debug(docSession, 'snapshot status', {
+      ...snapshotProgress,
+      lastChangeAt: normalizedDateTimeString(snapshotProgress.lastChangeAt),
+      lastWindowStartedAt: normalizedDateTimeString(snapshotProgress.lastWindowStartedAt),
+      lastWindowDoneAt: normalizedDateTimeString(snapshotProgress.lastWindowDoneAt),
+      delay,
+    });
+  }
+
   private _logDocMetrics(docSession: OptDocSession, triggeredBy: 'docOpen' | 'interval'| 'docClose') {
     this.logTelemetryEvent(docSession, 'documentUsage', {
       limited: {
@@ -2513,6 +2531,9 @@ export class ActiveDoc extends EventEmitter {
         ...this._getCustomWidgetMetrics(),
       },
     });
+    // Log progress on making snapshots periodically, to catch anything
+    // excessively slow.
+    this._logSnapshotProgress(docSession);
   }
 
   private _getAccessRuleMetrics() {
diff --git a/app/server/lib/DocStorageManager.ts b/app/server/lib/DocStorageManager.ts
index 24c8628ea0..dbcb0e5928 100644
--- a/app/server/lib/DocStorageManager.ts
+++ b/app/server/lib/DocStorageManager.ts
@@ -11,7 +11,7 @@ import * as gutil from 'app/common/gutil';
 import {Comm} from 'app/server/lib/Comm';
 import * as docUtils from 'app/server/lib/docUtils';
 import {GristServer} from 'app/server/lib/GristServer';
-import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
+import {IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager';
 import {IShell} from 'app/server/lib/IShell';
 import log from 'app/server/lib/log';
 import uuidv4 from "uuid/v4";
@@ -257,6 +257,10 @@ export class DocStorageManager implements IDocStorageManager {
     throw new Error('removeSnapshots not implemented');
   }
 
+  public getSnapshotProgress(): SnapshotProgress {
+    throw new Error('getSnapshotProgress not implemented');
+  }
+
   public async replace(docName: string, options: any): Promise<void> {
     throw new Error('replacement not implemented');
   }
diff --git a/app/server/lib/HostedStorageManager.ts b/app/server/lib/HostedStorageManager.ts
index 03f801381c..9be39f8547 100644
--- a/app/server/lib/HostedStorageManager.ts
+++ b/app/server/lib/HostedStorageManager.ts
@@ -15,7 +15,7 @@ import {IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
 import {ChecksummedExternalStorage, DELETED_TOKEN, ExternalStorage, Unchanged} from 'app/server/lib/ExternalStorage';
 import {HostedMetadataManager} from 'app/server/lib/HostedMetadataManager';
 import {ICreate} from 'app/server/lib/ICreate';
-import {IDocStorageManager} from 'app/server/lib/IDocStorageManager';
+import {IDocStorageManager, SnapshotProgress} from 'app/server/lib/IDocStorageManager';
 import {LogMethods} from "app/server/lib/LogMethods";
 import {fromCallback} from 'app/server/lib/serverUtils';
 import * as fse from 'fs-extra';
@@ -94,6 +94,9 @@ export class HostedStorageManager implements IDocStorageManager {
   // Time at which document was last changed.
   private _timestamps = new Map<string, string>();
 
+  // Statistics related to snapshot generation.
+  private _snapshotProgress = new Map<string, SnapshotProgress>();
+
   // Access external storage.
   private _ext: ChecksummedExternalStorage;
   private _extMeta: ChecksummedExternalStorage;
@@ -223,6 +226,25 @@ export class HostedStorageManager implements IDocStorageManager {
     return path.basename(altDocName, '.grist');
   }
 
+  /**
+   * Read some statistics related to generating snapshots.
+   */
+  public getSnapshotProgress(docName: string): SnapshotProgress {
+    let snapshotProgress = this._snapshotProgress.get(docName);
+    if (!snapshotProgress) {
+      snapshotProgress = {
+        pushes: 0,
+        skippedPushes: 0,
+        errors: 0,
+        changes: 0,
+        windowsStarted: 0,
+        windowsDone: 0,
+      };
+      this._snapshotProgress.set(docName, snapshotProgress);
+    }
+    return snapshotProgress;
+  }
+
   /**
    * Prepares a document for use locally. Here we sync the doc from S3 to the local filesystem.
    * Returns whether the document is new (needs to be created).
@@ -476,7 +498,11 @@ export class HostedStorageManager implements IDocStorageManager {
    * This is called when a document may have been changed, via edits or migrations etc.
    */
   public markAsChanged(docName: string, reason?: string): void {
-    const timestamp = new Date().toISOString();
+    const now = new Date();
+    const snapshotProgress = this.getSnapshotProgress(docName);
+    snapshotProgress.lastChangeAt = now.getTime();
+    snapshotProgress.changes++;
+    const timestamp = now.toISOString();
     this._timestamps.set(docName, timestamp);
     try {
       if (parseUrlId(docName).snapshotId) { return; }
@@ -486,6 +512,10 @@ export class HostedStorageManager implements IDocStorageManager {
       }
       if (this._disableS3) { return; }
       if (this._closed) { throw new Error("HostedStorageManager.markAsChanged called after closing"); }
+      if (!this._uploads.hasPendingOperation(docName)) {
+        snapshotProgress.lastWindowStartedAt = now.getTime();
+        snapshotProgress.windowsStarted++;
+      }
       this._uploads.addOperation(docName);
     } finally {
       if (reason === 'edit') {
@@ -729,6 +759,7 @@ export class HostedStorageManager implements IDocStorageManager {
   private async _pushToS3(docId: string): Promise<void> {
     let tmpPath: string|null = null;
 
+    const snapshotProgress = this.getSnapshotProgress(docId);
     try {
       if (this._prepareFiles.has(docId)) {
         throw new Error('too soon to consider pushing');
@@ -748,14 +779,18 @@ export class HostedStorageManager implements IDocStorageManager {
       await this._inventory.uploadAndAdd(docId, async () => {
         const prevSnapshotId = this._latestVersions.get(docId) || null;
         const newSnapshotId = await this._ext.upload(docId, tmpPath as string, metadata);
+        snapshotProgress.lastWindowDoneAt = Date.now();
+        snapshotProgress.windowsDone++;
         if (newSnapshotId === Unchanged) {
           // Nothing uploaded because nothing changed
+          snapshotProgress.skippedPushes++;
           return { prevSnapshotId };
         }
         if (!newSnapshotId) {
           // This is unexpected.
           throw new Error('No snapshotId allocated after upload');
         }
+        snapshotProgress.pushes++;
         const snapshot = {
           lastModified: t,
           snapshotId: newSnapshotId,
@@ -767,6 +802,10 @@ export class HostedStorageManager implements IDocStorageManager {
       if (changeMade) {
         await this._onInventoryChange(docId);
       }
+    } catch (e) {
+      snapshotProgress.errors++;
+      // Snapshot window completion time deliberately not set.
+      throw e;
     } finally {
       // Clean up backup.
       // NOTE: fse.remove succeeds also when the file does not exist.
diff --git a/app/server/lib/IDocStorageManager.ts b/app/server/lib/IDocStorageManager.ts
index 54180a9807..a7eba5405f 100644
--- a/app/server/lib/IDocStorageManager.ts
+++ b/app/server/lib/IDocStorageManager.ts
@@ -36,6 +36,8 @@ export interface IDocStorageManager {
   // Metadata may not be returned in this case.
   getSnapshots(docName: string, skipMetadataCache?: boolean): Promise<DocSnapshots>;
   removeSnapshots(docName: string, snapshotIds: string[]): Promise<void>;
+  // Get information about how snapshot generation is going.
+  getSnapshotProgress(docName: string): SnapshotProgress;
   replace(docName: string, options: DocReplacementOptions): Promise<void>;
 }
 
@@ -66,5 +68,45 @@ export class TrivialDocStorageManager implements IDocStorageManager {
   public async flushDoc() {}
   public async getSnapshots(): Promise<never> { throw new Error('no'); }
   public async removeSnapshots(): Promise<never> { throw new Error('no'); }
+  public getSnapshotProgress(): SnapshotProgress { throw new Error('no'); }
   public async replace(): Promise<never> { throw new Error('no'); }
 }
+
+
+/**
+ * Some summary information about how snapshot generation is going.
+ * Any times are in ms.
+ * All information is within the lifetime of a doc worker, not global.
+ */
+export interface SnapshotProgress {
+  // The last time the document was marked as having changed.
+  lastChangeAt?: number;
+
+  // The last time a save window started for the document (checking to see
+  // if it needs to be pushed, and pushing it if so, possibly waiting
+  // quite some time to bundle any other changes).
+  lastWindowStartedAt?: number;
+
+  // The last time the document was either pushed or determined to not
+  // actually need to be pushed, after having been marked as changed.
+  lastWindowDoneAt?: number;
+
+  // Number of times the document was pushed.
+  pushes: number;
+
+  // Number of times the document was not pushed because no change found.
+  skippedPushes: number;
+
+  // Number of times there was an error trying to push.
+  errors: number;
+
+  // Number of times the document was marked as changed.
+  // Will generally be a lot greater than saves.
+  changes: number;
+
+  // Number of times a save window was started.
+  windowsStarted: number;
+
+  // Number of times a save window was completed.
+  windowsDone: number;
+}

From 95b8134614e093ca114e166ff8a9717af6297488 Mon Sep 17 00:00:00 2001
From: Paul Fitzpatrick <paulfitz@alum.mit.edu>
Date: Tue, 2 Jul 2024 06:52:57 -0400
Subject: [PATCH 12/16] add a getSnapshotProgress implementation to
 DocStorageManager

---
 app/server/lib/DocStorageManager.ts | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/app/server/lib/DocStorageManager.ts b/app/server/lib/DocStorageManager.ts
index dbcb0e5928..8879db42ee 100644
--- a/app/server/lib/DocStorageManager.ts
+++ b/app/server/lib/DocStorageManager.ts
@@ -258,7 +258,14 @@ export class DocStorageManager implements IDocStorageManager {
   }
 
   public getSnapshotProgress(): SnapshotProgress {
-    throw new Error('getSnapshotProgress not implemented');
+    return {
+      pushes: 0,
+      skippedPushes: 0,
+      errors: 0,
+      changes: 0,
+      windowsStarted: 0,
+      windowsDone: 0,
+    };
   }
 
   public async replace(docName: string, options: any): Promise<void> {

From 5f9ecdcfe4b5589ffb14590e796977e56ce41d36 Mon Sep 17 00:00:00 2001
From: Paul Fitzpatrick <paulfitz@alum.mit.edu>
Date: Wed, 3 Jul 2024 11:16:42 -0400
Subject: [PATCH 13/16] docstrings, moment import, fix log format

---
 app/common/normalizedDateTimeString.ts |  2 +-
 app/server/lib/ActiveDoc.ts            |  3 ++-
 app/server/lib/IDocStorageManager.ts   | 32 +++++++++++++++-----------
 3 files changed, 22 insertions(+), 15 deletions(-)

diff --git a/app/common/normalizedDateTimeString.ts b/app/common/normalizedDateTimeString.ts
index ad07c3bfcc..5197d78494 100644
--- a/app/common/normalizedDateTimeString.ts
+++ b/app/common/normalizedDateTimeString.ts
@@ -1,4 +1,4 @@
-import moment from 'moment';
+import moment from 'moment-timezone';
 
 /**
  * Output an ISO8601 format datetime string, with timezone.
diff --git a/app/server/lib/ActiveDoc.ts b/app/server/lib/ActiveDoc.ts
index c3fb5be024..e4e9d7a0cf 100644
--- a/app/server/lib/ActiveDoc.ts
+++ b/app/server/lib/ActiveDoc.ts
@@ -2505,7 +2505,8 @@ export class ActiveDoc extends EventEmitter {
         snapshotProgress.lastWindowDoneAt : Date.now();
     const delay = snapshotProgress.lastWindowStartedAt ?
         lastWindowTime - snapshotProgress.lastWindowStartedAt : null;
-    this._log.debug(docSession, 'snapshot status', {
+    log.rawInfo('snapshot status', {
+      ...this.getLogMeta(docSession),
       ...snapshotProgress,
       lastChangeAt: normalizedDateTimeString(snapshotProgress.lastChangeAt),
       lastWindowStartedAt: normalizedDateTimeString(snapshotProgress.lastWindowStartedAt),
diff --git a/app/server/lib/IDocStorageManager.ts b/app/server/lib/IDocStorageManager.ts
index a7eba5405f..6bde92cb1f 100644
--- a/app/server/lib/IDocStorageManager.ts
+++ b/app/server/lib/IDocStorageManager.ts
@@ -79,34 +79,40 @@ export class TrivialDocStorageManager implements IDocStorageManager {
  * All information is within the lifetime of a doc worker, not global.
  */
 export interface SnapshotProgress {
-  // The last time the document was marked as having changed.
+  /** The last time the document was marked as having changed. */
   lastChangeAt?: number;
 
-  // The last time a save window started for the document (checking to see
-  // if it needs to be pushed, and pushing it if so, possibly waiting
-  // quite some time to bundle any other changes).
+  /**
+   * The last time a save window started for the document (checking to see
+   * if it needs to be pushed, and pushing it if so, possibly waiting
+   * quite some time to bundle any other changes).
+   */
   lastWindowStartedAt?: number;
 
-  // The last time the document was either pushed or determined to not
-  // actually need to be pushed, after having been marked as changed.
+  /**
+   * The last time the document was either pushed or determined to not
+   * actually need to be pushed, after having been marked as changed.
+   */
   lastWindowDoneAt?: number;
 
-  // Number of times the document was pushed.
+  /** Number of times the document was pushed. */
   pushes: number;
 
-  // Number of times the document was not pushed because no change found.
+  /** Number of times the document was not pushed because no change found. */
   skippedPushes: number;
 
-  // Number of times there was an error trying to push.
+  /** Number of times there was an error trying to push. */
   errors: number;
 
-  // Number of times the document was marked as changed.
-  // Will generally be a lot greater than saves.
+  /**
+   * Number of times the document was marked as changed.
+   * Will generally be a lot greater than saves.
+   */
   changes: number;
 
-  // Number of times a save window was started.
+  /** Number of times a save window was started. */
   windowsStarted: number;
 
-  // Number of times a save window was completed.
+  /** Number of times a save window was completed. */
   windowsDone: number;
 }

From 2750ed6bd9831510d6169a5500df8398650c4c86 Mon Sep 17 00:00:00 2001
From: Leslie H <142967379+SleepyLeslie@users.noreply.github.com>
Date: Wed, 3 Jul 2024 15:36:17 -0400
Subject: [PATCH 14/16] Enable external contributors to create previews (#1068)

Reorganize preview workflows so that previews can be made for PRs from outside contributors.
---
 .github/workflows/fly-build.yml   | 43 +++++++++++++++
 .github/workflows/fly-cleanup.yml | 20 +++----
 .github/workflows/fly-deploy.yml  | 70 +++++++++++++++++++++++
 .github/workflows/fly-destroy.yml | 36 ++++++++++++
 .github/workflows/fly.yml         | 64 ---------------------
 buildtools/fly-deploy.js          | 92 ++++++++++++++++++-------------
 buildtools/fly-template.toml      |  5 ++
 7 files changed, 217 insertions(+), 113 deletions(-)
 create mode 100644 .github/workflows/fly-build.yml
 create mode 100644 .github/workflows/fly-deploy.yml
 create mode 100644 .github/workflows/fly-destroy.yml
 delete mode 100644 .github/workflows/fly.yml

diff --git a/.github/workflows/fly-build.yml b/.github/workflows/fly-build.yml
new file mode 100644
index 0000000000..26c5fee5a6
--- /dev/null
+++ b/.github/workflows/fly-build.yml
@@ -0,0 +1,43 @@
+# fly-deploy will be triggered on completion of this workflow to actually deploy the code to fly.io.
+
+name: fly.io Build
+on:
+  pull_request:
+    branches: [ main ]
+    types: [labeled, opened, synchronize, reopened]
+
+  # Allows running this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  build:
+    name: Build Docker image
+    runs-on: ubuntu-latest
+    # Build when the 'preview' label is added, or when PR is updated with this label present.
+    if: >
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'pull_request' &&
+      contains(github.event.pull_request.labels.*.name, 'preview'))
+    steps:
+      - uses: actions/checkout@v4
+      - name: Build and export Docker image
+        id: docker-build
+        run: >
+          docker build -t grist-core:preview . &&
+          docker image save grist-core:preview -o grist-core.tar
+      - name: Save PR information
+        run: |
+          echo PR_NUMBER=${{ github.event.number }} >> ./pr-info.txt
+          echo PR_SOURCE=${{ github.event.pull_request.head.repo.full_name }}-${{ github.event.pull_request.head.ref }} >> ./pr-info.txt
+          echo PR_SHASUM=${{ github.event.pull_request.head.sha }} >> ./pr-info.txt
+        # PR_SOURCE looks like <owner>/<repo>-<branch>.
+        # For example, if the GitHub user "foo" forked grist-core as "grist-bar", and makes a PR from their branch named "baz",
+        # it will be "foo/grist-bar-baz". deploy.js later replaces "/" with "-", making it "foo-grist-bar-baz".
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker-image
+          path: |
+            ./grist-core.tar
+            ./pr-info.txt
+          if-no-files-found: "error"
diff --git a/.github/workflows/fly-cleanup.yml b/.github/workflows/fly-cleanup.yml
index 256d2f0fe2..6250e589ae 100644
--- a/.github/workflows/fly-cleanup.yml
+++ b/.github/workflows/fly-cleanup.yml
@@ -1,4 +1,4 @@
-name: Fly Cleanup
+name: fly.io Cleanup
 on:
   schedule:
     # Once a day, clean up jobs marked as expired
@@ -12,12 +12,12 @@ env:
 
 jobs:
   clean:
-      name: Clean stale deployed apps
-      runs-on: ubuntu-latest
-      if: github.repository_owner == 'gristlabs'
-      steps:
-        - uses: actions/checkout@v3
-        - uses: superfly/flyctl-actions/setup-flyctl@master
-          with:
-            version: 0.1.66
-        - run: node buildtools/fly-deploy.js clean
+    name: Clean stale deployed apps
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'gristlabs'
+    steps:
+      - uses: actions/checkout@v3
+      - uses: superfly/flyctl-actions/setup-flyctl@master
+        with:
+          version: 0.2.72
+      - run: node buildtools/fly-deploy.js clean
diff --git a/.github/workflows/fly-deploy.yml b/.github/workflows/fly-deploy.yml
new file mode 100644
index 0000000000..5a4c071182
--- /dev/null
+++ b/.github/workflows/fly-deploy.yml
@@ -0,0 +1,70 @@
+# Follow-up of fly-build, with access to secrets for making deployments.
+# This workflow runs in the target repo context. It does not, and should never execute user-supplied code.
+# See https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+
+name: fly.io Deploy
+on:
+  workflow_run:
+    workflows: ["fly.io Build"]
+    types:
+      - completed
+
+jobs:
+  deploy:
+    name: Deploy app to fly.io
+    runs-on: ubuntu-latest
+    if: |
+      github.event.workflow_run.event == 'pull_request' &&
+      github.event.workflow_run.conclusion == 'success'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up flyctl
+        uses: superfly/flyctl-actions/setup-flyctl@master
+        with:
+          version: 0.2.72
+      - name: Download artifacts
+        uses: actions/github-script@v7
+        with:
+          script: |
+            var artifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: ${{ github.event.workflow_run.id }},
+            });
+            var matchArtifact = artifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "docker-image"
+            })[0];
+            var download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            var fs = require('fs');
+            fs.writeFileSync('${{github.workspace}}/docker-image.zip', Buffer.from(download.data));
+      - name: Extract artifacts
+        id: extract_artifacts
+        run: |
+          unzip docker-image.zip
+          cat ./pr-info.txt >> $GITHUB_OUTPUT
+      - name: Load Docker image
+        run: docker load --input grist-core.tar
+      - name: Deploy to fly.io
+        id: fly_deploy
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+          BRANCH_NAME: ${{ steps.extract_artifacts.outputs.PR_SOURCE }}
+        run: |
+          node buildtools/fly-deploy.js deploy
+          flyctl config -c ./fly.toml env | awk '/APP_HOME_URL/{print "DEPLOY_URL=" $2}' >> $GITHUB_OUTPUT
+          flyctl config -c ./fly.toml env | awk '/FLY_DEPLOY_EXPIRATION/{print "EXPIRES=" $2}' >> $GITHUB_OUTPUT
+      - name: Comment on PR
+        uses: actions/github-script@v7
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: ${{ steps.extract_artifacts.outputs.PR_NUMBER }},
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: `Deployed commit \`${{ steps.extract_artifacts.outputs.PR_SHASUM }}\` as ${{ steps.fly_deploy.outputs.DEPLOY_URL }} (until ${{ steps.fly_deploy.outputs.EXPIRES }})`
+            })
diff --git a/.github/workflows/fly-destroy.yml b/.github/workflows/fly-destroy.yml
new file mode 100644
index 0000000000..1fe204a72e
--- /dev/null
+++ b/.github/workflows/fly-destroy.yml
@@ -0,0 +1,36 @@
+# This workflow runs in the target repo context, as it is triggered via pull_request_target.
+# It does not, and should not have access to code in the PR.
+# See https://securitylab.github.com/research/github-actions-preventing-pwn-requests/
+
+name: fly.io Destroy
+on:
+  pull_request_target:
+    branches: [ main ]
+    types: [unlabeled, closed]
+
+  # Allows running this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  destroy:
+    name: Remove app from fly.io
+    runs-on: ubuntu-latest
+    # Remove the deployment when 'preview' label is removed, or the PR is closed.
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      (github.event_name == 'pull_request_target' &&
+      (github.event.action == 'closed' ||
+      (github.event.action == 'unlabeled' && github.event.label.name == 'preview')))
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up flyctl
+        uses: superfly/flyctl-actions/setup-flyctl@master
+        with:
+          version: 0.2.72
+      - name: Destroy fly.io app
+        env:
+          FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
+          BRANCH_NAME: ${{ github.event.pull_request.head.repo.full_name }}-${{ github.event.pull_request.head.ref }}
+          # See fly-build for what BRANCH_NAME looks like.
+        id: fly_destroy
+        run: node buildtools/fly-deploy.js destroy
diff --git a/.github/workflows/fly.yml b/.github/workflows/fly.yml
deleted file mode 100644
index 5f7d10b8f2..0000000000
--- a/.github/workflows/fly.yml
+++ /dev/null
@@ -1,64 +0,0 @@
-name: Fly Deploy
-on:
-  pull_request:
-    branches: [ main ]
-    types: [labeled, unlabeled, closed, opened, synchronize, reopened]
-
-  # Allows running this workflow manually from the Actions tab
-  workflow_dispatch:
-
-env:
-  FLY_API_TOKEN: ${{ secrets.FLY_API_TOKEN }}
-  BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
-
-jobs:
-  deploy:
-    name: Deploy app
-    runs-on: ubuntu-latest
-    # Deploy when the 'preview' label is added, or when PR is updated with this label present.
-    if: |
-          github.repository_owner == 'gristlabs' &&
-          github.event_name == 'pull_request' && (
-            github.event.action == 'labeled' ||
-            github.event.action == 'opened' ||
-            github.event.action == 'synchronize' ||
-            github.event.action == 'reopened'
-          ) &&
-          contains(github.event.pull_request.labels.*.name, 'preview')
-    steps:
-      - uses: actions/checkout@v3
-      - uses: superfly/flyctl-actions/setup-flyctl@master
-        with:
-          version: 0.1.89
-      - id: fly_deploy
-        run: |
-          node buildtools/fly-deploy.js deploy
-          flyctl config -c ./fly.toml env | awk '/APP_HOME_URL/{print "DEPLOY_URL=" $2}' >> $GITHUB_OUTPUT
-          flyctl config -c ./fly.toml env | awk '/FLY_DEPLOY_EXPIRATION/{print "EXPIRES=" $2}' >> $GITHUB_OUTPUT
-
-      - uses: actions/github-script@v6
-        with:
-          script: |
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: `Deployed as ${{ steps.fly_deploy.outputs.DEPLOY_URL }} (until ${{ steps.fly_deploy.outputs.EXPIRES }})`
-            })
-
-  destroy:
-    name: Remove app
-    runs-on: ubuntu-latest
-    # Remove the deployment when 'preview' label is removed, or the PR is closed.
-    if: |
-          github.repository_owner == 'gristlabs' &&
-          github.event_name == 'pull_request' &&
-          (github.event.action == 'closed' ||
-           (github.event.action == 'unlabeled' && github.event.label.name == 'preview'))
-    steps:
-      - uses: actions/checkout@v3
-      - uses: superfly/flyctl-actions/setup-flyctl@master
-        with:
-          version: 0.1.89
-      - id: fly_destroy
-        run: node buildtools/fly-deploy.js destroy
diff --git a/buildtools/fly-deploy.js b/buildtools/fly-deploy.js
index e5432f29d6..f2bb5c8e0b 100644
--- a/buildtools/fly-deploy.js
+++ b/buildtools/fly-deploy.js
@@ -1,7 +1,6 @@
 const util = require('util');
 const childProcess = require('child_process');
 const fs = require('fs/promises');
-const {existsSync} = require('fs');
 
 const exec = util.promisify(childProcess.exec);
 
@@ -17,66 +16,81 @@ const getBranchName = () => {
 };
 
 async function main() {
-  if (process.argv[2] === 'deploy') {
-    const appRoot = process.argv[3] || ".";
-    if (!existsSync(`${appRoot}/Dockerfile`)) {
-      console.log(`Dockerfile not found in appRoot of ${appRoot}`);
-      process.exit(1);
-    }
-
-    const name = getAppName();
-    const volName = getVolumeName();
-    if (!await appExists(name)) {
-      await appCreate(name);
-      await volCreate(name, volName);
-    } else {
-      // Check if volume exists, and create it if not. This is needed because there was an API
-      // change in flyctl (mandatory -y flag) and some apps were created without a volume.
-      if (!(await volList(name)).length) {
+  switch (process.argv[2]) {
+    case "deploy": {
+      const name = getAppName();
+      const volName = getVolumeName();
+      if (!await appExists(name)) {
+        await appCreate(name);
         await volCreate(name, volName);
+      } else {
+        // Check if volume exists, and create it if not. This is needed because there was an API
+        // change in flyctl (mandatory -y flag) and some apps were created without a volume.
+        if (!(await volList(name)).length) {
+          await volCreate(name, volName);
+        }
       }
+      await prepConfig(name, volName);
+      await appDeploy(name);
+      break;
     }
-    await prepConfig(name, appRoot, volName);
-    await appDeploy(name, appRoot);
-  } else if (process.argv[2] === 'destroy') {
-    const name = getAppName();
-    if (await appExists(name)) {
-      await appDestroy(name);
+    case "destroy": {
+      const name = getAppName();
+      if (await appExists(name)) {
+        await appDestroy(name);
+      }
+      break;
     }
-  } else if (process.argv[2] === 'clean') {
-    const staleApps = await findStaleApps();
-    for (const appName of staleApps) {
-      await appDestroy(appName);
+    case "clean": {
+      const staleApps = await findStaleApps();
+      for (const appName of staleApps) {
+        await appDestroy(appName);
+      }
+      break;
     }
-  } else {
-    console.log(`Usage:
-  deploy [appRoot]:
-            create (if needed) and deploy fly app grist-{BRANCH_NAME}.
-            appRoot may specify the working directory that contains the Dockerfile to build.
+    default: {
+      console.log(`Usage:
+  deploy:   create (if needed) and deploy fly app grist-{BRANCH_NAME}.
   destroy:  destroy fly app grist-{BRANCH_NAME}
   clean:    destroy all grist-* fly apps whose time has come
             (according to FLY_DEPLOY_EXPIRATION env var set at deploy time)
 
   DRYRUN=1 in environment will show what would be done
 `);
-    process.exit(1);
+      process.exit(1);
+    }
   }
 }
 
+function getDockerTag(name) {
+  return `registry.fly.io/${name}:latest`;
+}
+
 const appExists = (name) => runFetch(`flyctl status -a ${name}`).then(() => true).catch(() => false);
-const appCreate = (name) => runAction(`flyctl launch --auto-confirm --name ${name} -r ewr -o ${org} --vm-memory 1024`);
+// We do not deploy at the create stage, since the Docker image isn't ready yet.
+// Assigning --image prevents flyctl from making inferences based on the codebase and provisioning unnecessary postgres/redis instances.
+const appCreate = (name) => runAction(`flyctl launch --no-deploy --auto-confirm --image ${getDockerTag(name)} --name ${name} -r ewr -o ${org}`);
 const volCreate = (name, vol) => runAction(`flyctl volumes create ${vol} -s 1 -r ewr -y -a ${name}`);
 const volList = (name) => runFetch(`flyctl volumes list -a ${name} -j`).then(({stdout}) => JSON.parse(stdout));
-const appDeploy = (name, appRoot) => runAction(`flyctl deploy ${appRoot} --remote-only --region=ewr --vm-memory 1024`,
-  {shell: true, stdio: 'inherit'});
+const appDeploy = async (name) => {
+  try {
+    await runAction("flyctl auth docker")
+    await runAction(`docker image tag grist-core:preview ${getDockerTag(name)}`);
+    await runAction(`docker push ${getDockerTag(name)}`);
+    await runAction(`flyctl deploy --app ${name} --image ${getDockerTag(name)}`);
+  } catch (e) {
+    console.log(`Error occurred when deploying: ${e}`);
+    process.exit(1);
+  }
+};
 
 async function appDestroy(name) {
   await runAction(`flyctl apps destroy ${name} -y`);
 }
 
-async function prepConfig(name, appRoot, volName) {
-  const configPath = `${appRoot}/fly.toml`;
-  const configTemplatePath = `${appRoot}/buildtools/fly-template.toml`;
+async function prepConfig(name, volName) {
+  const configPath = "./fly.toml";
+  const configTemplatePath = "./buildtools/fly-template.toml";
   const template = await fs.readFile(configTemplatePath, {encoding: 'utf8'});
 
   // Calculate the time when we can destroy the app, used by findStaleApps.
diff --git a/buildtools/fly-template.toml b/buildtools/fly-template.toml
index 4eba32ffb7..b1b2a8073d 100644
--- a/buildtools/fly-template.toml
+++ b/buildtools/fly-template.toml
@@ -48,3 +48,8 @@ processes = []
 [mounts]
 source="{VOLUME_NAME}"
 destination="/persist"
+
+[[vm]]
+  memory = '1gb'
+  cpu_kind = 'shared'
+  cpus = 1

From 0bfdaa9c02c761b3c24e83c57960e015c8796614 Mon Sep 17 00:00:00 2001
From: CamilleLegeron <camille@telescoop.fr>
Date: Thu, 4 Jul 2024 14:17:10 +0200
Subject: [PATCH 15/16] Add authorization header in webhooks stored in secrets
 table (#941)

Summary:
Adding authorization header support for webhooks.

Issue:  https://github.com/gristlabs/grist-core/issues/827

---------

Co-authored-by: Florent <florent.git@zeteo.me>
---
 app/client/ui/WebhookPage.ts        | 17 ++++++++++++-----
 app/common/Triggers-ti.ts           |  4 ++++
 app/common/Triggers.ts              |  4 ++++
 app/gen-server/lib/HomeDBManager.ts | 24 +++++++++++++++++++++---
 app/server/lib/DocApi.ts            | 15 ++++++++-------
 app/server/lib/Triggers.ts          | 12 ++++++++++--
 static/locales/en.client.json       |  3 ++-
 test/nbrowser/WebhookPage.ts        | 28 +++++++++++++++++++++++++---
 test/server/lib/DocApi.ts           |  5 +++++
 9 files changed, 91 insertions(+), 21 deletions(-)

diff --git a/app/client/ui/WebhookPage.ts b/app/client/ui/WebhookPage.ts
index 9e263fc0c7..ff93061c01 100644
--- a/app/client/ui/WebhookPage.ts
+++ b/app/client/ui/WebhookPage.ts
@@ -107,6 +107,12 @@ const WEBHOOK_COLUMNS = [
     type: 'Text',
     label: t('Status'),
   },
+  {
+    id: VirtualId(),
+    colId: 'authorization',
+    type: 'Text',
+    label: t('Header Authorization'),
+  },
 ] as const;
 
 /**
@@ -114,10 +120,11 @@ const WEBHOOK_COLUMNS = [
  */
 const WEBHOOK_VIEW_FIELDS: Array<(typeof WEBHOOK_COLUMNS)[number]['colId']> = [
   'name', 'memo',
-  'eventTypes', 'url',
-  'tableId', 'isReadyColumn',
-  'watchedColIdsText', 'webhookId',
-  'enabled', 'status'
+  'eventTypes', 'tableId',
+  'watchedColIdsText', 'isReadyColumn',
+  'url', 'authorization',
+  'webhookId', 'enabled',
+  'status'
 ];
 
 /**
@@ -136,7 +143,7 @@ class WebhookExternalTable implements IExternalTable {
   public name = 'GristHidden_WebhookTable';
   public initialActions = _prepareWebhookInitialActions(this.name);
   public saveableFields = [
-    'tableId', 'watchedColIdsText', 'url', 'eventTypes', 'enabled', 'name', 'memo', 'isReadyColumn',
+    'tableId', 'watchedColIdsText', 'url', 'authorization', 'eventTypes', 'enabled', 'name', 'memo', 'isReadyColumn',
   ];
   public webhooks: ObservableArray<UIWebhookSummary> = observableArray<UIWebhookSummary>([]);
 
diff --git a/app/common/Triggers-ti.ts b/app/common/Triggers-ti.ts
index bb04bbaec6..f93d12ae4f 100644
--- a/app/common/Triggers-ti.ts
+++ b/app/common/Triggers-ti.ts
@@ -14,6 +14,7 @@ export const Webhook = t.iface([], {
 
 export const WebhookFields = t.iface([], {
   "url": "string",
+  "authorization": t.opt("string"),
   "eventTypes": t.array(t.union(t.lit("add"), t.lit("update"))),
   "tableId": "string",
   "watchedColIds": t.opt(t.array("string")),
@@ -29,6 +30,7 @@ export const WebhookStatus = t.union(t.lit('idle'), t.lit('sending'), t.lit('ret
 
 export const WebhookSubscribe = t.iface([], {
   "url": "string",
+  "authorization": t.opt("string"),
   "eventTypes": t.array(t.union(t.lit("add"), t.lit("update"))),
   "watchedColIds": t.opt(t.array("string")),
   "enabled": t.opt("boolean"),
@@ -45,6 +47,7 @@ export const WebhookSummary = t.iface([], {
   "id": "string",
   "fields": t.iface([], {
     "url": "string",
+    "authorization": t.opt("string"),
     "unsubscribeKey": "string",
     "eventTypes": t.array("string"),
     "isReadyColumn": t.union("string", "null"),
@@ -64,6 +67,7 @@ export const WebhookUpdate = t.iface([], {
 
 export const WebhookPatch = t.iface([], {
   "url": t.opt("string"),
+  "authorization": t.opt("string"),
   "eventTypes": t.opt(t.array(t.union(t.lit("add"), t.lit("update")))),
   "tableId": t.opt("string"),
   "watchedColIds": t.opt(t.array("string")),
diff --git a/app/common/Triggers.ts b/app/common/Triggers.ts
index d3b492d610..a53dd1feed 100644
--- a/app/common/Triggers.ts
+++ b/app/common/Triggers.ts
@@ -8,6 +8,7 @@ export interface Webhook {
 
 export interface WebhookFields {
   url: string;
+  authorization?: string;
   eventTypes: Array<"add"|"update">;
   tableId: string;
   watchedColIds?: string[];
@@ -26,6 +27,7 @@ export type WebhookStatus = 'idle'|'sending'|'retrying'|'postponed'|'error'|'inv
 // tableId from the url) but generics are not yet supported by ts-interface-builder
 export interface WebhookSubscribe {
   url: string;
+  authorization?: string;
   eventTypes: Array<"add"|"update">;
   watchedColIds?: string[];
   enabled?: boolean;
@@ -42,6 +44,7 @@ export interface WebhookSummary {
   id: string;
   fields: {
     url: string;
+    authorization?: string;
     unsubscribeKey: string;
     eventTypes: string[];
     isReadyColumn: string|null;
@@ -64,6 +67,7 @@ export interface WebhookUpdate {
 // ts-interface-builder
 export interface WebhookPatch {
   url?: string;
+  authorization?: string;
   eventTypes?: Array<"add"|"update">;
   tableId?: string;
   watchedColIds?: string[];
diff --git a/app/gen-server/lib/HomeDBManager.ts b/app/gen-server/lib/HomeDBManager.ts
index 06409d1c8b..a4f897211f 100644
--- a/app/gen-server/lib/HomeDBManager.ts
+++ b/app/gen-server/lib/HomeDBManager.ts
@@ -1608,7 +1608,7 @@ export class HomeDBManager extends EventEmitter {
       .where("id = :id AND doc_id = :docId", {id, docId})
       .execute();
     if (res.affected !== 1) {
-      throw new ApiError('secret with given id not found', 404);
+      throw new ApiError('secret with given id not found or nothing was updated', 404);
     }
   }
 
@@ -1623,14 +1623,32 @@ export class HomeDBManager extends EventEmitter {
 
   // Update the webhook url in the webhook's corresponding secret (note: the webhook identifier is
   // its secret identifier).
-  public async updateWebhookUrl(id: string, docId: string, url: string, outerManager?: EntityManager) {
+  public async updateWebhookUrlAndAuth(
+    props: {
+      id: string,
+      docId: string,
+      url: string | undefined,
+      auth: string | undefined,
+      outerManager?: EntityManager}
+    ) {
+    const {id, docId, url, auth, outerManager} = props;
     return await this._runInTransaction(outerManager, async manager => {
+      if (url === undefined && auth === undefined) {
+        throw new ApiError('None of the Webhook url and auth are defined', 404);
+      }
       const value = await this.getSecret(id, docId, manager);
       if (!value) {
         throw new ApiError('Webhook with given id not found', 404);
       }
       const webhookSecret = JSON.parse(value);
-      webhookSecret.url = url;
+      // As we want to patch the webhookSecret object, only set the url and the authorization when they are defined.
+      // When the user wants to empty the value, we are expected to receive empty strings.
+      if (url !== undefined) {
+        webhookSecret.url = url;
+      }
+      if (auth !== undefined) {
+        webhookSecret.authorization = auth;
+      }
       await this.updateSecret(id, docId, JSON.stringify(webhookSecret), manager);
     });
   }
diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts
index e5f66df439..297cafff76 100644
--- a/app/server/lib/DocApi.ts
+++ b/app/server/lib/DocApi.ts
@@ -324,7 +324,7 @@ export class DocWorkerApi {
     );
 
     const registerWebhook = async (activeDoc: ActiveDoc, req: RequestWithLogin, webhook: WebhookFields) => {
-      const {fields, url} = await getWebhookSettings(activeDoc, req, null, webhook);
+      const {fields, url, authorization} = await getWebhookSettings(activeDoc, req, null, webhook);
       if (!fields.eventTypes?.length) {
         throw new ApiError(`eventTypes must be a non-empty array`, 400);
       }
@@ -336,7 +336,7 @@ export class DocWorkerApi {
       }
 
       const unsubscribeKey = uuidv4();
-      const webhookSecret: WebHookSecret = {unsubscribeKey, url};
+      const webhookSecret: WebHookSecret = {unsubscribeKey, url, authorization};
       const secretValue = JSON.stringify(webhookSecret);
       const webhookId = (await this._dbManager.addSecret(secretValue, activeDoc.docName)).id;
 
@@ -392,7 +392,7 @@ export class DocWorkerApi {
       const tablesTable = activeDoc.docData!.getMetaTable("_grist_Tables");
       const trigger = webhookId ? activeDoc.triggers.getWebhookTriggerRecord(webhookId) : undefined;
       let currentTableId = trigger ? tablesTable.getValue(trigger.tableRef, 'tableId')! : undefined;
-      const {url, eventTypes, watchedColIds, isReadyColumn, name} = webhook;
+      const {url, authorization, eventTypes, watchedColIds, isReadyColumn, name} = webhook;
       const tableId = await getRealTableId(req.params.tableId || webhook.tableId, {metaTables});
 
       const fields: Partial<SchemaTypes['_grist_Triggers']> = {};
@@ -454,6 +454,7 @@ export class DocWorkerApi {
       return {
         fields,
         url,
+        authorization,
       };
     }
 
@@ -926,16 +927,16 @@ export class DocWorkerApi {
 
         const docId = activeDoc.docName;
         const webhookId = req.params.webhookId;
-        const {fields, url} = await getWebhookSettings(activeDoc, req, webhookId, req.body);
+        const {fields, url, authorization} = await getWebhookSettings(activeDoc, req, webhookId, req.body);
         if (fields.enabled === false) {
           await activeDoc.triggers.clearSingleWebhookQueue(webhookId);
         }
 
         const triggerRowId = activeDoc.triggers.getWebhookTriggerRecord(webhookId).id;
 
-        // update url in homedb
-        if (url) {
-          await this._dbManager.updateWebhookUrl(webhookId, docId, url);
+        // update url and authorization header in homedb
+        if (url || authorization) {
+          await this._dbManager.updateWebhookUrlAndAuth({id: webhookId, docId, url, auth: authorization});
           activeDoc.triggers.webhookDeleted(webhookId); // clear cache
         }
 
diff --git a/app/server/lib/Triggers.ts b/app/server/lib/Triggers.ts
index c90ee54812..e9d51484b5 100644
--- a/app/server/lib/Triggers.ts
+++ b/app/server/lib/Triggers.ts
@@ -72,6 +72,7 @@ type Trigger = MetaRowRecord<"_grist_Triggers">;
 export interface WebHookSecret {
   url: string;
   unsubscribeKey: string;
+  authorization?: string;
 }
 
 // Work to do after fetching values from the document
@@ -259,6 +260,7 @@ export class DocTriggers {
     const getTableId = docData.getMetaTable("_grist_Tables").getRowPropFunc("tableId");
     const getColId = docData.getMetaTable("_grist_Tables_column").getRowPropFunc("colId");
     const getUrl = async (id: string) => (await this._getWebHook(id))?.url ?? '';
+    const getAuthorization = async (id: string) => (await this._getWebHook(id))?.authorization ?? '';
     const getUnsubscribeKey = async (id: string) => (await this._getWebHook(id))?.unsubscribeKey ?? '';
     const resultTable: WebhookSummary[] = [];
 
@@ -271,6 +273,7 @@ export class DocTriggers {
       for (const act of webhookActions) {
         // Url, probably should be hidden for non-owners (but currently this API is owners only).
         const url = await getUrl(act.id);
+        const authorization = await getAuthorization(act.id);
         // Same story, should be hidden.
         const unsubscribeKey = await getUnsubscribeKey(act.id);
         if (!url || !unsubscribeKey) {
@@ -285,6 +288,7 @@ export class DocTriggers {
           fields: {
             // Url, probably should be hidden for non-owners (but currently this API is owners only).
             url,
+            authorization,
             unsubscribeKey,
             // Other fields used to register this webhook.
             eventTypes: decodeObject(t.eventTypes) as string[],
@@ -683,6 +687,7 @@ export class DocTriggers {
       const batch = _.takeWhile(this._webHookEventQueue.slice(0, 100), {id});
       const body = JSON.stringify(batch.map(e => e.payload));
       const url = await this._getWebHookUrl(id);
+      const authorization = (await this._getWebHook(id))?.authorization || "";
       if (this._loopAbort.signal.aborted) {
         continue;
       }
@@ -698,7 +703,8 @@ export class DocTriggers {
         this._activeDoc.logTelemetryEvent(null, 'sendingWebhooks', {
           limited: {numEvents: meta.numEvents},
         });
-        success = await this._sendWebhookWithRetries(id, url, body, batch.length, this._loopAbort.signal);
+        success = await this._sendWebhookWithRetries(
+          id, url, authorization, body, batch.length, this._loopAbort.signal);
         if (this._loopAbort.signal.aborted) {
           continue;
         }
@@ -770,7 +776,8 @@ export class DocTriggers {
     return this._drainingQueue ? Math.min(5, TRIGGER_MAX_ATTEMPTS) : TRIGGER_MAX_ATTEMPTS;
   }
 
-  private async _sendWebhookWithRetries(id: string, url: string, body: string, size: number, signal: AbortSignal) {
+  private async _sendWebhookWithRetries(
+    id: string, url: string, authorization: string, body: string, size: number, signal: AbortSignal) {
     const maxWait = 64;
     let wait = 1;
     for (let attempt = 0; attempt < this._maxWebhookAttempts; attempt++) {
@@ -786,6 +793,7 @@ export class DocTriggers {
           body,
           headers: {
             'Content-Type': 'application/json',
+            ...(authorization ? {'Authorization': authorization} : {}),
           },
           signal,
           agent: proxyAgent(new URL(url)),
diff --git a/static/locales/en.client.json b/static/locales/en.client.json
index 420e580e7c..76790bb1b3 100644
--- a/static/locales/en.client.json
+++ b/static/locales/en.client.json
@@ -1241,7 +1241,8 @@
         "URL": "URL",
         "Webhook Id": "Webhook Id",
         "Table": "Table",
-        "Filter for changes in these columns (semicolon-separated ids)": "Filter for changes in these columns (semicolon-separated ids)"
+        "Filter for changes in these columns (semicolon-separated ids)": "Filter for changes in these columns (semicolon-separated ids)",
+        "Header Authorization": "Header Authorization"
     },
     "FormulaAssistant": {
         "Ask the bot.": "Ask the bot.",
diff --git a/test/nbrowser/WebhookPage.ts b/test/nbrowser/WebhookPage.ts
index 5db3e97209..d53f4ebe28 100644
--- a/test/nbrowser/WebhookPage.ts
+++ b/test/nbrowser/WebhookPage.ts
@@ -52,10 +52,11 @@ describe('WebhookPage', function () {
       'Name',
       'Memo',
       'Event Types',
-      'URL',
       'Table',
-      'Ready Column',
       'Filter for changes in these columns (semicolon-separated ids)',
+      'Ready Column',
+      'URL',
+      'Header Authorization',
       'Webhook Id',
       'Enabled',
       'Status',
@@ -81,7 +82,7 @@ describe('WebhookPage', function () {
     await gu.waitToPass(async () => {
       assert.equal(await getField(1, 'Webhook Id'), id);
     });
-    // Now other fields like name, memo and watchColIds are persisted.
+    // Now other fields like name, memo, watchColIds, and Header Auth are persisted.
     await setField(1, 'Name', 'Test Webhook');
     await setField(1, 'Memo', 'Test Memo');
     await setField(1, 'Filter for changes in these columns (semicolon-separated ids)', 'A; B');
@@ -115,6 +116,27 @@ describe('WebhookPage', function () {
     assert.lengthOf((await docApi.getRows('Table2')).A, 0);
   });
 
+  it('can create webhook with persistant header authorization', async function () {
+    // The webhook won't work because the header auth doesn't match the api key of the current test user.
+    await openWebhookPage();
+    await setField(1, 'Event Types', 'add\nupdate\n');
+    await setField(1, 'URL', `http://${host}/api/docs/${doc.id}/tables/Table2/records?flat=1`);
+    await setField(1, 'Table', 'Table1');
+    await gu.waitForServer();
+    await driver.navigate().refresh();
+    await waitForWebhookPage();
+    await setField(1, 'Header Authorization', 'Bearer 1234');
+    await gu.waitForServer();
+    await driver.navigate().refresh();
+    await waitForWebhookPage();
+    await gu.waitToPass(async () => {
+      assert.equal(await getField(1, 'Header Authorization'), 'Bearer 1234');
+    });
+    await gu.getDetailCell({col:'Header Authorization', rowNum: 1}).click();
+    await gu.enterCell(Key.DELETE, Key.ENTER);
+    await gu.waitForServer();
+  });
+
   it('can create two webhooks', async function () {
     await openWebhookPage();
     await setField(1, 'Event Types', 'add\nupdate\n');
diff --git a/test/server/lib/DocApi.ts b/test/server/lib/DocApi.ts
index 86b515d3b8..a12f17565a 100644
--- a/test/server/lib/DocApi.ts
+++ b/test/server/lib/DocApi.ts
@@ -4625,6 +4625,7 @@ function testDocApi() {
             id: first.webhookId,
             fields: {
               url: `${serving.url}/200`,
+              authorization: '',
               unsubscribeKey: first.unsubscribeKey,
               eventTypes: ['add', 'update'],
               enabled: true,
@@ -4643,6 +4644,7 @@ function testDocApi() {
             id: second.webhookId,
             fields: {
               url: `${serving.url}/404`,
+              authorization: '',
               unsubscribeKey: second.unsubscribeKey,
               eventTypes: ['add', 'update'],
               enabled: true,
@@ -5010,6 +5012,7 @@ function testDocApi() {
 
             const expectedFields = {
               url: `${serving.url}/foo`,
+              authorization: '',
               eventTypes: ['add'],
               isReadyColumn: 'B',
               tableId: 'Table1',
@@ -5079,6 +5082,8 @@ function testDocApi() {
 
           await check({isReadyColumn: null}, 200);
           await check({isReadyColumn: "bar"}, 404, `Column not found "bar"`);
+
+          await check({authorization: 'Bearer fake-token'}, 200);
         });
 
       });

From 786ba6b31e9af4ec2d87156d299d2f9042e18973 Mon Sep 17 00:00:00 2001
From: Florent <florent.git@zeteo.me>
Date: Fri, 5 Jul 2024 16:02:39 +0200
Subject: [PATCH 16/16] Move HomeDBManager to gen-server/lib/homedb (#1076)

---
 app/gen-server/ApiServer.ts                      | 2 +-
 app/gen-server/lib/Activations.ts                | 2 +-
 app/gen-server/lib/DocApiForwarder.ts            | 2 +-
 app/gen-server/lib/Doom.ts                       | 2 +-
 app/gen-server/lib/Housekeeper.ts                | 2 +-
 app/gen-server/lib/Usage.ts                      | 2 +-
 app/gen-server/lib/{ => homedb}/HomeDBManager.ts | 0
 app/gen-server/lib/homedb/UsersManager.ts        | 2 +-
 app/server/companion.ts                          | 2 +-
 app/server/lib/AppEndpoint.ts                    | 2 +-
 app/server/lib/Authorizer.ts                     | 2 +-
 app/server/lib/Client.ts                         | 2 +-
 app/server/lib/DocApi.ts                         | 2 +-
 app/server/lib/DocManager.ts                     | 2 +-
 app/server/lib/DocWorker.ts                      | 2 +-
 app/server/lib/FlexServer.ts                     | 2 +-
 app/server/lib/GranularAccess.ts                 | 2 +-
 app/server/lib/GristServer.ts                    | 2 +-
 app/server/lib/HostedMetadataManager.ts          | 2 +-
 app/server/lib/HostedStorageManager.ts           | 2 +-
 app/server/lib/ICreate.ts                        | 2 +-
 app/server/lib/InstallAdmin.ts                   | 2 +-
 app/server/lib/Telemetry.ts                      | 2 +-
 app/server/lib/TestLogin.ts                      | 2 +-
 app/server/lib/extractOrg.ts                     | 2 +-
 app/server/lib/requestUtils.ts                   | 2 +-
 app/server/lib/sendAppPage.ts                    | 2 +-
 stubs/app/server/server.ts                       | 2 +-
 test/gen-server/ApiServer.ts                     | 2 +-
 test/gen-server/ApiServerAccess.ts               | 2 +-
 test/gen-server/ApiServerBugs.ts                 | 2 +-
 test/gen-server/AuthCaching.ts                   | 2 +-
 test/gen-server/apiUtils.ts                      | 2 +-
 test/gen-server/migrations.ts                    | 2 +-
 test/gen-server/seed.ts                          | 2 +-
 test/gen-server/testUtils.ts                     | 2 +-
 test/nbrowser/homeUtil.ts                        | 2 +-
 test/nbrowser/testServer.ts                      | 2 +-
 test/server/lib/Authorizer.ts                    | 2 +-
 test/server/lib/HostedStorageManager.ts          | 2 +-
 test/testUtils.ts                                | 2 +-
 41 files changed, 40 insertions(+), 40 deletions(-)
 rename app/gen-server/lib/{ => homedb}/HomeDBManager.ts (100%)

diff --git a/app/gen-server/ApiServer.ts b/app/gen-server/ApiServer.ts
index 915012ad14..b927d62f4a 100644
--- a/app/gen-server/ApiServer.ts
+++ b/app/gen-server/ApiServer.ts
@@ -9,7 +9,7 @@ import {FullUser} from 'app/common/LoginSessionAPI';
 import {BasicRole} from 'app/common/roles';
 import {OrganizationProperties, PermissionDelta} from 'app/common/UserAPI';
 import {User} from 'app/gen-server/entity/User';
-import {BillingOptions, HomeDBManager, QueryResult, Scope} from 'app/gen-server/lib/HomeDBManager';
+import {BillingOptions, HomeDBManager, QueryResult, Scope} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {getAuthorizedUserId, getUserId, getUserProfiles, RequestWithLogin} from 'app/server/lib/Authorizer';
 import {getSessionUser, linkOrgWithEmail} from 'app/server/lib/BrowserSession';
 import {expressWrap} from 'app/server/lib/expressWrap';
diff --git a/app/gen-server/lib/Activations.ts b/app/gen-server/lib/Activations.ts
index b089efbe40..2648c98b73 100644
--- a/app/gen-server/lib/Activations.ts
+++ b/app/gen-server/lib/Activations.ts
@@ -1,6 +1,6 @@
 import { makeId } from 'app/server/lib/idUtils';
 import { Activation } from 'app/gen-server/entity/Activation';
-import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
+import { HomeDBManager } from 'app/gen-server/lib/homedb/HomeDBManager';
 
 /**
  * Manage activations. Not much to do currently, there is at most one
diff --git a/app/gen-server/lib/DocApiForwarder.ts b/app/gen-server/lib/DocApiForwarder.ts
index 3545a63a28..ed58e03bf2 100644
--- a/app/gen-server/lib/DocApiForwarder.ts
+++ b/app/gen-server/lib/DocApiForwarder.ts
@@ -5,7 +5,7 @@ import {AbortController} from 'node-abort-controller';
 import { ApiError } from 'app/common/ApiError';
 import { SHARE_KEY_PREFIX } from 'app/common/gristUrls';
 import { removeTrailingSlash } from 'app/common/gutil';
-import { HomeDBManager } from "app/gen-server/lib/HomeDBManager";
+import { HomeDBManager } from "app/gen-server/lib/homedb/HomeDBManager";
 import { assertAccess, getOrSetDocAuth, getTransitiveHeaders, RequestWithLogin } from 'app/server/lib/Authorizer';
 import { IDocWorkerMap } from "app/server/lib/DocWorkerMap";
 import { expressWrap } from "app/server/lib/expressWrap";
diff --git a/app/gen-server/lib/Doom.ts b/app/gen-server/lib/Doom.ts
index cbce2587dd..1d6bc0d648 100644
--- a/app/gen-server/lib/Doom.ts
+++ b/app/gen-server/lib/Doom.ts
@@ -1,7 +1,7 @@
 import { ApiError } from 'app/common/ApiError';
 import { FullUser } from 'app/common/UserAPI';
 import { Organization } from 'app/gen-server/entity/Organization';
-import { HomeDBManager, Scope } from 'app/gen-server/lib/HomeDBManager';
+import { HomeDBManager, Scope } from 'app/gen-server/lib/homedb/HomeDBManager';
 import { INotifier } from 'app/server/lib/INotifier';
 import { scrubUserFromOrg } from 'app/gen-server/lib/scrubUserFromOrg';
 import { GristLoginSystem } from 'app/server/lib/GristServer';
diff --git a/app/gen-server/lib/Housekeeper.ts b/app/gen-server/lib/Housekeeper.ts
index c40379fe15..c3012bec6b 100644
--- a/app/gen-server/lib/Housekeeper.ts
+++ b/app/gen-server/lib/Housekeeper.ts
@@ -7,7 +7,7 @@ import { Document } from 'app/gen-server/entity/Document';
 import { Organization } from 'app/gen-server/entity/Organization';
 import { Product } from 'app/gen-server/entity/Product';
 import { Workspace } from 'app/gen-server/entity/Workspace';
-import { HomeDBManager, Scope } from 'app/gen-server/lib/HomeDBManager';
+import { HomeDBManager, Scope } from 'app/gen-server/lib/homedb/HomeDBManager';
 import { fromNow } from 'app/gen-server/sqlUtils';
 import { getAuthorizedUserId } from 'app/server/lib/Authorizer';
 import { expressWrap } from 'app/server/lib/expressWrap';
diff --git a/app/gen-server/lib/Usage.ts b/app/gen-server/lib/Usage.ts
index 865fa2b12e..9082bdcca9 100644
--- a/app/gen-server/lib/Usage.ts
+++ b/app/gen-server/lib/Usage.ts
@@ -1,7 +1,7 @@
 import {Document} from 'app/gen-server/entity/Document';
 import {Organization} from 'app/gen-server/entity/Organization';
 import {User} from 'app/gen-server/entity/User';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import log from 'app/server/lib/log';
 
 // Frequency of logging usage information.  Not something we need
diff --git a/app/gen-server/lib/HomeDBManager.ts b/app/gen-server/lib/homedb/HomeDBManager.ts
similarity index 100%
rename from app/gen-server/lib/HomeDBManager.ts
rename to app/gen-server/lib/homedb/HomeDBManager.ts
diff --git a/app/gen-server/lib/homedb/UsersManager.ts b/app/gen-server/lib/homedb/UsersManager.ts
index 168665f354..e070273ac5 100644
--- a/app/gen-server/lib/homedb/UsersManager.ts
+++ b/app/gen-server/lib/homedb/UsersManager.ts
@@ -17,7 +17,7 @@ import { Group } from 'app/gen-server/entity/Group';
 import { Login } from 'app/gen-server/entity/Login';
 import { User } from 'app/gen-server/entity/User';
 import { appSettings } from 'app/server/lib/AppSettings';
-import { HomeDBManager, PermissionDeltaAnalysis, Scope } from 'app/gen-server/lib/HomeDBManager';
+import { HomeDBManager, PermissionDeltaAnalysis, Scope } from 'app/gen-server/lib/homedb/HomeDBManager';
 import {
   AvailableUsers, GetUserOptions, NonGuestGroup, QueryResult, Resource, RunInTransaction, UserProfileChange
 } from 'app/gen-server/lib/homedb/Interfaces';
diff --git a/app/server/companion.ts b/app/server/companion.ts
index f28475c809..bad8092ce5 100644
--- a/app/server/companion.ts
+++ b/app/server/companion.ts
@@ -1,7 +1,7 @@
 import { Level, TelemetryContracts } from 'app/common/Telemetry';
 import { version } from 'app/common/version';
 import { synchronizeProducts } from 'app/gen-server/entity/Product';
-import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
+import { HomeDBManager } from 'app/gen-server/lib/homedb/HomeDBManager';
 import { applyPatch } from 'app/gen-server/lib/TypeORMPatches';
 import { getMigrations, getOrCreateConnection, getTypeORMSettings,
          undoLastMigration, updateDb } from 'app/server/lib/dbUtils';
diff --git a/app/server/lib/AppEndpoint.ts b/app/server/lib/AppEndpoint.ts
index b327a4e1fb..8147bfcf1f 100644
--- a/app/server/lib/AppEndpoint.ts
+++ b/app/server/lib/AppEndpoint.ts
@@ -11,7 +11,7 @@ import {LocalPlugin} from "app/common/plugin";
 import {TELEMETRY_TEMPLATE_SIGNUP_COOKIE_NAME} from 'app/common/Telemetry';
 import {Document as APIDocument, PublicDocWorkerUrlInfo} from 'app/common/UserAPI';
 import {Document} from "app/gen-server/entity/Document";
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {assertAccess, getTransitiveHeaders, getUserId, isAnonymousUser,
         RequestWithLogin} from 'app/server/lib/Authorizer';
 import {DocStatus, IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
diff --git a/app/server/lib/Authorizer.ts b/app/server/lib/Authorizer.ts
index 6f9a674125..0386a6d304 100644
--- a/app/server/lib/Authorizer.ts
+++ b/app/server/lib/Authorizer.ts
@@ -7,7 +7,7 @@ import {canEdit, canView, getWeakestRole, Role} from 'app/common/roles';
 import {UserOptions} from 'app/common/UserAPI';
 import {Document} from 'app/gen-server/entity/Document';
 import {User} from 'app/gen-server/entity/User';
-import {DocAuthKey, DocAuthResult, HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {DocAuthKey, DocAuthResult, HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {forceSessionChange, getSessionProfiles, getSessionUser, getSignInStatus, linkOrgWithEmail, SessionObj,
         SessionUserObj, SignInStatus} from 'app/server/lib/BrowserSession';
 import {RequestWithOrg} from 'app/server/lib/extractOrg';
diff --git a/app/server/lib/Client.ts b/app/server/lib/Client.ts
index 0364ca365a..ce2a9b0bc6 100644
--- a/app/server/lib/Client.ts
+++ b/app/server/lib/Client.ts
@@ -8,7 +8,7 @@ import {TelemetryMetadata} from 'app/common/Telemetry';
 import {ANONYMOUS_USER_EMAIL} from 'app/common/UserAPI';
 import {normalizeEmail} from 'app/common/emails';
 import {User} from 'app/gen-server/entity/User';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {Authorizer} from 'app/server/lib/Authorizer';
 import {ScopedSession} from 'app/server/lib/BrowserSession';
diff --git a/app/server/lib/DocApi.ts b/app/server/lib/DocApi.ts
index 297cafff76..f7d0a94647 100644
--- a/app/server/lib/DocApi.ts
+++ b/app/server/lib/DocApi.ts
@@ -30,7 +30,7 @@ import {TelemetryMetadataByLevel} from "app/common/Telemetry";
 import {WebhookFields} from "app/common/Triggers";
 import TriggersTI from 'app/common/Triggers-ti';
 import {DocReplacementOptions, DocState, DocStateComparison, DocStates, NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
-import {HomeDBManager, makeDocAuthResult} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager, makeDocAuthResult} from 'app/gen-server/lib/homedb/HomeDBManager';
 import * as Types from "app/plugin/DocApiTypes";
 import DocApiTypesTI from "app/plugin/DocApiTypes-ti";
 import {GristObjCode} from "app/plugin/GristData";
diff --git a/app/server/lib/DocManager.ts b/app/server/lib/DocManager.ts
index 437559f857..f342ab9a45 100644
--- a/app/server/lib/DocManager.ts
+++ b/app/server/lib/DocManager.ts
@@ -15,7 +15,7 @@ import {Invite} from 'app/common/sharing';
 import {tbind} from 'app/common/tbind';
 import {TelemetryMetadataByLevel} from 'app/common/Telemetry';
 import {NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {assertAccess, Authorizer, DocAuthorizer, DummyAuthorizer, isSingleUserMode,
         RequestWithLogin} from 'app/server/lib/Authorizer';
 import {Client} from 'app/server/lib/Client';
diff --git a/app/server/lib/DocWorker.ts b/app/server/lib/DocWorker.ts
index 7bb8d8e610..b8f0d608e1 100644
--- a/app/server/lib/DocWorker.ts
+++ b/app/server/lib/DocWorker.ts
@@ -3,7 +3,7 @@
  * In hosted environment, this comprises the functionality of the DocWorker instance type.
  */
 import {isAffirmative} from 'app/common/gutil';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {ActionHistoryImpl} from 'app/server/lib/ActionHistoryImpl';
 import {assertAccess, getOrSetDocAuth, RequestWithLogin} from 'app/server/lib/Authorizer';
 import {Client} from 'app/server/lib/Client';
diff --git a/app/server/lib/FlexServer.ts b/app/server/lib/FlexServer.ts
index 812e1d2101..a3568ecb77 100644
--- a/app/server/lib/FlexServer.ts
+++ b/app/server/lib/FlexServer.ts
@@ -20,7 +20,7 @@ import {Activations} from 'app/gen-server/lib/Activations';
 import {DocApiForwarder} from 'app/gen-server/lib/DocApiForwarder';
 import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap';
 import {Doom} from 'app/gen-server/lib/Doom';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {Housekeeper} from 'app/gen-server/lib/Housekeeper';
 import {Usage} from 'app/gen-server/lib/Usage';
 import {AccessTokens, IAccessTokens} from 'app/server/lib/AccessTokens';
diff --git a/app/server/lib/GranularAccess.ts b/app/server/lib/GranularAccess.ts
index 69fcf2ee70..0862686985 100644
--- a/app/server/lib/GranularAccess.ts
+++ b/app/server/lib/GranularAccess.ts
@@ -35,7 +35,7 @@ import { EmptyRecordView, InfoView, RecordView } from 'app/common/RecordView';
 import { canEdit, canView, isValidRole, Role } from 'app/common/roles';
 import { User } from 'app/common/User';
 import { FullUser, UserAccessData } from 'app/common/UserAPI';
-import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
+import { HomeDBManager } from 'app/gen-server/lib/homedb/HomeDBManager';
 import { GristObjCode } from 'app/plugin/GristData';
 import { DocClients } from 'app/server/lib/DocClients';
 import { getDocSessionAccess, getDocSessionAltSessionId, getDocSessionShare,
diff --git a/app/server/lib/GristServer.ts b/app/server/lib/GristServer.ts
index 1e92639544..265535d7bd 100644
--- a/app/server/lib/GristServer.ts
+++ b/app/server/lib/GristServer.ts
@@ -8,7 +8,7 @@ import { Organization } from 'app/gen-server/entity/Organization';
 import { User } from 'app/gen-server/entity/User';
 import { Workspace } from 'app/gen-server/entity/Workspace';
 import { Activations } from 'app/gen-server/lib/Activations';
-import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
+import { HomeDBManager } from 'app/gen-server/lib/homedb/HomeDBManager';
 import { IAccessTokens } from 'app/server/lib/AccessTokens';
 import { RequestWithLogin } from 'app/server/lib/Authorizer';
 import { Comm } from 'app/server/lib/Comm';
diff --git a/app/server/lib/HostedMetadataManager.ts b/app/server/lib/HostedMetadataManager.ts
index bce0a0552c..f49a545bd0 100644
--- a/app/server/lib/HostedMetadataManager.ts
+++ b/app/server/lib/HostedMetadataManager.ts
@@ -1,4 +1,4 @@
-import {DocumentMetadata, HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {DocumentMetadata, HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import log from 'app/server/lib/log';
 
 /**
diff --git a/app/server/lib/HostedStorageManager.ts b/app/server/lib/HostedStorageManager.ts
index 9be39f8547..fa73beb630 100644
--- a/app/server/lib/HostedStorageManager.ts
+++ b/app/server/lib/HostedStorageManager.ts
@@ -8,7 +8,7 @@ import {DocumentUsage} from 'app/common/DocUsage';
 import {buildUrlId, parseUrlId} from 'app/common/gristUrls';
 import {KeyedOps} from 'app/common/KeyedOps';
 import {DocReplacementOptions, NEW_DOCUMENT_CODE} from 'app/common/UserAPI';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {checksumFile} from 'app/server/lib/checksumFile';
 import {DocSnapshotInventory, DocSnapshotPruner} from 'app/server/lib/DocSnapshots';
 import {IDocWorkerMap} from 'app/server/lib/DocWorkerMap';
diff --git a/app/server/lib/ICreate.ts b/app/server/lib/ICreate.ts
index 3ca48d0cd9..4b4d66eeea 100644
--- a/app/server/lib/ICreate.ts
+++ b/app/server/lib/ICreate.ts
@@ -1,7 +1,7 @@
 import {GristDeploymentType} from 'app/common/gristUrls';
 import {getThemeBackgroundSnippet} from 'app/common/Themes';
 import {Document} from 'app/gen-server/entity/Document';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {ExternalStorage} from 'app/server/lib/ExternalStorage';
 import {createDummyTelemetry, GristServer} from 'app/server/lib/GristServer';
 import {IBilling} from 'app/server/lib/IBilling';
diff --git a/app/server/lib/InstallAdmin.ts b/app/server/lib/InstallAdmin.ts
index 0a00bfa18e..f7fdba0d03 100644
--- a/app/server/lib/InstallAdmin.ts
+++ b/app/server/lib/InstallAdmin.ts
@@ -1,5 +1,5 @@
 import {ApiError} from 'app/common/ApiError';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {appSettings} from 'app/server/lib/AppSettings';
 import {getUser, RequestWithLogin} from 'app/server/lib/Authorizer';
 import {User} from 'app/gen-server/entity/User';
diff --git a/app/server/lib/Telemetry.ts b/app/server/lib/Telemetry.ts
index 6d34160046..a08381c145 100644
--- a/app/server/lib/Telemetry.ts
+++ b/app/server/lib/Telemetry.ts
@@ -17,7 +17,7 @@ import {
 import {TelemetryPrefsWithSources} from 'app/common/InstallAPI';
 import {Activation} from 'app/gen-server/entity/Activation';
 import {Activations} from 'app/gen-server/lib/Activations';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {RequestWithLogin} from 'app/server/lib/Authorizer';
 import {getDocSessionUser, OptDocSession} from 'app/server/lib/DocSession';
 import {expressWrap} from 'app/server/lib/expressWrap';
diff --git a/app/server/lib/TestLogin.ts b/app/server/lib/TestLogin.ts
index 2cd0d5ac19..12ef87a3c2 100644
--- a/app/server/lib/TestLogin.ts
+++ b/app/server/lib/TestLogin.ts
@@ -1,4 +1,4 @@
-import {SUPPORT_EMAIL} from 'app/gen-server/lib/HomeDBManager';
+import {SUPPORT_EMAIL} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {GristLoginSystem, GristServer} from 'app/server/lib/GristServer';
 import {Request} from 'express';
 
diff --git a/app/server/lib/extractOrg.ts b/app/server/lib/extractOrg.ts
index 787fa1b84f..524aa97e38 100644
--- a/app/server/lib/extractOrg.ts
+++ b/app/server/lib/extractOrg.ts
@@ -3,7 +3,7 @@ import { mapGetOrSet, MapWithTTL } from 'app/common/AsyncCreate';
 import { extractOrgParts, getHostType, getKnownOrg } from 'app/common/gristUrls';
 import { isAffirmative } from 'app/common/gutil';
 import { Organization } from 'app/gen-server/entity/Organization';
-import { HomeDBManager } from 'app/gen-server/lib/HomeDBManager';
+import { HomeDBManager } from 'app/gen-server/lib/homedb/HomeDBManager';
 import { GristServer } from 'app/server/lib/GristServer';
 import { getOriginUrl } from 'app/server/lib/requestUtils';
 import { NextFunction, Request, RequestHandler, Response } from 'express';
diff --git a/app/server/lib/requestUtils.ts b/app/server/lib/requestUtils.ts
index de0326d478..7f69396692 100644
--- a/app/server/lib/requestUtils.ts
+++ b/app/server/lib/requestUtils.ts
@@ -1,7 +1,7 @@
 import {ApiError} from 'app/common/ApiError';
 import { DEFAULT_HOME_SUBDOMAIN, isOrgInPathOnly, parseSubdomain, sanitizePathTail } from 'app/common/gristUrls';
 import * as gutil from 'app/common/gutil';
-import {DocScope, QueryResult, Scope} from 'app/gen-server/lib/HomeDBManager';
+import {DocScope, QueryResult, Scope} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {getUserId, RequestWithLogin} from 'app/server/lib/Authorizer';
 import {RequestWithOrg} from 'app/server/lib/extractOrg';
 import {RequestWithGrist} from 'app/server/lib/GristServer';
diff --git a/app/server/lib/sendAppPage.ts b/app/server/lib/sendAppPage.ts
index ec53f2be69..3fce3c382d 100644
--- a/app/server/lib/sendAppPage.ts
+++ b/app/server/lib/sendAppPage.ts
@@ -12,7 +12,7 @@ import {isAffirmative} from 'app/common/gutil';
 import {getTagManagerSnippet} from 'app/common/tagManager';
 import {Document} from 'app/common/UserAPI';
 import {AttachedCustomWidgets, IAttachedCustomWidget} from "app/common/widgetTypes";
-import {SUPPORT_EMAIL} from 'app/gen-server/lib/HomeDBManager';
+import {SUPPORT_EMAIL} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {isAnonymousUser, isSingleUserMode, RequestWithLogin} from 'app/server/lib/Authorizer';
 import {RequestWithOrg} from 'app/server/lib/extractOrg';
 import {GristServer} from 'app/server/lib/GristServer';
diff --git a/stubs/app/server/server.ts b/stubs/app/server/server.ts
index e8761cef15..6fbffdf52b 100644
--- a/stubs/app/server/server.ts
+++ b/stubs/app/server/server.ts
@@ -6,7 +6,7 @@
 
 import {commonUrls} from 'app/common/gristUrls';
 import {isAffirmative} from 'app/common/gutil';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {fixSiteProducts} from 'app/gen-server/lib/Housekeeper';
 
 const debugging = isAffirmative(process.env.DEBUG) || isAffirmative(process.env.VERBOSE);
diff --git a/test/gen-server/ApiServer.ts b/test/gen-server/ApiServer.ts
index 9546a51cb3..39aba8c35d 100644
--- a/test/gen-server/ApiServer.ts
+++ b/test/gen-server/ApiServer.ts
@@ -8,7 +8,7 @@ import {createEmptyOrgUsageSummary, OrgUsageSummary} from 'app/common/DocUsage';
 import {Document, Workspace} from 'app/common/UserAPI';
 import {Organization} from 'app/gen-server/entity/Organization';
 import {Product} from 'app/gen-server/entity/Product';
-import {HomeDBManager, UserChange} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager, UserChange} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {TestServer} from 'test/gen-server/apiUtils';
 import {TEAM_FREE_PLAN} from 'app/common/Features';
 
diff --git a/test/gen-server/ApiServerAccess.ts b/test/gen-server/ApiServerAccess.ts
index 2742fed551..33abe90b1d 100644
--- a/test/gen-server/ApiServerAccess.ts
+++ b/test/gen-server/ApiServerAccess.ts
@@ -4,7 +4,7 @@ import {Deps} from 'app/gen-server/ApiServer';
 import {Organization} from 'app/gen-server/entity/Organization';
 import {Product} from 'app/gen-server/entity/Product';
 import {User} from 'app/gen-server/entity/User';
-import {HomeDBManager, UserChange} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager, UserChange} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {SendGridConfig, SendGridMail} from 'app/gen-server/lib/NotifierTypes';
 import axios, {AxiosResponse} from 'axios';
 import {delay} from 'bluebird';
diff --git a/test/gen-server/ApiServerBugs.ts b/test/gen-server/ApiServerBugs.ts
index 020b09c7d6..f8ef21126c 100644
--- a/test/gen-server/ApiServerBugs.ts
+++ b/test/gen-server/ApiServerBugs.ts
@@ -4,7 +4,7 @@ import * as chai from 'chai';
 import {configForUser} from 'test/gen-server/testUtils';
 import * as testUtils from 'test/server/testUtils';
 
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 
 import {TestServer} from 'test/gen-server/apiUtils';
 
diff --git a/test/gen-server/AuthCaching.ts b/test/gen-server/AuthCaching.ts
index 5789d1ceb5..d28f738274 100644
--- a/test/gen-server/AuthCaching.ts
+++ b/test/gen-server/AuthCaching.ts
@@ -1,5 +1,5 @@
 import {delay} from 'app/common/delay';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {FlexServer} from 'app/server/lib/FlexServer';
 import log from 'app/server/lib/log';
 import {main as mergedServerMain} from 'app/server/mergedServerMain';
diff --git a/test/gen-server/apiUtils.ts b/test/gen-server/apiUtils.ts
index 5737a4967e..2dbaf73cdd 100644
--- a/test/gen-server/apiUtils.ts
+++ b/test/gen-server/apiUtils.ts
@@ -11,7 +11,7 @@ import {User} from 'app/gen-server/entity/User';
 import {Workspace} from 'app/gen-server/entity/Workspace';
 import {SessionUserObj} from 'app/server/lib/BrowserSession';
 import {getDocWorkerMap} from 'app/gen-server/lib/DocWorkerMap';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import * as docUtils from 'app/server/lib/docUtils';
 import {FlexServer, FlexServerOptions} from 'app/server/lib/FlexServer';
 import {main as mergedServerMain, ServerType} from 'app/server/mergedServerMain';
diff --git a/test/gen-server/migrations.ts b/test/gen-server/migrations.ts
index e6a45b9862..f3cc32dfc9 100644
--- a/test/gen-server/migrations.ts
+++ b/test/gen-server/migrations.ts
@@ -1,7 +1,7 @@
 import {QueryRunner} from "typeorm";
 import * as roles from "app/common/roles";
 import {Organization} from 'app/gen-server/entity/Organization';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {Permissions} from 'app/gen-server/lib/Permissions';
 import {assert} from 'chai';
 import {addSeedData, createInitialDb, removeConnection, setUpDB} from 'test/gen-server/seed';
diff --git a/test/gen-server/seed.ts b/test/gen-server/seed.ts
index 5a6addc942..274283cea5 100644
--- a/test/gen-server/seed.ts
+++ b/test/gen-server/seed.ts
@@ -40,7 +40,7 @@ import {Organization} from "app/gen-server/entity/Organization";
 import {Product, PRODUCTS, synchronizeProducts, teamFreeFeatures} from "app/gen-server/entity/Product";
 import {User} from "app/gen-server/entity/User";
 import {Workspace} from "app/gen-server/entity/Workspace";
-import {EXAMPLE_WORKSPACE_NAME} from 'app/gen-server/lib/HomeDBManager';
+import {EXAMPLE_WORKSPACE_NAME} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {Permissions} from 'app/gen-server/lib/Permissions';
 import {getOrCreateConnection, runMigrations, undoLastMigration, updateDb} from 'app/server/lib/dbUtils';
 import {FlexServer} from 'app/server/lib/FlexServer';
diff --git a/test/gen-server/testUtils.ts b/test/gen-server/testUtils.ts
index ef14ad00c9..3f7707abe5 100644
--- a/test/gen-server/testUtils.ts
+++ b/test/gen-server/testUtils.ts
@@ -2,7 +2,7 @@ import {GristLoadConfig} from 'app/common/gristUrls';
 import {BillingAccount} from 'app/gen-server/entity/BillingAccount';
 import {Organization} from 'app/gen-server/entity/Organization';
 import {Product} from 'app/gen-server/entity/Product';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {INotifier} from 'app/server/lib/INotifier';
 import {AxiosRequestConfig} from "axios";
 import {delay} from 'bluebird';
diff --git a/test/nbrowser/homeUtil.ts b/test/nbrowser/homeUtil.ts
index f19bfc5808..b5edccd4f4 100644
--- a/test/nbrowser/homeUtil.ts
+++ b/test/nbrowser/homeUtil.ts
@@ -13,7 +13,7 @@ import {normalizeEmail} from 'app/common/emails';
 import {UserProfile} from 'app/common/LoginSessionAPI';
 import {BehavioralPrompt, UserPrefs, WelcomePopup} from 'app/common/Prefs';
 import {DocWorkerAPI, UserAPI, UserAPIImpl} from 'app/common/UserAPI';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {TestingHooksClient} from 'app/server/lib/TestingHooks';
 import EventEmitter = require('events');
 
diff --git a/test/nbrowser/testServer.ts b/test/nbrowser/testServer.ts
index 417ad8dc5c..4928a2537e 100644
--- a/test/nbrowser/testServer.ts
+++ b/test/nbrowser/testServer.ts
@@ -11,7 +11,7 @@
  * into a file whose path is printed when server starts.
  */
 import {encodeUrl, IGristUrlState, parseSubdomain} from 'app/common/gristUrls';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import log from 'app/server/lib/log';
 import {getAppRoot} from 'app/server/lib/places';
 import {makeGristConfig} from 'app/server/lib/sendAppPage';
diff --git a/test/server/lib/Authorizer.ts b/test/server/lib/Authorizer.ts
index d8d6389d3d..191e3920e4 100644
--- a/test/server/lib/Authorizer.ts
+++ b/test/server/lib/Authorizer.ts
@@ -1,5 +1,5 @@
 import {parseUrlId} from 'app/common/gristUrls';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {DocManager} from 'app/server/lib/DocManager';
 import {FlexServer} from 'app/server/lib/FlexServer';
 import axios from 'axios';
diff --git a/test/server/lib/HostedStorageManager.ts b/test/server/lib/HostedStorageManager.ts
index ae8224c3c2..2fee78c208 100644
--- a/test/server/lib/HostedStorageManager.ts
+++ b/test/server/lib/HostedStorageManager.ts
@@ -2,7 +2,7 @@ import {ErrorOrValue, freezeError, mapGetOrSet, MapWithTTL} from 'app/common/Asy
 import {ObjMetadata, ObjSnapshot, ObjSnapshotWithMetadata} from 'app/common/DocSnapshot';
 import {SCHEMA_VERSION} from 'app/common/schema';
 import {DocWorkerMap} from 'app/gen-server/lib/DocWorkerMap';
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 import {ActiveDoc} from 'app/server/lib/ActiveDoc';
 import {create} from 'app/server/lib/create';
 import {DocManager} from 'app/server/lib/DocManager';
diff --git a/test/testUtils.ts b/test/testUtils.ts
index bda21d131c..e8affb5a49 100644
--- a/test/testUtils.ts
+++ b/test/testUtils.ts
@@ -1,4 +1,4 @@
-import {HomeDBManager} from 'app/gen-server/lib/HomeDBManager';
+import {HomeDBManager} from 'app/gen-server/lib/homedb/HomeDBManager';
 
 export async function getDatabase(typeormDb?: string): Promise<HomeDBManager> {
   const origTypeormDB = process.env.TYPEORM_DATABASE;