Merge pull request #75 from nibble-4bits/feature/retrier-jitterstrate…

…gy-maxdelayseconds

Feature/retrier JitterStrategy MaxDelaySeconds

__tests__/EventLogger.test.ts

@@ -9,6 +9,9 @@ import type {
 } from '../src/typings/EventLogs';
 import { StatesRuntimeError } from '../src/error/predefined/StatesRuntimeError';
 import { EventLogger } from '../src/stateMachine/EventLogger';
+
+// NOTE: We need to import the custom matcher declarations, since VSCode doesn't recognize custom tsconfigs
+// See: https://github.com/microsoft/vscode/issues/12463
 import './_customMatchers';
 
 afterEach(() => {

__tests__/StateExecutor.test.ts

@@ -3,6 +3,10 @@ import { StateExecutor } from '../src/stateMachine/StateExecutor';
 import { EventLogger } from '../src/stateMachine/EventLogger';
 import * as utilModule from '../src/util';
 
+// NOTE: We need to import the custom matcher declarations, since VSCode doesn't recognize custom tsconfigs
+// See: https://github.com/microsoft/vscode/issues/12463
+import './_customMatchers';
+
 afterEach(() => {
   jest.clearAllMocks();
 });
@@ -17,9 +21,11 @@ describe('State Executor', () => {
 
   describe('Retry behavior', () => {
     const defaultMaxRetries = 3;
+    const sleepFnMock = jest.fn();
 
     beforeEach(() => {
-      jest.spyOn(utilModule, 'sleep').mockImplementation(jest.fn());
+      sleepFnMock.mockClear();
+      jest.spyOn(utilModule, 'sleep').mockImplementation(sleepFnMock);
     });
 
     test('should retry state if `Retry` field is specified and thrown error is specified in retrier', async () => {
@@ -185,6 +191,95 @@ describe('State Executor', () => {
       await expect(executorResult).rejects.toThrow();
     });
 
+    test('should wait at most the number of seconds specified in `MaxDelaySeconds`', async () => {
+      const stateDefinition: TaskState = {
+        Type: 'Task',
+        Resource: 'mock-arn',
+        Retry: [
+          {
+            ErrorEquals: ['CustomError'],
+            IntervalSeconds: 3,
+            MaxDelaySeconds: 8,
+          },
+        ],
+        End: true,
+      };
+      const input = {};
+      const context = {};
+      const abortSignal = new AbortController().signal;
+      let retryCount = 0;
+
+      const stateExecutor = new StateExecutor('TaskState', stateDefinition);
+      const { stateResult } = await stateExecutor.execute(input, context, {
+        abortSignal,
+        eventLogger: new EventLogger(),
+        stateMachineOptions: undefined,
+        runOptions: {
+          overrides: {
+            taskResourceLocalHandlers: {
+              TaskState: async () => {
+                if (retryCount < defaultMaxRetries) {
+                  retryCount++;
+                  throw new CustomError('Task state failed');
+                }
+
+                return 1;
+              },
+            },
+          },
+        },
+      });
+
+      expect(stateResult).toBe(1);
+      expect(sleepFnMock).toHaveBeenNthCalledWith(1, 3000, abortSignal);
+      expect(sleepFnMock).toHaveBeenNthCalledWith(2, 6000, abortSignal);
+      expect(sleepFnMock).toHaveBeenNthCalledWith(3, 8000, abortSignal);
+    });
+
+    test('should wait a random amount of seconds if `JitterStrategy` is set to `FULL`', async () => {
+      const stateDefinition: TaskState = {
+        Type: 'Task',
+        Resource: 'mock-arn',
+        Retry: [
+          {
+            ErrorEquals: ['CustomError'],
+            JitterStrategy: 'FULL',
+          },
+        ],
+        End: true,
+      };
+      const input = {};
+      const context = {};
+      const abortSignal = new AbortController().signal;
+      let retryCount = 0;
+
+      const stateExecutor = new StateExecutor('TaskState', stateDefinition);
+      const { stateResult } = await stateExecutor.execute(input, context, {
+        abortSignal,
+        eventLogger: new EventLogger(),
+        stateMachineOptions: undefined,
+        runOptions: {
+          overrides: {
+            taskResourceLocalHandlers: {
+              TaskState: async () => {
+                if (retryCount < defaultMaxRetries) {
+                  retryCount++;
+                  throw new CustomError('Task state failed');
+                }
+
+                return 1;
+              },
+            },
+          },
+        },
+      });
+
+      expect(stateResult).toBe(1);
+      expect(sleepFnMock).toHaveBeenNthCalledWith(1, expect.numberBetween(0, 1000), abortSignal);
+      expect(sleepFnMock).toHaveBeenNthCalledWith(2, expect.numberBetween(0, 2000), abortSignal);
+      expect(sleepFnMock).toHaveBeenNthCalledWith(3, expect.numberBetween(0, 4000), abortSignal);
+    });
+
     describe('Task state', () => {
       test('should retry state if retrier specifies `States.TaskFailed` error name', async () => {
         const stateDefinition: TaskState = {

__tests__/_customMatchers.d.ts

@@ -0,0 +1,15 @@
+/* eslint-disable @typescript-eslint/no-empty-interface */
+export {};
+
+interface CustomMatchers<R = unknown> {
+  toSettle(ms: number): Promise<R>;
+  numberBetween(argumentOne: number, argumentTwo: number): R;
+}
+
+declare global {
+  namespace jest {
+    interface Expect extends CustomMatchers {}
+    interface Matchers<R> extends CustomMatchers<R> {}
+    interface InverseAsymmetricMatchers extends CustomMatchers {}
+  }
+}

__tests__/_customMatchers.ts → __tests__/customMatchers.ts

@@ -1,18 +1,3 @@
-/* eslint-disable @typescript-eslint/no-empty-interface */
-/* eslint-disable @typescript-eslint/no-namespace */
-
-interface CustomMatchers<R = unknown> {
-  toSettle(ms: number): Promise<R>;
-}
-
-declare global {
-  namespace jest {
-    interface Expect extends CustomMatchers {}
-    interface Matchers<R> extends CustomMatchers<R> {}
-    interface InverseAsymmetricMatchers extends CustomMatchers {}
-  }
-}
-
 import { expect } from '@jest/globals';
 
 const TIMED_OUT = Symbol();
@@ -40,4 +25,19 @@ expect.extend({
       pass: false,
     };
   },
+  numberBetween(received: number, argumentOne: number, argumentTwo: number) {
+    if (argumentOne > argumentTwo) {
+      // Switch values
+      [argumentOne, argumentTwo] = [argumentTwo, argumentOne];
+    }
+
+    const pass = received >= argumentOne && received <= argumentTwo;
+
+    return {
+      pass,
+      message: pass
+        ? () => `expected ${received} not to be between ${argumentOne} and ${argumentTwo}`
+        : () => `expected ${received} to be between ${argumentOne} and ${argumentTwo}`,
+    };
+  },
 });

__tests__/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "skipLibCheck": false,
+    "noEmit": true
+  },
+  "files": ["_customMatchers.d.ts"]
+}

docs/execution-event-logs.md

@@ -488,6 +488,8 @@ interface RetryData {
     IntervalSeconds?: number;
     MaxAttempts?: number;
     BackoffRate?: number;
+    MaxDelaySeconds?: number;
+    JitterStrategy?: 'NONE' | 'FULL';
   };
   attempt: number;
 }

jest.config.cjs

@@ -1,7 +1,21 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+
+const { defaults: tsjPreset } = require('ts-jest/presets');
+
 /** @type {import('ts-jest').JestConfigWithTsJest} */
 module.exports = {
-  preset: 'ts-jest',
   testEnvironment: 'node',
   testMatch: ['**/__tests__/**/*.test.ts'],
+  setupFilesAfterEnv: ['./__tests__/customMatchers.ts'],
+  moduleFileExtensions: ['js', 'mjs', 'cjs', 'ts', 'd.ts', 'json'],
   verbose: true,
+  transform: {
+    '^.+\\.ts$': [
+      'ts-jest',
+      {
+        ...tsjPreset.transform,
+        tsconfig: './__tests__/tsconfig.json',
+      },
+    ],
+  },
 };

src/stateMachine/StateExecutor.ts

@@ -29,7 +29,7 @@ import { SucceedStateAction } from './stateActions/SucceedStateAction';
 import { TaskStateAction } from './stateActions/TaskStateAction';
 import { WaitStateAction } from './stateActions/WaitStateAction';
 import { StatesTimeoutError } from '../error/predefined/StatesTimeoutError';
-import { sleep } from '../util';
+import { clamp, sleep, getRandomNumber } from '../util';
 import cloneDeep from 'lodash/cloneDeep.js';
 
 /**
@@ -47,6 +47,11 @@ const DEFAULT_INTERVAL_SECONDS = 1;
  */
 const DEFAULT_BACKOFF_RATE = 2.0;
 
+/**
+ * Default jitter strategy for retry.
+ */
+const DEFAULT_JITTER_STRATEGY = 'NONE';
+
 /**
  * The wildcard error. This matches all thrown errors.
  */
@@ -223,12 +228,18 @@ export class StateExecutor {
 
     for (let i = 0; i < this.stateDefinition.Retry.length; i++) {
       const retrier = this.stateDefinition.Retry[i];
+      const jitterStrategy = retrier.JitterStrategy ?? DEFAULT_JITTER_STRATEGY;
       const maxAttempts = retrier.MaxAttempts ?? DEFAULT_MAX_ATTEMPTS;
       const intervalSeconds = retrier.IntervalSeconds ?? DEFAULT_INTERVAL_SECONDS;
       const backoffRate = retrier.BackoffRate ?? DEFAULT_BACKOFF_RATE;
-      const waitTimeBeforeRetry = intervalSeconds * Math.pow(backoffRate, this.retrierAttempts[i]) * 1000;
+      const waitInterval = intervalSeconds * Math.pow(backoffRate, this.retrierAttempts[i]);
       const retryable = error.isRetryable ?? true;
 
+      let waitTimeBeforeRetry = clamp(waitInterval, 1, retrier.MaxDelaySeconds) * 1000;
+      if (jitterStrategy === 'FULL') {
+        waitTimeBeforeRetry = getRandomNumber(0, waitTimeBeforeRetry);
+      }
+
       for (const retrierError of retrier.ErrorEquals) {
         const isErrorMatch = retrierError === error.name;
         const isErrorWildcard = retrierError === WILDCARD_ERROR;

src/typings/ErrorHandling.ts

@@ -3,6 +3,8 @@ export type Retrier = {
   IntervalSeconds?: number; // default 1
   MaxAttempts?: number; // default 3
   BackoffRate?: number; // default 2.0
+  MaxDelaySeconds?: number;
+  JitterStrategy?: 'NONE' | 'FULL';
 };
 
 export interface RetryableState {

src/util/index.ts

@@ -73,4 +73,10 @@ export function stringifyJSONValue(value: unknown) {
   return JSON.stringify(value);
 }
 
+export function clamp(value: number, min: number | undefined, max: number | undefined) {
+  if (min && value < min) return min;
+  if (max && value > max) return max;
+  return value;
+}
+
 export { getRandomNumber, sfc32, cyrb128 } from './random';