chore: introduce update-snapshots=changed (#33735)

docs/src/test-api/class-fullconfig.md

@@ -114,7 +114,7 @@ See [`property: TestConfig.shard`].
 
 ## property: FullConfig.updateSnapshots
 * since: v1.10
-- type: <[UpdateSnapshots]<"all"|"none"|"missing">>
+- type: <[UpdateSnapshots]<"all"|"changed"|"missing"|"none">>
 
 See [`property: TestConfig.updateSnapshots`].
 

docs/src/test-api/class-testconfig.md

@@ -570,12 +570,13 @@ export default defineConfig({
 
 ## property: TestConfig.updateSnapshots
 * since: v1.10
-- type: ?<[UpdateSnapshots]<"all"|"none"|"missing">>
+- type: ?<[UpdateSnapshots]<"all"|"changed"|"missing"|"none">>
 
 Whether to update expected snapshots with the actual results produced by the test run. Defaults to `'missing'`.
-* `'all'` - All tests that are executed will update snapshots that did not match. Matching snapshots will not be updated.
-* `'none'` - No snapshots are updated.
+* `'all'` - All tests that are executed will update snapshots.
+* `'changed'` - All tests that are executed will update snapshots that did not match. Matching snapshots will not be updated.
 * `'missing'` - Missing snapshots are created, for example when authoring a new test and running it for the first time. This is the default.
+* `'none'` - No snapshots are updated.
 
 Learn more about [snapshots](../test-snapshots.md).
 

docs/src/test-cli-js.md

@@ -76,33 +76,40 @@ Here are the most common options available in the command line.
 
 Complete set of Playwright Test options is available in the [configuration file](./test-use-options.md). Following options can be passed to a command line and take priority over the configuration file:
 
+<!-- // Note: packages/playwright/src/program.ts is the source of truth. -->
+
 | Option | Description |
 | :- | :- |
-| Non-option arguments | Each argument is treated as a regular expression matched against the full test file path. Only tests from the files matching the pattern will be executed. Special symbols like `$` or `*` should be escaped with `\`. In many shells/terminals you may need to quote the arguments. |
-| `-c <file>` or `--config <file>`| Configuration file. If not passed, defaults to `playwright.config.ts` or `playwright.config.js` in the current directory. |
-| `--debug`| Run tests with Playwright Inspector. Shortcut for `PWDEBUG=1` environment variable and `--timeout=0 --max-failures=1 --headed --workers=1` options.|
-| `--fail-on-flaky-tests` | Fails test runs that contain flaky tests. By default flaky tests count as successes. |
-| `--forbid-only` | Whether to disallow `test.only`. Useful on CI.|
-| `--global-timeout <number>` | Total timeout for the whole test run in milliseconds. By default, there is no global timeout. Learn more about [various timeouts](./test-timeouts.md).|
-| `-g <grep>` or `--grep <grep>` | Only run tests matching this regular expression. For example, this will run `'should add to cart'` when passed `-g "add to cart"`.  The regular expression will be tested against the string that consists of the project name, test file name, `test.describe` titles if any, test title and all test tags, separated by spaces, e.g. `chromium my-test.spec.ts my-suite my-test @smoke`. The filter does not apply to the tests from dependency projects, i.e. Playwright will still run all tests from [project dependencies](./test-projects.md#dependencies). |
-| `--grep-invert <grep>` | Only run tests **not** matching this regular expression. The opposite of `--grep`. The filter does not apply to the tests from dependency projects, i.e. Playwright will still run all tests from [project dependencies](./test-projects.md#dependencies).|
-| `--headed` | Run tests in headed browsers. Useful for debugging. |
-| `--ignore-snapshots` | Whether to ignore [snapshots](./test-snapshots.md). Use this when snapshot expectations are known to be different, e.g. running tests on Linux against Windows screenshots. |
-| `--last-failed` | Only re-run the failures.|
-| `--list` | list all the tests, but do not run them.|
-| `--max-failures <N>` or `-x`| Stop after the first `N` test failures. Passing `-x` stops after the first failure.|
-| `--no-deps` | Ignore the dependencies between projects and behave as if they were not specified. |
-| `--output <dir>` | Directory for artifacts produced by tests, defaults to `test-results`. |
-| `--only-changed [ref]` | Only run test files that have been changed between working tree and "ref". Defaults to running all uncommitted changes with ref=HEAD. Only supports Git. |
-| `--pass-with-no-tests` | Allows the test suite to pass when no files are found. |
-| `--project <name>` | Only run tests from the specified [projects](./test-projects.md), supports '*' wildcard. Defaults to running all projects defined in the configuration file.|
-| `--quiet` | Whether to suppress stdout and stderr from the tests. |
-| `--repeat-each <N>` | Run each test `N` times, defaults to one. |
-| `--reporter <reporter>` | Choose a reporter: minimalist `dot`, concise `line` or detailed `list`. See [reporters](./test-reporters.md) for more information. You can also pass a path to a [custom reporter](./test-reporters.md#custom-reporters) file. |
-| `--retries <number>` | The maximum number of [retries](./test-retries.md#retries) for flaky tests, defaults to zero (no retries). |
-| `--shard <shard>` | [Shard](./test-parallel.md#shard-tests-between-multiple-machines) tests and execute only selected shard, specified in the form `current/all`, 1-based, for example `3/5`.|
-| `--timeout <number>` | Maximum timeout in milliseconds for each test, defaults to 30 seconds. Learn more about [various timeouts](./test-timeouts.md).|
-| `--trace <mode>` | Force tracing mode, can be `on`, `off`, `on-first-retry`, `on-all-retries`, `retain-on-failure` |
-| `--tsconfig <path>` | Path to a single tsconfig applicable to all imported files. See [tsconfig resolution](./test-typescript.md#tsconfig-resolution) for more details. |
-| `--update-snapshots` or `-u` | Whether to update [snapshots](./test-snapshots.md) with actual results instead of comparing them. Use this when snapshot expectations have changed.|
-| `--workers <number>` or `-j <number>`| The maximum number of concurrent worker processes that run in [parallel](./test-parallel.md). |
+| Non-option arguments | Each argument is treated as a regular expression matched against the full test file path. Only tests from files matching the pattern will be executed. Special symbols like `$` or `*` should be escaped with `\`. In many shells/terminals you may need to quote the arguments. |
+| `-c <file>` or `--config <file>` | Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}". Defaults to `playwright.config.ts` or `playwright.config.js` in the current directory. |
+| `--debug` | Run tests with Playwright Inspector. Shortcut for `PWDEBUG=1` environment variable and `--timeout=0 --max-failures=1 --headed --workers=1` options. |
+| `--fail-on-flaky-tests` | Fail if any test is flagged as flaky (default: false). |
+| `--forbid-only` | Fail if `test.only` is called (default: false). Useful on CI. |
+| `--fully-parallel` | Run all tests in parallel (default: false). |
+| `--global-timeout <timeout>` | Maximum time this test suite can run in milliseconds (default: unlimited). |
+| `-g <grep>` or `--grep <grep>` | Only run tests matching this regular expression (default: ".*"). |
+| `-gv <grep>` or `--grep-invert <grep>` | Only run tests that do not match this regular expression. |
+| `--headed` | Run tests in headed browsers (default: headless). |
+| `--ignore-snapshots` | Ignore screenshot and snapshot expectations. |
+| `--last-failed` | Only re-run the failures. |
+| `--list` | Collect all the tests and report them, but do not run. |
+| `--max-failures <N>` or `-x` | Stop after the first `N` failures. Passing `-x` stops after the first failure. |
+| `--no-deps` | Do not run project dependencies. |
+| `--output <dir>` | Folder for output artifacts (default: "test-results"). |
+| `--only-changed [ref]` | Only run test files that have been changed between 'HEAD' and 'ref'. Defaults to running all uncommitted changes. Only supports Git. |
+| `--pass-with-no-tests` | Makes test run succeed even if no tests were found. |
+| `--project <project-name...>` | Only run tests from the specified list of projects, supports '*' wildcard (default: run all projects). |
+| `--quiet` | Suppress stdio. |
+| `--repeat-each <N>` | Run each test `N` times (default: 1). |
+| `--reporter <reporter>` | Reporter to use, comma-separated, can be "dot", "line", "list", or others (default: "list"). You can also pass a path to a custom reporter file. |
+| `--retries <retries>` | Maximum retry count for flaky tests, zero for no retries (default: no retries). |
+| `--shard <shard>` | Shard tests and execute only the selected shard, specified in the form "current/all", 1-based, e.g., "3/5". |
+| `--timeout <timeout>` | Specify test timeout threshold in milliseconds, zero for unlimited (default: 30 seconds). |
+| `--trace <mode>` | Force tracing mode, can be "on", "off", "on-first-retry", "on-all-retries", "retain-on-failure", "retain-on-first-failure". |
+| `--tsconfig <path>` | Path to a single tsconfig applicable to all imported files (default: look up tsconfig for each imported file separately). |
+| `--ui` | Run tests in interactive UI mode. |
+| `--ui-host <host>` | Host to serve UI on; specifying this option opens UI in a browser tab. |
+| `--ui-port <port>` | Port to serve UI on, 0 for any free port; specifying this option opens UI in a browser tab. |
+| `-u` or `--update-snapshots [mode]` | Update snapshots with actual results. Possible values are "all", "changed", "missing", and "none". Not passing defaults to "missing"; passing without a value defaults to "changed". |
+| `-j <workers>` or `--workers <workers>` | Number of concurrent workers or percentage of logical CPU cores, use 1 to run in a single worker (default: 50%). |
+| `-x` | Stop after the first failure. |

packages/playwright-core/src/utils/comparators.ts

@@ -38,7 +38,7 @@ export function getComparator(mimeType: string): Comparator {
 
 const JPEG_JS_MAX_BUFFER_SIZE_IN_MB = 5 * 1024; // ~5 GB
 
-function compareBuffersOrStrings(actualBuffer: Buffer | string, expectedBuffer: Buffer): ComparatorResult {
+export function compareBuffersOrStrings(actualBuffer: Buffer | string, expectedBuffer: Buffer): ComparatorResult {
   if (typeof actualBuffer === 'string')
     return compareText(actualBuffer, expectedBuffer);
   if (!actualBuffer || !(actualBuffer instanceof Buffer))

packages/playwright/src/common/ipc.ts

@@ -38,7 +38,7 @@ export type ConfigCLIOverrides = {
   timeout?: number;
   tsconfig?: string;
   ignoreSnapshots?: boolean;
-  updateSnapshots?: 'all'|'none'|'missing';
+  updateSnapshots?: 'all'|'changed'|'missing'|'none';
   workers?: number | string;
   projects?: { name: string, use?: any }[],
   use?: any;

packages/playwright/src/matchers/toMatchAriaSnapshot.ts

@@ -57,8 +57,6 @@ export async function toMatchAriaSnapshot(
   }
 
   const generateMissingBaseline = updateSnapshots === 'missing' && !expected;
-  const generateNewBaseline = updateSnapshots === 'all' || generateMissingBaseline;
-
   if (generateMissingBaseline) {
     if (this.isNot) {
       const message = `Matchers using ".not" can't generate new baselines`;
@@ -100,10 +98,13 @@ export async function toMatchAriaSnapshot(
     }
   };
 
-  if (!this.isNot && pass === this.isNot && generateNewBaseline) {
-    // Only rebaseline failed snapshots.
-    const suggestedRebaseline = `toMatchAriaSnapshot(\`\n${escapeTemplateString(indent(typedReceived.regex, '{indent}  '))}\n{indent}\`)`;
-    return { pass: this.isNot, message: () => '', name: 'toMatchAriaSnapshot', suggestedRebaseline };
+  if (!this.isNot) {
+    if ((updateSnapshots === 'all') ||
+        (updateSnapshots === 'changed' && pass === this.isNot) ||
+        generateMissingBaseline) {
+      const suggestedRebaseline = `toMatchAriaSnapshot(\`\n${escapeTemplateString(indent(typedReceived.regex, '{indent}  '))}\n{indent}\`)`;
+      return { pass: this.isNot, message: () => '', name: 'toMatchAriaSnapshot', suggestedRebaseline };
+    }
   }
 
   return {

packages/playwright/src/matchers/toMatchSnapshot.ts

@@ -18,7 +18,7 @@ import type { Locator, Page } from 'playwright-core';
 import type { ExpectScreenshotOptions, Page as PageEx } from 'playwright-core/lib/client/page';
 import { currentTestInfo } from '../common/globals';
 import type { ImageComparatorOptions, Comparator } from 'playwright-core/lib/utils';
-import { getComparator, isString, sanitizeForFilePath } from 'playwright-core/lib/utils';
+import { compareBuffersOrStrings, getComparator, isString, sanitizeForFilePath } from 'playwright-core/lib/utils';
 import {
   addSuffixToFilePath,
   trimLongString, callLogText,
@@ -83,7 +83,7 @@ class SnapshotHelper {
   readonly diffPath: string;
   readonly mimeType: string;
   readonly kind: 'Screenshot'|'Snapshot';
-  readonly updateSnapshots: 'all' | 'none' | 'missing';
+  readonly updateSnapshots: 'all' | 'changed' | 'missing' | 'none';
   readonly comparator: Comparator;
   readonly options: Omit<ToHaveScreenshotOptions, '_comparator'> & { comparator?: string };
   readonly matcherName: string;
@@ -199,7 +199,7 @@ class SnapshotHelper {
   }
 
   handleMissingNegated(): ImageMatcherResult {
-    const isWriteMissingMode = this.updateSnapshots === 'all' || this.updateSnapshots === 'missing';
+    const isWriteMissingMode = this.updateSnapshots !== 'none';
     const message = `A snapshot doesn't exist at ${this.expectedPath}${isWriteMissingMode ? ', matchers using ".not" won\'t write them automatically.' : '.'}`;
     // NOTE: 'isNot' matcher implies inversed value.
     return this.createMatcherResult(message, true);
@@ -221,14 +221,14 @@ class SnapshotHelper {
   }
 
   handleMissing(actual: Buffer | string): ImageMatcherResult {
-    const isWriteMissingMode = this.updateSnapshots === 'all' || this.updateSnapshots === 'missing';
+    const isWriteMissingMode = this.updateSnapshots !== 'none';
     if (isWriteMissingMode)
       writeFileSync(this.expectedPath, actual);
     this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-expected'), contentType: this.mimeType, path: this.expectedPath });
     writeFileSync(this.actualPath, actual);
     this.testInfo.attachments.push({ name: addSuffixToFilePath(this.attachmentBaseName, '-actual'), contentType: this.mimeType, path: this.actualPath });
     const message = `A snapshot doesn't exist at ${this.expectedPath}${isWriteMissingMode ? ', writing actual.' : '.'}`;
-    if (this.updateSnapshots === 'all') {
+    if (this.updateSnapshots === 'all' || this.updateSnapshots === 'changed') {
       /* eslint-disable no-console */
       console.log(message);
       return this.createMatcherResult(message, true);
@@ -317,17 +317,30 @@ export function toMatchSnapshot(
     return helper.handleMissing(received);
 
   const expected = fs.readFileSync(helper.expectedPath);
-  const result = helper.comparator(received, expected, helper.options);
-  if (!result)
-    return helper.handleMatching();
 
   if (helper.updateSnapshots === 'all') {
+    if (!compareBuffersOrStrings(received, expected))
+      return helper.handleMatching();
+    writeFileSync(helper.expectedPath, received);
+    /* eslint-disable no-console */
+    console.log(helper.expectedPath + ' is not the same, writing actual.');
+    return helper.createMatcherResult(helper.expectedPath + ' running with --update-snapshots, writing actual.', true);
+  }
+
+  if (helper.updateSnapshots === 'changed') {
+    const result = helper.comparator(received, expected, helper.options);
+    if (!result)
+      return helper.handleMatching();
     writeFileSync(helper.expectedPath, received);
     /* eslint-disable no-console */
     console.log(helper.expectedPath + ' does not match, writing actual.');
     return helper.createMatcherResult(helper.expectedPath + ' running with --update-snapshots, writing actual.', true);
   }
 
+  const result = helper.comparator(received, expected, helper.options);
+  if (!result)
+    return helper.handleMatching();
+
   const receiver = isString(received) ? 'string' : 'Buffer';
   const header = matcherHint(this, undefined, 'toMatchSnapshot', receiver, undefined, undefined);
   return helper.handleDifferent(received, expected, undefined, result.diff, header, result.errorMessage, undefined);
@@ -421,21 +434,30 @@ export async function toHaveScreenshot(
   // General case:
   // - snapshot exists
   // - regular matcher (i.e. not a `.not`)
-  // - perhaps an 'all' flag to update non-matching screenshots
-  expectScreenshotOptions.expected = await fs.promises.readFile(helper.expectedPath);
-  const { actual, previous, diff, errorMessage, log, timedOut } = await page._expectScreenshot(expectScreenshotOptions);
-
-  if (!errorMessage)
-    return helper.handleMatching();
+  const expected = await fs.promises.readFile(helper.expectedPath);
+  expectScreenshotOptions.expected = helper.updateSnapshots === 'all' ? undefined : expected;
 
-  if (helper.updateSnapshots === 'all') {
+  const { actual, previous, diff, errorMessage, log, timedOut } = await page._expectScreenshot(expectScreenshotOptions);
+  const writeFiles = () => {
     writeFileSync(helper.expectedPath, actual!);
     writeFileSync(helper.actualPath, actual!);
     /* eslint-disable no-console */
     console.log(helper.expectedPath + ' is re-generated, writing actual.');
     return helper.createMatcherResult(helper.expectedPath + ' running with --update-snapshots, writing actual.', true);
+  };
+
+  if (!errorMessage) {
+    // Screenshot is matching, but is not necessarily the same as the expected.
+    if (helper.updateSnapshots === 'all' && actual && compareBuffersOrStrings(actual, expected)) {
+      console.log(helper.expectedPath + ' is re-generated, writing actual.');
+      return writeFiles();
+    }
+    return helper.handleMatching();
   }
 
+  if (helper.updateSnapshots === 'changed' || helper.updateSnapshots === 'all')
+    return writeFiles();
+
   const header = matcherHint(this, undefined, 'toHaveScreenshot', receiver, undefined, undefined, timedOut ? timeout : undefined);
   return helper.handleDifferent(actual, expectScreenshotOptions.expected, previous, diff, header, errorMessage, log);
 }