From 83deb9f0373c586f6db8c56afa94bc6e0e994432 Mon Sep 17 00:00:00 2001 From: Michal Dorner Date: Wed, 24 Jun 2020 21:53:31 +0200 Subject: [PATCH] Improve change detection for feature branches (#16) * Detect changes against configured base branch * Update README and action.yml * Add job.outputs example * Update CHANGELOG --- CHANGELOG.md | 3 + README.md | 56 ++++++++++++++-- __tests__/{main.test.ts => filter.test.ts} | 0 __tests__/git.test.ts | 19 ++++++ action.yml | 6 ++ dist/index.js | 75 ++++++++++++++++++---- src/git.ts | 30 +++++++-- src/main.ts | 48 +++++++++++--- 8 files changed, 202 insertions(+), 35 deletions(-) rename __tests__/{main.test.ts => filter.test.ts} (100%) create mode 100644 __tests__/git.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b1d5eb62..da33177d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## v2.2.0 +- [Improve change detection for feature branches](https://github.com/dorny/paths-filter/pull/16) + ## v2.1.0 - [Support reusable paths blocks with yaml anchors](https://github.com/dorny/paths-filter/pull/13) diff --git a/README.md b/README.md index 5e66b817..8ea0ca5c 100644 --- a/README.md +++ b/README.md @@ -10,9 +10,12 @@ It saves time and resources especially in monorepo setups, where you can run slo Github workflows built-in [path filters](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#onpushpull_requestpaths) doesn't allow this because they doesn't work on a level of individual jobs or steps. -Action supports workflows triggered by: -- **[pull_request](https://help.github.com/en/actions/reference/events-that-trigger-workflows#pull-request-event-pull_request)**: changes are detected against the base branch -- **[push](https://help.github.com/en/actions/reference/events-that-trigger-workflows#push-event-push)**: changes are detected against the most recent commit on the same branch before the push +Supported workflows: +- Action triggered by **[pull_request](https://help.github.com/en/actions/reference/events-that-trigger-workflows#pull-request-event-pull_request)** event: + - changes detected against the pull request base branch +- Action triggered by **[push](https://help.github.com/en/actions/reference/events-that-trigger-workflows#push-event-push)** event: + - changes detected against the most recent commit on the same branch before the push + - changes detected against the top of the configured *base* branch (e.g. master) ## Usage @@ -22,7 +25,10 @@ Corresponding output variable will be created to indicate if there's a changed f Output variables can be later used in the `if` clause to conditionally run specific steps. ### Inputs -- **`token`**: GitHub Access Token - defaults to `${{ github.token }}` +- **`token`**: GitHub Access Token - defaults to `${{ github.token }}` so you don't have to explicitly provide it. +- **`base`**: Git reference (e.g. branch name) against which the changes will be detected. Defaults to repository default branch (e.g. master). + If it references same branch it was pushed to, changes are detected against the most recent commit before the push. + This option is ignored if action is triggered by *pull_request* event. - **`filters`**: Path to the configuration file or directly embedded string in YAML format. Filter configuration is a dictionary, where keys specifies rule names and values are lists of file path patterns. ### Outputs @@ -34,6 +40,8 @@ Output variables can be later used in the `if` clause to conditionally run speci - minimatch [dot](https://www.npmjs.com/package/minimatch#dot) option is set to true - therefore globbing will match also paths where file or folder name starts with a dot. - You can use YAML anchors to reuse path expression(s) inside another rule. See example in the tests. +- If changes are detected against the previous commit and there is none (i.e. first push of a new branch), all filter rules will report changed files. +- You can use `base: ${{ github.ref }}` to configure change detection against previous commit for every branch you create. ### Example ```yaml @@ -49,7 +57,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - uses: dorny/paths-filter@v2.1.0 + - uses: dorny/paths-filter@v2.2.0 id: filter with: # inline YAML or path to separate file (e.g.: .github/filters.yaml) @@ -75,13 +83,47 @@ jobs: run: ... ``` +If your workflow uses multiple jobs, you can put *paths-filter* into own job and use +[job outputs](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjobs_idoutputs) +in other jobs [if](https://help.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idif) statements: +```yml +on: + pull_request: + branches: + - master +jobs: + changes: + runs-on: ubuntu-latest + # Set job outputs to values from filter step + outputs: + backend: ${{ steps.filter.outputs.backend }} + frontend: ${{ steps.filter.outputs.frontend }} + steps: + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@v2.2.0 + id: filter + with: + # Filters stored in own yaml file + filters: '.github/filters.yml' + backend: + if: ${{ needs.changes.outputs.backend == 'true' }} + steps: + - ... + frontend: + if: ${{ needs.changes.outputs.frontend == 'true' }} + steps: + - ... +``` + ## How it works 1. If action was triggered by pull request: - If access token was provided it's used to fetch list of changed files from Github API. - - If access token was not provided, top of the base branch is fetched and changed files are detected using `git diff-index` command. + - If access token was not provided, top of the base branch is fetched and changed files are detected using `git diff-index ` command. 2. If action was triggered by push event - - Last commit before the push is fetched and changed files are detected using `git diff-index` command. + - if *base* input parameter references same branch it was pushed to, most recent commit before the push is fetched + - If *base* input parameter references other branch, top of that branch is fetched + - changed files are detected using `git diff-index FETCH_HEAD` command. 3. For each filter rule it checks if there is any matching file 4. Output variables are set diff --git a/__tests__/main.test.ts b/__tests__/filter.test.ts similarity index 100% rename from __tests__/main.test.ts rename to __tests__/filter.test.ts diff --git a/__tests__/git.test.ts b/__tests__/git.test.ts new file mode 100644 index 00000000..b5c25b01 --- /dev/null +++ b/__tests__/git.test.ts @@ -0,0 +1,19 @@ +import * as git from '../src/git' + +describe('git utility function tests (those not invoking git)', () => { + test('Detects if ref references a tag', () => { + expect(git.isTagRef('refs/tags/v1.0')).toBeTruthy() + expect(git.isTagRef('refs/heads/master')).toBeFalsy() + expect(git.isTagRef('master')).toBeFalsy() + }) + test('Trims "refs/" from ref', () => { + expect(git.trimRefs('refs/heads/master')).toBe('heads/master') + expect(git.trimRefs('heads/master')).toBe('heads/master') + expect(git.trimRefs('master')).toBe('master') + }) + test('Trims "refs/" and "heads/" from ref', () => { + expect(git.trimRefsHeads('refs/heads/master')).toBe('master') + expect(git.trimRefsHeads('heads/master')).toBe('master') + expect(git.trimRefsHeads('master')).toBe('master') + }) +}) diff --git a/action.yml b/action.yml index 8865f335..723e07eb 100644 --- a/action.yml +++ b/action.yml @@ -6,6 +6,12 @@ inputs: description: 'GitHub Access Token' required: false default: ${{ github.token }} + base: + description: | + Git reference (e.g. branch name) against which the changes will be detected. Defaults to repository default branch (e.g. master). + If it references same branch it was pushed to, changes are detected against the most recent commit before the push. + This option is ignored if action is triggered by pull_request event. + required: false filters: description: 'Path to the configuration file or YAML string with filters definition' required: true diff --git a/dist/index.js b/dist/index.js index 3be18260..9f57a0b6 100644 --- a/dist/index.js +++ b/dist/index.js @@ -3798,21 +3798,23 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge }); }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.getChangedFiles = exports.fetchCommit = void 0; +exports.trimRefsHeads = exports.trimRefs = exports.isTagRef = exports.getChangedFiles = exports.fetchCommit = exports.FETCH_HEAD = exports.NULL_SHA = void 0; const exec_1 = __webpack_require__(986); -function fetchCommit(sha) { +exports.NULL_SHA = '0000000000000000000000000000000000000000'; +exports.FETCH_HEAD = 'FETCH_HEAD'; +function fetchCommit(ref) { return __awaiter(this, void 0, void 0, function* () { - const exitCode = yield exec_1.exec('git', ['fetch', '--depth=1', 'origin', sha]); + const exitCode = yield exec_1.exec('git', ['fetch', '--depth=1', '--no-tags', 'origin', ref]); if (exitCode !== 0) { - throw new Error(`Fetching commit ${sha} failed`); + throw new Error(`Fetching ${ref} failed`); } }); } exports.fetchCommit = fetchCommit; -function getChangedFiles(sha) { +function getChangedFiles(ref) { return __awaiter(this, void 0, void 0, function* () { let output = ''; - const exitCode = yield exec_1.exec('git', ['diff-index', '--name-only', sha], { + const exitCode = yield exec_1.exec('git', ['diff-index', '--name-only', ref], { listeners: { stdout: (data) => (output += data.toString()) } @@ -3827,6 +3829,22 @@ function getChangedFiles(sha) { }); } exports.getChangedFiles = getChangedFiles; +function isTagRef(ref) { + return ref.startsWith('refs/tags/'); +} +exports.isTagRef = isTagRef; +function trimRefs(ref) { + return trimStart(ref, 'refs/'); +} +exports.trimRefs = trimRefs; +function trimRefsHeads(ref) { + const trimRef = trimStart(ref, 'refs/'); + return trimStart(trimRef, 'heads/'); +} +exports.trimRefsHeads = trimRefsHeads; +function trimStart(ref, start) { + return ref.startsWith(start) ? ref.substr(start.length) : ref; +} /***/ }), @@ -4487,9 +4505,18 @@ function run() { const filtersYaml = isPathInput(filtersInput) ? getConfigFileContent(filtersInput) : filtersInput; const filter = new filter_1.default(filtersYaml); const files = yield getChangedFiles(token); - const result = filter.match(files); - for (const key in result) { - core.setOutput(key, String(result[key])); + if (files === null) { + // Change detection was not possible + // Set all filter keys to true (i.e. changed) + for (const key in filter.rules) { + core.setOutput(key, String(true)); + } + } + else { + const result = filter.match(files); + for (const key in result) { + core.setOutput(key, String(result[key])); + } } } catch (error) { @@ -4516,20 +4543,40 @@ function getChangedFiles(token) { return token ? yield getChangedFilesFromApi(token, pr) : yield getChangedFilesFromGit(pr.base.sha); } else if (github.context.eventName === 'push') { - const push = github.context.payload; - return yield getChangedFilesFromGit(push.before); + return getChangedFilesFromPush(); } else { throw new Error('This action can be triggered only by pull_request or push event'); } }); } +function getChangedFilesFromPush() { + return __awaiter(this, void 0, void 0, function* () { + const push = github.context.payload; + // No change detection for pushed tags + if (git.isTagRef(push.ref)) + return null; + // Get base from input or use repo default branch. + // It it starts with 'refs/', it will be trimmed (git fetch refs/heads/ doesn't work) + const baseInput = git.trimRefs(core.getInput('base', { required: false }) || push.repository.default_branch); + // If base references same branch it was pushed to, we will do comparison against the previously pushed commit. + // Otherwise changes are detected against the base reference + const base = git.trimRefsHeads(baseInput) === git.trimRefsHeads(push.ref) ? push.before : baseInput; + // There is no previous commit for comparison + // e.g. change detection against previous commit of just pushed new branch + if (base === git.NULL_SHA) + return null; + return yield getChangedFilesFromGit(base); + }); +} // Fetch base branch and use `git diff` to determine changed files -function getChangedFilesFromGit(sha) { +function getChangedFilesFromGit(ref) { return __awaiter(this, void 0, void 0, function* () { core.debug('Fetching base branch and using `git diff-index` to determine changed files'); - yield git.fetchCommit(sha); - return yield git.getChangedFiles(sha); + yield git.fetchCommit(ref); + // FETCH_HEAD will always point to the just fetched commit + // No matter if ref is SHA, branch or tag name or full git ref + return yield git.getChangedFiles(git.FETCH_HEAD); }); } // Uses github REST api to get list of files changed in PR diff --git a/src/git.ts b/src/git.ts index 4cfc5db3..25829e11 100644 --- a/src/git.ts +++ b/src/git.ts @@ -1,15 +1,18 @@ import {exec} from '@actions/exec' -export async function fetchCommit(sha: string): Promise { - const exitCode = await exec('git', ['fetch', '--depth=1', 'origin', sha]) +export const NULL_SHA = '0000000000000000000000000000000000000000' +export const FETCH_HEAD = 'FETCH_HEAD' + +export async function fetchCommit(ref: string): Promise { + const exitCode = await exec('git', ['fetch', '--depth=1', '--no-tags', 'origin', ref]) if (exitCode !== 0) { - throw new Error(`Fetching commit ${sha} failed`) + throw new Error(`Fetching ${ref} failed`) } } -export async function getChangedFiles(sha: string): Promise { +export async function getChangedFiles(ref: string): Promise { let output = '' - const exitCode = await exec('git', ['diff-index', '--name-only', sha], { + const exitCode = await exec('git', ['diff-index', '--name-only', ref], { listeners: { stdout: (data: Buffer) => (output += data.toString()) } @@ -24,3 +27,20 @@ export async function getChangedFiles(sha: string): Promise { .map(s => s.trim()) .filter(s => s.length > 0) } + +export function isTagRef(ref: string): boolean { + return ref.startsWith('refs/tags/') +} + +export function trimRefs(ref: string): string { + return trimStart(ref, 'refs/') +} + +export function trimRefsHeads(ref: string): string { + const trimRef = trimStart(ref, 'refs/') + return trimStart(trimRef, 'heads/') +} + +function trimStart(ref: string, start: string): string { + return ref.startsWith(start) ? ref.substr(start.length) : ref +} diff --git a/src/main.ts b/src/main.ts index f67d7cb7..78c480c6 100644 --- a/src/main.ts +++ b/src/main.ts @@ -15,9 +15,17 @@ async function run(): Promise { const filter = new Filter(filtersYaml) const files = await getChangedFiles(token) - const result = filter.match(files) - for (const key in result) { - core.setOutput(key, String(result[key])) + if (files === null) { + // Change detection was not possible + // Set all filter keys to true (i.e. changed) + for (const key in filter.rules) { + core.setOutput(key, String(true)) + } + } else { + const result = filter.match(files) + for (const key in result) { + core.setOutput(key, String(result[key])) + } } } catch (error) { core.setFailed(error.message) @@ -40,23 +48,45 @@ function getConfigFileContent(configPath: string): string { return fs.readFileSync(configPath, {encoding: 'utf8'}) } -async function getChangedFiles(token: string): Promise { +async function getChangedFiles(token: string): Promise { if (github.context.eventName === 'pull_request') { const pr = github.context.payload.pull_request as Webhooks.WebhookPayloadPullRequestPullRequest return token ? await getChangedFilesFromApi(token, pr) : await getChangedFilesFromGit(pr.base.sha) } else if (github.context.eventName === 'push') { - const push = github.context.payload as Webhooks.WebhookPayloadPush - return await getChangedFilesFromGit(push.before) + return getChangedFilesFromPush() } else { throw new Error('This action can be triggered only by pull_request or push event') } } +async function getChangedFilesFromPush(): Promise { + const push = github.context.payload as Webhooks.WebhookPayloadPush + + // No change detection for pushed tags + if (git.isTagRef(push.ref)) return null + + // Get base from input or use repo default branch. + // It it starts with 'refs/', it will be trimmed (git fetch refs/heads/ doesn't work) + const baseInput = git.trimRefs(core.getInput('base', {required: false}) || push.repository.default_branch) + + // If base references same branch it was pushed to, we will do comparison against the previously pushed commit. + // Otherwise changes are detected against the base reference + const base = git.trimRefsHeads(baseInput) === git.trimRefsHeads(push.ref) ? push.before : baseInput + + // There is no previous commit for comparison + // e.g. change detection against previous commit of just pushed new branch + if (base === git.NULL_SHA) return null + + return await getChangedFilesFromGit(base) +} + // Fetch base branch and use `git diff` to determine changed files -async function getChangedFilesFromGit(sha: string): Promise { +async function getChangedFilesFromGit(ref: string): Promise { core.debug('Fetching base branch and using `git diff-index` to determine changed files') - await git.fetchCommit(sha) - return await git.getChangedFiles(sha) + await git.fetchCommit(ref) + // FETCH_HEAD will always point to the just fetched commit + // No matter if ref is SHA, branch or tag name or full git ref + return await git.getChangedFiles(git.FETCH_HEAD) } // Uses github REST api to get list of files changed in PR