From fc4ef499c23a18e2e6192c9cc835ea2714310824 Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Thu, 21 Nov 2024 23:00:43 +0530 Subject: [PATCH 01/18] adding more granular diff format for autoedits model training --- vscode/src/autoedits/autoedits-provider.ts | 10 +- vscode/src/autoedits/prompt-utils.ts | 9 +- .../context/context-data-logging.ts | 2 +- .../completions/context/context-strategy.ts | 4 +- .../auotedit-short-term-diff.test.ts | 80 ++++++++-------- .../auotedit-short-term-diff.ts | 75 +++++++++++---- ...{recent-edits-diff-strategy.ts => base.ts} | 7 ++ .../recent-edits-diff-helpers/unified-diff.ts | 7 +- .../recent-edits-diff-helpers/utils.ts | 92 ++++++++++++++++++- .../recent-edits-retriever.test.ts | 2 +- .../recent-edits-retriever.ts | 92 ++++++++++++++++++- .../supercompletion-provider.ts | 2 +- 12 files changed, 304 insertions(+), 78 deletions(-) rename vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/{recent-edits-diff-strategy.ts => base.ts} (91%) diff --git a/vscode/src/autoedits/autoedits-provider.ts b/vscode/src/autoedits/autoedits-provider.ts index dd605a47247d..7fc96f062758 100644 --- a/vscode/src/autoedits/autoedits-provider.ts +++ b/vscode/src/autoedits/autoedits-provider.ts @@ -233,7 +233,7 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v document.lineAt(position).range.end ) ) - autoeditsLogger.logDebug('Autocomplete Inline Response: ', autocompleteResponse) + // autoeditsLogger.logDebug('Autocomplete Inline Response: ', autocompleteResponse) return [inlineCompletionItem] } return null @@ -303,10 +303,10 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v suffix: codeToReplaceData.suffixInArea + codeToReplaceData.suffixAfterArea, }) ) { - autoeditsLogger.logDebug( - 'Autoedits', - 'Skipping autoedit - predicted text already exists in suffix' - ) + // autoeditsLogger.logDebug( + // 'Autoedits', + // 'Skipping autoedit - predicted text already exists in suffix' + // ) return } await this.rendererManager.showEdit({ diff --git a/vscode/src/autoedits/prompt-utils.ts b/vscode/src/autoedits/prompt-utils.ts index 4287e2c40c0a..2e35db2a865c 100644 --- a/vscode/src/autoedits/prompt-utils.ts +++ b/vscode/src/autoedits/prompt-utils.ts @@ -140,7 +140,8 @@ ${recentCopyPrompt} ${areaPrompt} ${FINAL_USER_PROMPT} ` - autoeditsLogger.logDebug('AutoEdits', 'Prompt\n', finalPrompt) + // autoeditsLogger.logDebug('AutoEdits', 'Prompt\n', finalPrompt) + autoeditsLogger.logDebug('AutoEdits', 'Diff Values\n', recentEditsPrompt) return { codeToReplace: codeToReplace, prompt: finalPrompt, @@ -333,7 +334,7 @@ export function getRecentEditsPrompt(contextItems: AutocompleteContextSnippet[]) return ps`` } const recentEditsPrompts = recentEdits.map(item => - getContextPromptWithPath( + getContextPromptForDiffPrompt( PromptString.fromDisplayPath(item.uri), PromptString.fromAutocompleteContextSnippet(item).content ) @@ -455,3 +456,7 @@ function getContextItemsForIdentifier( function getContextPromptWithPath(filePath: PromptString, content: PromptString): PromptString { return ps`(\`${filePath}\`)\n\n${content}\n` } + +function getContextPromptForDiffPrompt(filePath: PromptString, content: PromptString): PromptString { + return ps`${filePath}\n${content}` +} diff --git a/vscode/src/completions/context/context-data-logging.ts b/vscode/src/completions/context/context-data-logging.ts index 23cb9119c1cc..7e56fe6a7acb 100644 --- a/vscode/src/completions/context/context-data-logging.ts +++ b/vscode/src/completions/context/context-data-logging.ts @@ -13,7 +13,7 @@ import type { RetrievedContextResults } from './completions-context-ranker' import { JaccardSimilarityRetriever } from './retrievers/jaccard-similarity/jaccard-similarity-retriever' import { DiagnosticsRetriever } from './retrievers/recent-user-actions/diagnostics-retriever' import { RecentCopyRetriever } from './retrievers/recent-user-actions/recent-copy' -import { RecentEditsRetrieverDiffStrategyIdentifier } from './retrievers/recent-user-actions/recent-edits-diff-helpers/recent-edits-diff-strategy' +import { RecentEditsRetrieverDiffStrategyIdentifier } from './retrievers/recent-user-actions/recent-edits-diff-helpers/base' import { RecentEditsRetriever } from './retrievers/recent-user-actions/recent-edits-retriever' import { RecentViewPortRetriever } from './retrievers/recent-user-actions/recent-view-port' import { RetrieverIdentifier } from './utils' diff --git a/vscode/src/completions/context/context-strategy.ts b/vscode/src/completions/context/context-strategy.ts index 67e8c433ec3a..61cbe0dbcdfb 100644 --- a/vscode/src/completions/context/context-strategy.ts +++ b/vscode/src/completions/context/context-strategy.ts @@ -11,7 +11,7 @@ import { JaccardSimilarityRetriever } from './retrievers/jaccard-similarity/jacc import { LspLightRetriever } from './retrievers/lsp-light/lsp-light-retriever' import { DiagnosticsRetriever } from './retrievers/recent-user-actions/diagnostics-retriever' import { RecentCopyRetriever } from './retrievers/recent-user-actions/recent-copy' -import { RecentEditsRetrieverDiffStrategyIdentifier } from './retrievers/recent-user-actions/recent-edits-diff-helpers/recent-edits-diff-strategy' +import { RecentEditsRetrieverDiffStrategyIdentifier } from './retrievers/recent-user-actions/recent-edits-diff-helpers/base' import { RecentEditsRetriever } from './retrievers/recent-user-actions/recent-edits-retriever' import { RecentViewPortRetriever } from './retrievers/recent-user-actions/recent-view-port' import { loadTscRetriever } from './retrievers/tsc/load-tsc-retriever' @@ -128,7 +128,7 @@ export class DefaultContextStrategyFactory implements ContextStrategyFactory { new RecentEditsRetriever({ maxAgeMs: 10 * 60 * 1000, diffStrategyIdentifier: - RecentEditsRetrieverDiffStrategyIdentifier.UnifiedDiffWithLineNumbers, + RecentEditsRetrieverDiffStrategyIdentifier.AutoeditWithShortTermDiff, }), new DiagnosticsRetriever({ contextLines: 0, diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.test.ts index 1ab858071d14..0e23f6a26aec 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.test.ts @@ -1,45 +1,45 @@ -import { describe, expect, it } from 'vitest' -import { Uri } from 'vscode' -import { range } from '../../../../../testutils/textDocument' -import { AutoeditWithShortTermDiffStrategy } from './auotedit-short-term-diff' -import type { TextDocumentChange } from './recent-edits-diff-strategy' +// import { describe, expect, it } from 'vitest' +// import { Uri } from 'vscode' +// import { range } from '../../../../../testutils/textDocument' +// import { AutoeditWithShortTermDiffStrategy } from './auotedit-short-term-diff' +// import type { TextDocumentChange } from './base' -describe('AutoeditWithShortTermDiffStrategy', () => { - const strategy = new AutoeditWithShortTermDiffStrategy() - const mockUri = Uri.parse('file:///test.txt') +// describe('AutoeditWithShortTermDiffStrategy', () => { +// const strategy = new AutoeditWithShortTermDiffStrategy() +// const mockUri = Uri.parse('file:///test.txt') - const createChange = (timestamp: number, oldText: string, text: string) => ({ - timestamp, - change: { - range: range(0, 0, 0, 0), - text, - rangeLength: oldText.length, - rangeOffset: 0, - }, - }) +// const createChange = (timestamp: number, oldText: string, text: string) => ({ +// timestamp, +// change: { +// range: range(0, 0, 0, 0), +// text, +// rangeLength: oldText.length, +// rangeOffset: 0, +// }, +// }) - it('should divide changes into short-term and long-term windows', () => { - const now = Date.now() - const initialContent = 'initial content' - const changes: TextDocumentChange[] = [ - createChange(now - 10000, initialContent, 'change 1'), - createChange(now - 2000, 'change 1', 'change 2'), - ] +// it('should divide changes into short-term and long-term windows', () => { +// const now = Date.now() +// const initialContent = 'initial content' +// const changes: TextDocumentChange[] = [ +// createChange(now - 10000, initialContent, 'change 1'), +// createChange(now - 2000, 'change 1', 'change 2'), +// ] - const hunks = strategy.getDiffHunks({ - uri: mockUri, - oldContent: initialContent, - changes, - }) +// const hunks = strategy.getDiffHunks({ +// uri: mockUri, +// oldContent: initialContent, +// changes, +// }) - expect(hunks).toHaveLength(2) - expect(hunks[0].diff.toString()).toMatchInlineSnapshot(` - "1-| initial content - 1+| change 1" - `) - expect(hunks[1].diff.toString()).toMatchInlineSnapshot(` - "1-| change 1 - 1+| change 2" - `) - }) -}) +// expect(hunks).toHaveLength(2) +// expect(hunks[0].diff.toString()).toMatchInlineSnapshot(` +// "1-| initial content +// 1+| change 1" +// `) +// expect(hunks[1].diff.toString()).toMatchInlineSnapshot(` +// "1-| change 1 +// 1+| change 2" +// `) +// }) +// }) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts index 49190044f92e..9fdb756458ba 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts @@ -1,11 +1,17 @@ +import path from 'node:path' import type * as vscode from 'vscode' import type { DiffCalculationInput, DiffHunk, RecentEditsRetrieverDiffStrategy, TextDocumentChange, -} from './recent-edits-diff-strategy' -import { applyTextDocumentChanges, computeDiffWithLineNumbers } from './utils' +} from './base' +import { + applyTextDocumentChanges, + computeDiffWithLineNumbers, + groupChangesForSimilarLinesTogether, +} from './utils' +import type { GroupedTextDocumentChange } from './utils' /** * Generates a single unified diff patch that combines all changes @@ -17,20 +23,54 @@ export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDi private shortTermContextLines = 0 public getDiffHunks(input: DiffCalculationInput): DiffHunk[] { - const [shortTermChanges, longTermChanges] = this.divideChangesIntoWindows(input.changes) - const [shortTermHunks, shortTermNewContent] = this.getDiffHunksForChanges( - input.uri, - input.oldContent, - shortTermChanges, - this.shortTermContextLines - ) - const [longTermHunks, _] = this.getDiffHunksForChanges( - input.uri, - shortTermNewContent, - longTermChanges, - this.longTermContextLines - ) - return [shortTermHunks, longTermHunks] + const changes = groupChangesForSimilarLinesTogether(input.changes) + this.logGroupedChanges(input.uri, input.oldContent, changes) + const allDiffHunks: DiffHunk[] = [] + + let oldContent = input.oldContent + for (const changeList of changes) { + const [diffHunk, newContent] = this.getDiffHunksForChanges( + input.uri, + oldContent, + changeList.changes, + this.shortTermContextLines + ) + oldContent = newContent + allDiffHunks.push(diffHunk) + } + return allDiffHunks + + // const [shortTermChanges, longTermChanges] = this.divideChangesIntoWindows(input.changes) + // const [shortTermHunks, shortTermNewContent] = this.getDiffHunksForChanges( + // input.uri, + // input.oldContent, + // shortTermChanges, + // this.shortTermContextLines + // ) + // const [longTermHunks, _] = this.getDiffHunksForChanges( + // input.uri, + // shortTermNewContent, + // longTermChanges, + // this.longTermContextLines + // ) + // return [shortTermHunks, longTermHunks] + } + + private logGroupedChanges( + uri: vscode.Uri, + oldContent: string, + changes: GroupedTextDocumentChange[] + ) { + const fileName = uri.fsPath.split('/').pop()?.split('.')[0] || 'document' + const logPath = uri.fsPath.replace(/[^/\\]+$/, `${fileName}_grouped.json`) + const finalLogPath = path.join('/Users/hiteshsagtani/Desktop/diff-logs', path.basename(logPath)) + const fs = require('fs') + const logData = { + uri: uri.toString(), + oldContent: oldContent, + changes: changes.map(c => c.changes), + } + fs.writeFileSync(finalLogPath, JSON.stringify(logData, null, 2)) } private getDiffHunksForChanges( @@ -45,8 +85,9 @@ export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDi ) const gitDiff = computeDiffWithLineNumbers(uri, oldContent, newContent, numContextLines) const diffHunk = { - diff: gitDiff, + uri, latestEditTimestamp: Math.max(...changes.map(c => c.timestamp)), + diff: gitDiff, } return [diffHunk, newContent] } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/recent-edits-diff-strategy.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts similarity index 91% rename from vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/recent-edits-diff-strategy.ts rename to vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts index fc3d70c92a96..e258bd9e0189 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/recent-edits-diff-strategy.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts @@ -47,6 +47,12 @@ export interface RecentEditsRetrieverDiffStrategy { export interface TextDocumentChange { timestamp: number + oldCursorPosition: vscode.Position + newCursorPosition: vscode.Position + oldContent: string + newContent: string + replacedRange: vscode.Range + insertedRange: vscode.Range change: vscode.TextDocumentContentChangeEvent } @@ -57,6 +63,7 @@ export interface DiffCalculationInput { } export interface DiffHunk { + uri: vscode.Uri latestEditTimestamp: number diff: PromptString } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts index eb57e638f9bd..2bd3de7c2d0e 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts @@ -1,9 +1,5 @@ import { PromptString } from '@sourcegraph/cody-shared' -import type { - DiffCalculationInput, - DiffHunk, - RecentEditsRetrieverDiffStrategy, -} from './recent-edits-diff-strategy' +import type { DiffCalculationInput, DiffHunk, RecentEditsRetrieverDiffStrategy } from './base' import { applyTextDocumentChanges, computeDiffWithLineNumbers } from './utils' interface UnifiedDiffStrategyOptions { @@ -30,6 +26,7 @@ export class UnifiedDiffStrategy implements RecentEditsRetrieverDiffStrategy { const diff = this.getDiffForUnifiedStrategy(input, newContent) return [ { + uri: input.uri, diff, latestEditTimestamp: Math.max(...input.changes.map(c => c.timestamp)), }, diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts index d0f211b0f0f7..a714d41cfe85 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts @@ -1,7 +1,51 @@ -import { PromptString } from '@sourcegraph/cody-shared' +import { PromptString, ps } from '@sourcegraph/cody-shared' import { displayPath } from '@sourcegraph/cody-shared/src/editor/displayPath' import { structuredPatch } from 'diff' import type * as vscode from 'vscode' +import type { DiffHunk, TextDocumentChange } from './base' + +export interface GroupedTextDocumentChange { + changes: TextDocumentChange[] +} + +export function groupChangesForSimilarLinesTogether( + changes: TextDocumentChange[] +): GroupedTextDocumentChange[] { + if (changes.length === 0) { + return [] + } + const groupedChanges: GroupedTextDocumentChange[] = [] + let currentGroup: GroupedTextDocumentChange = { + changes: [changes[0]], + } + for (let i = 1; i < changes.length; i++) { + const change = changes[i] + const lastChange = currentGroup.changes[currentGroup.changes.length - 1] + if (shouldCombineChanges(lastChange, change)) { + currentGroup.changes.push(change) + } else { + groupedChanges.push(currentGroup) + currentGroup = { + changes: [change], + } + } + } + if (currentGroup.changes.length > 0) { + groupedChanges.push(currentGroup) + } + return groupedChanges +} + +function shouldCombineChanges(lastChange: TextDocumentChange, change: TextDocumentChange): boolean { + return ( + doesLinesOverlap(lastChange.replacedRange, change.change.range) || + doesLinesOverlap(lastChange.insertedRange, change.change.range) + ) +} + +function doesLinesOverlap(a: vscode.Range, b: vscode.Range): boolean { + return a.start.line <= b.end.line && a.end.line >= b.start.line +} export function computeDiffWithLineNumbers( uri: vscode.Uri, @@ -28,6 +72,41 @@ export function computeDiffWithLineNumbers( return gitDiff } +export function combineDiffHunksFromSimilarFile(hunks: DiffHunk[]): DiffHunk[] { + const combinedHunks: DiffHunk[] = [] + let currentHunkList: DiffHunk[] = [hunks[0]] + for (const hunk of hunks) { + const lastHunk = currentHunkList[currentHunkList.length - 1] + if (shouldCombineHunks(hunk, lastHunk)) { + currentHunkList.push(hunk) + } else { + combinedHunks.push(combineMultipleHunks(currentHunkList)) + currentHunkList = [hunk] + } + } + if (currentHunkList.length > 0) { + combinedHunks.push(combineMultipleHunks(currentHunkList)) + } + return combinedHunks +} + +function combineMultipleHunks(hunks: DiffHunk[]): DiffHunk { + const lastestTime = Math.max(...hunks.map(h => h.latestEditTimestamp)) + const diffs = PromptString.join( + hunks.map(h => h.diff), + ps`\nthen\n` + ) + return { + uri: hunks[0].uri, + latestEditTimestamp: lastestTime, + diff: diffs, + } +} + +function shouldCombineHunks(hunk1: DiffHunk, hunk2: DiffHunk): boolean { + return hunk1.uri.toString() === hunk2.uri.toString() +} + export function getDiffStringForHunkWithLineNumbers(hunk: Diff.Hunk): string { const lines = [] let oldLineNumber = hunk.oldStart @@ -63,3 +142,14 @@ export function applyTextDocumentChanges( } return content } + +export function getNewContentAfterApplyingRange( + oldContent: string, + change: vscode.TextDocumentContentChangeEvent +): string { + return ( + oldContent.slice(0, change.rangeOffset) + + change.text + + oldContent.slice(change.rangeOffset + change.rangeLength) + ) +} diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts index 729582870e2b..ea96715e749b 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts @@ -4,7 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type * as vscode from 'vscode' import { range } from '../../../../testutils/textDocument' import { document } from '../../../test-helpers' -import { RecentEditsRetrieverDiffStrategyIdentifier } from './recent-edits-diff-helpers/recent-edits-diff-strategy' +import { RecentEditsRetrieverDiffStrategyIdentifier } from './recent-edits-diff-helpers/base' import { RecentEditsRetriever } from './recent-edits-retriever' const FIVE_MINUTES = 5 * 60 * 1000 diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts index b5d1d97fa3be..e18561498630 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts @@ -1,3 +1,4 @@ +import path from 'node:path' import { type PromptString, contextFiltersProvider } from '@sourcegraph/cody-shared' import type { AutocompleteContextSnippet } from '@sourcegraph/cody-shared' import * as vscode from 'vscode' @@ -9,8 +10,11 @@ import { type RecentEditsRetrieverDiffStrategyIdentifier, type TextDocumentChange, createDiffStrategy, -} from './recent-edits-diff-helpers/recent-edits-diff-strategy' -import { applyTextDocumentChanges } from './recent-edits-diff-helpers/utils' +} from './recent-edits-diff-helpers/base' +import { + applyTextDocumentChanges, + getNewContentAfterApplyingRange, +} from './recent-edits-diff-helpers/utils' interface TrackedDocument { content: string @@ -159,14 +163,41 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever } const now = Date.now() + const oldCursorPosition = event.contentChanges[0].range.end for (const change of event.contentChanges) { + const oldContent = + trackedDocument.changes.length > 0 + ? trackedDocument.changes[trackedDocument.changes.length - 1].newContent + : trackedDocument.content + const newCursorPosition = calculateNewCursorPositions(change, oldCursorPosition) + const newContent = getNewContentAfterApplyingRange(oldContent, change) + const insertedRange = calculateInsertedRangeInDocumentBasedOnChange( + oldContent, + newContent, + change + ) + trackedDocument.changes.push({ timestamp: now, + oldCursorPosition, + newCursorPosition, + oldContent, + newContent, + replacedRange: change.range, + insertedRange, change, }) } - this.reconcileOutdatedChanges() + this.logTextDocument(trackedDocument) + } + + private logTextDocument(trackedDocument: TrackedDocument): void { + const fileName = trackedDocument.uri.fsPath.split('/').pop()?.split('.')[0] || 'document' + const logPath = trackedDocument.uri.fsPath.replace(/[^/\\]+$/, `${fileName}.json`) + const finalLogPath = path.join('/Users/hiteshsagtani/Desktop/diff-logs', path.basename(logPath)) + const fs = require('fs') + fs.writeFileSync(finalLogPath, JSON.stringify(trackedDocument, null, 2)) } private onDidOpenTextDocument(document: vscode.TextDocument): void { @@ -227,3 +258,58 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever } } } + +export function calculateInsertedRangeInDocumentBasedOnChange( + oldContent: string, + newContent: string, + change: vscode.TextDocumentContentChangeEvent +): vscode.Range { + // Function calculates the updated range for the new document based on the change. + const startOffset = change.rangeOffset + const endOffset = startOffset + change.text.length + + const startPosition = getPositionAt(newContent, startOffset) + const endPosition = getPositionAt(newContent, endOffset) + + return new vscode.Range(startPosition, endPosition) +} + +// Helper function to convert an offset to a Position (line and character) +function getPositionAt(content: string, offset: number): vscode.Position { + let line = 0 + let character = 0 + let i = 0 + + while (i < offset) { + if (content[i] === '\n') { + line++ + character = 0 + } else { + character++ + } + i++ + } + + return new vscode.Position(line, character) +} + +export function calculateNewCursorPositions( + change: vscode.TextDocumentContentChangeEvent, + oldCursorPosition: vscode.Position +): vscode.Position { + // Starting position of the change + const start = change.range.start + + // Inserted text and its lines + const insertedText = change.text + const insertedLines = insertedText.split('\n') + let newCursorPosition: vscode.Position + if (insertedLines.length === 1) { + newCursorPosition = new vscode.Position(start.line, start.character + insertedText.length) + } else { + const newLineCount = insertedLines.length - 1 + const lastLineLength = insertedLines[insertedLines.length - 1].length + newCursorPosition = new vscode.Position(start.line + newLineCount, lastLineLength) + } + return newCursorPosition +} diff --git a/vscode/src/supercompletions/supercompletion-provider.ts b/vscode/src/supercompletions/supercompletion-provider.ts index 988502ba14bc..01008c030271 100644 --- a/vscode/src/supercompletions/supercompletion-provider.ts +++ b/vscode/src/supercompletions/supercompletion-provider.ts @@ -1,6 +1,6 @@ import type { ChatClient } from '@sourcegraph/cody-shared' import * as vscode from 'vscode' -import { RecentEditsRetrieverDiffStrategyIdentifier } from '../completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/recent-edits-diff-strategy' +import { RecentEditsRetrieverDiffStrategyIdentifier } from '../completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base' import { RecentEditsRetriever } from '../completions/context/retrievers/recent-user-actions/recent-edits-retriever' import type { CodyStatusBar } from '../services/StatusBar' import { type Supercompletion, getSupercompletions } from './get-supercompletion' From 7b9ccfa11d7f7b8d6b9d8b24eb5842626d806f8e Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Fri, 22 Nov 2024 00:09:28 +0530 Subject: [PATCH 02/18] temp changes --- .../recent-edits-diff-helpers/auotedit-short-term-diff.ts | 5 +++-- .../recent-user-actions/recent-edits-diff-helpers/base.ts | 1 + .../recent-edits-diff-helpers/utils.ts | 8 +++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts index 9fdb756458ba..a96d80ff45e2 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts @@ -11,7 +11,7 @@ import { computeDiffWithLineNumbers, groupChangesForSimilarLinesTogether, } from './utils' -import type { GroupedTextDocumentChange } from './utils' +import { type GroupedTextDocumentChange, combineDiffHunksFromSimilarFile } from './utils' /** * Generates a single unified diff patch that combines all changes @@ -38,7 +38,7 @@ export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDi oldContent = newContent allDiffHunks.push(diffHunk) } - return allDiffHunks + return combineDiffHunksFromSimilarFile(allDiffHunks) // const [shortTermChanges, longTermChanges] = this.divideChangesIntoWindows(input.changes) // const [shortTermHunks, shortTermNewContent] = this.getDiffHunksForChanges( @@ -86,6 +86,7 @@ export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDi const gitDiff = computeDiffWithLineNumbers(uri, oldContent, newContent, numContextLines) const diffHunk = { uri, + leastEditTimestamp: Math.min(...changes.map(c => c.timestamp)), latestEditTimestamp: Math.max(...changes.map(c => c.timestamp)), diff: gitDiff, } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts index e258bd9e0189..b8b0a56bb9bc 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts @@ -66,4 +66,5 @@ export interface DiffHunk { uri: vscode.Uri latestEditTimestamp: number diff: PromptString + leastEditTimestamp: number } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts index a714d41cfe85..b6c71a79b4ee 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts @@ -73,9 +73,13 @@ export function computeDiffWithLineNumbers( } export function combineDiffHunksFromSimilarFile(hunks: DiffHunk[]): DiffHunk[] { + if (hunks.length === 0) { + return [] + } const combinedHunks: DiffHunk[] = [] let currentHunkList: DiffHunk[] = [hunks[0]] - for (const hunk of hunks) { + for (let i = 1; i < hunks.length; i++) { + const hunk = hunks[i] const lastHunk = currentHunkList[currentHunkList.length - 1] if (shouldCombineHunks(hunk, lastHunk)) { currentHunkList.push(hunk) @@ -92,11 +96,13 @@ export function combineDiffHunksFromSimilarFile(hunks: DiffHunk[]): DiffHunk[] { function combineMultipleHunks(hunks: DiffHunk[]): DiffHunk { const lastestTime = Math.max(...hunks.map(h => h.latestEditTimestamp)) + const leastTime = Math.min(...hunks.map(h => h.leastEditTimestamp)) const diffs = PromptString.join( hunks.map(h => h.diff), ps`\nthen\n` ) return { + leastEditTimestamp: leastTime, uri: hunks[0].uri, latestEditTimestamp: lastestTime, diff: diffs, From 0e8c2c3ee8a530c1f8dd507be7634f76afff455b Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Fri, 22 Nov 2024 03:52:27 +0530 Subject: [PATCH 03/18] some change --- .../auotedit-short-term-diff.ts | 5 +- .../recent-edits-diff-helpers/base.ts | 1 - .../recent-edits-diff-helpers/utils.ts | 86 ++++++++++++++++--- .../recent-edits-retriever.ts | 5 +- 4 files changed, 80 insertions(+), 17 deletions(-) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts index a96d80ff45e2..1d7a7bfdf2ff 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts @@ -11,7 +11,7 @@ import { computeDiffWithLineNumbers, groupChangesForSimilarLinesTogether, } from './utils' -import { type GroupedTextDocumentChange, combineDiffHunksFromSimilarFile } from './utils' +import {GroupedTextDocumentChange, combineDiffHunksFromSimilarFile, combineNonOverlappingLinesSchemaTogether} from './utils'; /** * Generates a single unified diff patch that combines all changes @@ -23,7 +23,8 @@ export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDi private shortTermContextLines = 0 public getDiffHunks(input: DiffCalculationInput): DiffHunk[] { - const changes = groupChangesForSimilarLinesTogether(input.changes) + const rawChanges = groupChangesForSimilarLinesTogether(input.changes) + const changes = combineNonOverlappingLinesSchemaTogether(rawChanges) this.logGroupedChanges(input.uri, input.oldContent, changes) const allDiffHunks: DiffHunk[] = [] diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts index b8b0a56bb9bc..e258bd9e0189 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts @@ -66,5 +66,4 @@ export interface DiffHunk { uri: vscode.Uri latestEditTimestamp: number diff: PromptString - leastEditTimestamp: number } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts index b6c71a79b4ee..7fd553a01adb 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts @@ -6,8 +6,44 @@ import type { DiffHunk, TextDocumentChange } from './base' export interface GroupedTextDocumentChange { changes: TextDocumentChange[] + changeStartLine: number + changeEndLine: number } +export function combineNonOverlappingLinesSchemaTogether(groupedChanges: GroupedTextDocumentChange[]): GroupedTextDocumentChange[] { + const combinedChanges: GroupedTextDocumentChange[] = [] + if (groupedChanges.length === 0) { + return combinedChanges + } + let currentActiveChanges: GroupedTextDocumentChange[] = [groupedChanges[0]] + for (let i = 1; i < groupedChanges.length; i++) { + const groupedChange = groupedChanges[i] + if (shouldCombineGroupedChanges(currentActiveChanges[currentActiveChanges.length - 1], groupedChange)) { + currentActiveChanges.push(groupedChange) + } else { + combinedChanges.push(flatCombinedGroupedChanges(currentActiveChanges)) + currentActiveChanges = [groupedChange] + } + } + if (currentActiveChanges.length > 0) { + combinedChanges.push(flatCombinedGroupedChanges(currentActiveChanges)) + } + return combinedChanges +} + +function flatCombinedGroupedChanges(changes: GroupedTextDocumentChange[]): GroupedTextDocumentChange { + return { + changes: changes.flatMap(change => change.changes), + changeStartLine: Math.min(...changes.map(change => change.changeStartLine)), + changeEndLine: Math.max(...changes.map(change => change.changeEndLine)) + } +} + +function shouldCombineGroupedChanges(a: GroupedTextDocumentChange, b: GroupedTextDocumentChange): boolean { + return !(a.changeStartLine <= b.changeEndLine && a.changeEndLine >= b.changeStartLine) +} + + export function groupChangesForSimilarLinesTogether( changes: TextDocumentChange[] ): GroupedTextDocumentChange[] { @@ -15,27 +51,53 @@ export function groupChangesForSimilarLinesTogether( return [] } const groupedChanges: GroupedTextDocumentChange[] = [] - let currentGroup: GroupedTextDocumentChange = { - changes: [changes[0]], - } + let currentGroup: TextDocumentChange[] = [changes[0]] for (let i = 1; i < changes.length; i++) { const change = changes[i] - const lastChange = currentGroup.changes[currentGroup.changes.length - 1] + const lastChange = currentGroup[currentGroup.length - 1] if (shouldCombineChanges(lastChange, change)) { - currentGroup.changes.push(change) + currentGroup.push(change) } else { - groupedChanges.push(currentGroup) - currentGroup = { - changes: [change], - } + const range = getRangeValues(currentGroup) + groupedChanges.push({ + changes: currentGroup, + changeStartLine: range[0], + changeEndLine: range[1], + }) + currentGroup = [change] } } - if (currentGroup.changes.length > 0) { - groupedChanges.push(currentGroup) + if (currentGroup.length > 0) { + const range = getRangeValues(currentGroup) + groupedChanges.push({ + changes: currentGroup, + changeStartLine: range[0], + changeEndLine: range[1], + }) } return groupedChanges } + +function getRangeValues(documentChanges: TextDocumentChange[]): [number, number] { + let minRange = getMinRange(documentChanges[0].replacedRange, documentChanges[0].insertedRange) + let maxRange = getMaxRange(documentChanges[0].replacedRange, documentChanges[0].insertedRange) + for (let i = 1; i < documentChanges.length; i++) { + const change = documentChanges[i] + minRange = getMinRange(getMinRange(change.replacedRange, change.insertedRange), minRange) + maxRange = getMaxRange(getMaxRange(change.replacedRange, change.insertedRange), maxRange) + } + return [minRange.start.line, maxRange.end.line] +} + +function getMinRange(a: vscode.Range, b: vscode.Range): vscode.Range { + return a.start.isBeforeOrEqual(b.start) ? a : b +} + +function getMaxRange(a: vscode.Range, b: vscode.Range): vscode.Range { + return a.end.isBeforeOrEqual(b.end) ? b : a +} + function shouldCombineChanges(lastChange: TextDocumentChange, change: TextDocumentChange): boolean { return ( doesLinesOverlap(lastChange.replacedRange, change.change.range) || @@ -96,13 +158,11 @@ export function combineDiffHunksFromSimilarFile(hunks: DiffHunk[]): DiffHunk[] { function combineMultipleHunks(hunks: DiffHunk[]): DiffHunk { const lastestTime = Math.max(...hunks.map(h => h.latestEditTimestamp)) - const leastTime = Math.min(...hunks.map(h => h.leastEditTimestamp)) const diffs = PromptString.join( hunks.map(h => h.diff), ps`\nthen\n` ) return { - leastEditTimestamp: leastTime, uri: hunks[0].uri, latestEditTimestamp: lastestTime, diff: diffs, diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts index e18561498630..544f14964157 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts @@ -98,8 +98,11 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever const diffs: DiffAcrossDocuments[] = [] const diffPromises = Array.from(this.trackedDocuments.entries()).map( async ([uri, trackedDocument]) => { + if (trackedDocument.changes.length===0) { + return null + } const diffHunks = await this.getDiff(vscode.Uri.parse(uri)) - if (diffHunks && trackedDocument.changes.length > 0) { + if (diffHunks) { return diffHunks.map(diffHunk => ({ diff: diffHunk.diff, uri: trackedDocument.uri, From 9979e4cca51739d1d3b1665245b289f18a7eef9e Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Sat, 23 Nov 2024 00:48:27 +0530 Subject: [PATCH 04/18] checkpoint --- .../auotedit-short-term-diff.ts | 38 +-- .../recent-edits-diff-helpers/base.ts | 8 +- .../recent-edits-diff-helpers/utils-new.ts | 76 +++++ .../recent-edits-diff-helpers/utils.test.ts | 272 +++++++++++++++++ .../recent-edits-diff-helpers/utils.ts | 274 +++++++++--------- .../recent-edits-retriever.ts | 85 +----- 6 files changed, 496 insertions(+), 257 deletions(-) create mode 100644 vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils-new.ts create mode 100644 vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts index 1d7a7bfdf2ff..326127b1202c 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts @@ -11,15 +11,13 @@ import { computeDiffWithLineNumbers, groupChangesForSimilarLinesTogether, } from './utils' -import {GroupedTextDocumentChange, combineDiffHunksFromSimilarFile, combineNonOverlappingLinesSchemaTogether} from './utils'; +import { type TextDocumentChangeGroup, combineNonOverlappingLinesSchemaTogether } from './utils' /** * Generates a single unified diff patch that combines all changes * made to a document into one consolidated view. */ export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDiffStrategy { - private shortTermDiffWindowMs = 5 * 1000 // 5 seconds - private longTermContextLines = 3 private shortTermContextLines = 0 public getDiffHunks(input: DiffCalculationInput): DiffHunk[] { @@ -39,29 +37,10 @@ export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDi oldContent = newContent allDiffHunks.push(diffHunk) } - return combineDiffHunksFromSimilarFile(allDiffHunks) - - // const [shortTermChanges, longTermChanges] = this.divideChangesIntoWindows(input.changes) - // const [shortTermHunks, shortTermNewContent] = this.getDiffHunksForChanges( - // input.uri, - // input.oldContent, - // shortTermChanges, - // this.shortTermContextLines - // ) - // const [longTermHunks, _] = this.getDiffHunksForChanges( - // input.uri, - // shortTermNewContent, - // longTermChanges, - // this.longTermContextLines - // ) - // return [shortTermHunks, longTermHunks] + return allDiffHunks } - private logGroupedChanges( - uri: vscode.Uri, - oldContent: string, - changes: GroupedTextDocumentChange[] - ) { + private logGroupedChanges(uri: vscode.Uri, oldContent: string, changes: TextDocumentChangeGroup[]) { const fileName = uri.fsPath.split('/').pop()?.split('.')[0] || 'document' const logPath = uri.fsPath.replace(/[^/\\]+$/, `${fileName}_grouped.json`) const finalLogPath = path.join('/Users/hiteshsagtani/Desktop/diff-logs', path.basename(logPath)) @@ -93,15 +72,4 @@ export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDi } return [diffHunk, newContent] } - - private divideChangesIntoWindows( - changes: TextDocumentChange[] - ): [TextDocumentChange[], TextDocumentChange[]] { - // Divide the changes into 2 different windows, where the second window is the short term changes under 5 seconds - const now = Date.now() - const index = changes.findIndex(c => now - c.timestamp < this.shortTermDiffWindowMs) - const shortTermChanges = changes.slice(0, index) - const longTermChanges = changes.slice(index) - return [shortTermChanges, longTermChanges] - } } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts index e258bd9e0189..f29b755bbfcd 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts @@ -47,13 +47,9 @@ export interface RecentEditsRetrieverDiffStrategy { export interface TextDocumentChange { timestamp: number - oldCursorPosition: vscode.Position - newCursorPosition: vscode.Position - oldContent: string - newContent: string - replacedRange: vscode.Range - insertedRange: vscode.Range change: vscode.TextDocumentContentChangeEvent + // The range in the document where the text was inserted. + insertedRange: vscode.Range } export interface DiffCalculationInput { diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils-new.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils-new.ts new file mode 100644 index 000000000000..d80ae824e205 --- /dev/null +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils-new.ts @@ -0,0 +1,76 @@ +import * as vscode from 'vscode'; +import {PromptString} from '@sourcegraph/cody-shared'; +import {displayPath} from '@sourcegraph/cody-shared/src/editor/displayPath'; +import {structuredPatch} from 'diff'; + +export function computeDiffWithLineNumbers( + uri: vscode.Uri, + originalContent: string, + modifiedContent: string, + numContextLines: number +): PromptString { + const hunkDiffs = [] + const filename = displayPath(uri) + const patch = structuredPatch( + `a/${filename}`, + `b/${filename}`, + originalContent, + modifiedContent, + '', + '', + { context: numContextLines } + ) + for (const hunk of patch.hunks) { + const diffString = getDiffStringForHunkWithLineNumbers(hunk) + hunkDiffs.push(diffString) + } + const gitDiff = PromptString.fromStructuredGitDiff(uri, hunkDiffs.join('\nthen\n')) + return gitDiff +} + +export function getDiffStringForHunkWithLineNumbers(hunk: Diff.Hunk): string { + const lines = [] + let oldLineNumber = hunk.oldStart + let newLineNumber = hunk.newStart + for (const line of hunk.lines) { + if (line.length === 0) { + continue + } + if (line[0] === '-') { + lines.push(`${oldLineNumber}${line[0]}| ${line.slice(1)}`) + oldLineNumber++ + } else if (line[0] === '+') { + lines.push(`${newLineNumber}${line[0]}| ${line.slice(1)}`) + newLineNumber++ + } else if (line[0] === ' ') { + lines.push(`${newLineNumber}${line[0]}| ${line.slice(1)}`) + oldLineNumber++ + newLineNumber++ + } + } + return lines.join('\n') +} + +export function applyTextDocumentChanges( + content: string, + changes: vscode.TextDocumentContentChangeEvent[] +): string { + for (const change of changes) { + content = + content.slice(0, change.rangeOffset) + + change.text + + content.slice(change.rangeOffset + change.rangeLength) + } + return content +} + +export function getNewContentAfterApplyingRange( + oldContent: string, + change: vscode.TextDocumentContentChangeEvent +): string { + return ( + oldContent.slice(0, change.rangeOffset) + + change.text + + oldContent.slice(change.rangeOffset + change.rangeLength) + ) +} diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts new file mode 100644 index 000000000000..87f93590cc46 --- /dev/null +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts @@ -0,0 +1,272 @@ +import { PromptString } from '@sourcegraph/cody-shared' +import { describe, expect, it } from 'vitest' +import type * as vscode from 'vscode' +import { + applyTextDocumentChanges, + computeDiffWithLineNumbers, + doesLinesOverlapForRanges, + groupConsecutiveItemsByPredicate, +} from './utils' +import dedent from 'dedent' + +describe('applyTextDocumentChanges', () => { + const createChange = ( + offset: number, + length: number, + text: string + ): vscode.TextDocumentContentChangeEvent => + ({ + rangeOffset: offset, + rangeLength: length, + text, + }) as vscode.TextDocumentContentChangeEvent + + it('should insert text at the beginning', () => { + const content = 'world' + const changes = [createChange(0, 0, 'Hello ')] + expect(applyTextDocumentChanges(content, changes)).toBe('Hello world') + }) + + it('should insert text in the middle', () => { + const content = 'Hello world' + const changes = [createChange(5, 0, ' beautiful')] + expect(applyTextDocumentChanges(content, changes)).toBe('Hello beautiful world') + }) + + it('should replace text', () => { + const content = 'Hello world' + const changes = [createChange(6, 5, 'universe')] + expect(applyTextDocumentChanges(content, changes)).toBe('Hello universe') + }) + + it('should handle multiple changes in sequence', () => { + const content = 'Hello world' + const changes = [createChange(0, 5, 'Hi'), createChange(3, 5, 'everyone')] + expect(applyTextDocumentChanges(content, changes)).toBe('Hi everyone') + }) + + it('should handle deletion', () => { + const content = 'Hello beautiful world' + const changes = [createChange(5, 10, '')] + expect(applyTextDocumentChanges(content, changes)).toBe('Hello world') + }) + + it('should handle empty changes array', () => { + const content = 'Hello world' + const changes: vscode.TextDocumentContentChangeEvent[] = [] + expect(applyTextDocumentChanges(content, changes)).toBe('Hello world') + }) + + it('should handle empty content', () => { + const content = '' + const changes = [createChange(0, 0, 'Hello')] + expect(applyTextDocumentChanges(content, changes)).toBe('Hello') + }) +}) + +describe('doesLinesOverlapForRanges', () => { + const createRange = (startLine: number, endLine: number): vscode.Range => + ({ + start: { line: startLine, character: 0 }, + end: { line: endLine, character: 0 }, + }) as vscode.Range + + it('should detect overlapping ranges', () => { + const rangeA = createRange(1, 5) + const rangeB = createRange(3, 7) + expect(doesLinesOverlapForRanges(rangeA, rangeB)).toBe(true) + }) + + it('should detect adjacent non-overlapping ranges', () => { + const rangeA = createRange(1, 3) + const rangeB = createRange(4, 6) + expect(doesLinesOverlapForRanges(rangeA, rangeB)).toBe(false) + }) + + it('should detect contained ranges', () => { + const rangeA = createRange(1, 10) + const rangeB = createRange(3, 5) + expect(doesLinesOverlapForRanges(rangeA, rangeB)).toBe(true) + }) + + it('should detect ranges touching at endpoints', () => { + const rangeA = createRange(1, 3) + const rangeB = createRange(3, 5) + expect(doesLinesOverlapForRanges(rangeA, rangeB)).toBe(true) + }) +}) + +describe('groupConsecutiveItemsByPredicate', () => { + it('should return empty array when given an empty array', () => { + const result = groupConsecutiveItemsByPredicate([], (a, b) => a === b) + expect(result).toEqual([]) + }) + + it('should group all items together when predicate is always true', () => { + const items = [1, 2, 3, 4] + const result = groupConsecutiveItemsByPredicate(items, () => true) + expect(result).toEqual([[1, 2, 3, 4]]) + }) + + it('should not group any items when predicate is always false', () => { + const items = [1, 2, 3, 4] + const result = groupConsecutiveItemsByPredicate(items, () => false) + expect(result).toEqual([[1], [2], [3], [4]]) + }) + + it('should group consecutive identical items', () => { + const items = [1, 1, 2, 2, 2, 3, 1, 1] + const result = groupConsecutiveItemsByPredicate(items, (a, b) => a === b) + expect(result).toEqual([[1, 1], [2, 2, 2], [3], [1, 1]]) + }) + + it('should group consecutive items based on a custom predicate (even numbers)', () => { + const items = [1, 2, 4, 3, 6, 8, 7] + const result = groupConsecutiveItemsByPredicate(items, (a, b) => a % 2 === 0 && b % 2 === 0) + expect(result).toEqual([[1], [2, 4], [3], [6, 8], [7]]) + }) + + it('should correctly group items with complex objects', () => { + const items = [ + { type: 'A', value: 1 }, + { type: 'A', value: 2 }, + { type: 'B', value: 3 }, + { type: 'B', value: 4 }, + { type: 'A', value: 5 }, + ] + const result = groupConsecutiveItemsByPredicate(items, (a, b) => a.type === b.type) + expect(result).toEqual([ + [ + { type: 'A', value: 1 }, + { type: 'A', value: 2 }, + ], + [ + { type: 'B', value: 3 }, + { type: 'B', value: 4 }, + ], + [{ type: 'A', value: 5 }], + ]) + }) + + it('should group based on custom logic (sum of digits is even)', () => { + const items = [11, 22, 34, 45, 55] + const sumDigitsIsEven = (n: number) => + n + .toString() + .split('') + .map(Number) + .reduce((a, b) => a + b, 0) % + 2 === + 0 + + const result = groupConsecutiveItemsByPredicate( + items, + (a, b) => sumDigitsIsEven(a) === sumDigitsIsEven(b) + ) + expect(result).toEqual([[11, 22], [34, 45], [55]]) + }) +}) + +describe('computeDiffWithLineNumbers', () => { + const createTestUri = () => ({ + fsPath: '/path/to/file.ts', + toString: () => '/path/to/file.ts', + } as vscode.Uri) + + const assertDiffResult = (result: any, expectedSnapshot: string) => { + expect(result).toBeInstanceOf(PromptString) + expect(result).toMatchInlineSnapshot(expectedSnapshot) + } + + it('should compute diff with line numbers for added content', () => { + const uri = createTestUri() + const originalContent = 'line 1\nline 2\nline 3' + const modifiedContent = 'line 1\nline 2\nnew line\nline 3' + const numContextLines = 2 + + const result = computeDiffWithLineNumbers(uri, originalContent, modifiedContent, numContextLines) + + assertDiffResult( + result, + dedent` + "1 | line 1 + 2 | line 2 + 3+| new line + 4 | line 3" + ` + ) + }) + + it('should compute diff with line numbers for removed content', () => { + const uri = createTestUri() + const originalContent = 'line 1\nline 2\nline to remove\nline 3' + const modifiedContent = 'line 1\nline 2\nline 3' + const numContextLines = 2 + + const result = computeDiffWithLineNumbers(uri, originalContent, modifiedContent, numContextLines) + + assertDiffResult( + result, + dedent` + "1 | line 1 + 2 | line 2 + 3-| line to remove + 3 | line 3" + ` + ) + }) + + it('should compute diff with line numbers for modified content', () => { + const uri = createTestUri() + const originalContent = 'line 1\nold line\nline 3' + const modifiedContent = 'line 1\nnew line\nline 3' + const numContextLines = 1 + + const result = computeDiffWithLineNumbers(uri, originalContent, modifiedContent, numContextLines) + + assertDiffResult( + result, + dedent` + "1 | line 1 + 2-| old line + 2+| new line + 3 | line 3" + ` + ) + }) + + it('should respect numContextLines parameter', () => { + const uri = createTestUri() + const originalContent = 'line 1\nline 2\nline 3\nline 4\nline 5' + const modifiedContent = 'line 1\nline 2\nmodified line\nline 4\nline 5' + const numContextLines = 1 + + const result = computeDiffWithLineNumbers(uri, originalContent, modifiedContent, numContextLines) + + assertDiffResult( + result, + dedent` + "2 | line 2 + 3-| line 3 + 3+| modified line + 4 | line 4" + ` + ) + }) + + it('should handle empty content', () => { + const uri = createTestUri() + const result = computeDiffWithLineNumbers(uri, '', 'new content', 1) + + assertDiffResult( + result, + dedent` + "1+| new content" + ` + ) + }) +}) + + + + diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts index 7fd553a01adb..f566e4c387c5 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts @@ -1,112 +1,147 @@ -import { PromptString, ps } from '@sourcegraph/cody-shared' +import { PromptString } from '@sourcegraph/cody-shared' import { displayPath } from '@sourcegraph/cody-shared/src/editor/displayPath' import { structuredPatch } from 'diff' import type * as vscode from 'vscode' -import type { DiffHunk, TextDocumentChange } from './base' - -export interface GroupedTextDocumentChange { +import type { TextDocumentChange } from './base' + +/** + * Represents a group of text document changes with their line range information. + * The grouped changes are consecutive changes made in the document that should be treated as a single entity when computing diffs. + * + * @example + * When typing "hello world" in a document, each character typed generates a separate change event. + * These changes are grouped together as a single entity in this interface. + */ +export interface TextDocumentChangeGroup { + /** Array of individual text document changes in this group */ changes: TextDocumentChange[] - changeStartLine: number - changeEndLine: number -} - -export function combineNonOverlappingLinesSchemaTogether(groupedChanges: GroupedTextDocumentChange[]): GroupedTextDocumentChange[] { - const combinedChanges: GroupedTextDocumentChange[] = [] - if (groupedChanges.length === 0) { - return combinedChanges - } - let currentActiveChanges: GroupedTextDocumentChange[] = [groupedChanges[0]] - for (let i = 1; i < groupedChanges.length; i++) { - const groupedChange = groupedChanges[i] - if (shouldCombineGroupedChanges(currentActiveChanges[currentActiveChanges.length - 1], groupedChange)) { - currentActiveChanges.push(groupedChange) - } else { - combinedChanges.push(flatCombinedGroupedChanges(currentActiveChanges)) - currentActiveChanges = [groupedChange] - } - } - if (currentActiveChanges.length > 0) { - combinedChanges.push(flatCombinedGroupedChanges(currentActiveChanges)) - } - return combinedChanges -} -function flatCombinedGroupedChanges(changes: GroupedTextDocumentChange[]): GroupedTextDocumentChange { - return { - changes: changes.flatMap(change => change.changes), - changeStartLine: Math.min(...changes.map(change => change.changeStartLine)), - changeEndLine: Math.max(...changes.map(change => change.changeEndLine)) - } -} + /** + * The starting line number of the changes in this group + */ + changeStartLine: number -function shouldCombineGroupedChanges(a: GroupedTextDocumentChange, b: GroupedTextDocumentChange): boolean { - return !(a.changeStartLine <= b.changeEndLine && a.changeEndLine >= b.changeStartLine) + /** + * The ending line number of the changes in this group + */ + changeEndLine: number } - +/** + * Groups consecutive text document changes together based on line overlap. + * This function helps create more meaningful diffs by combining related changes that occur on overlapping lines. + * + * For example, when a user types multiple characters or performs multiple edits in the same area of text, + * these changes are grouped together as a single logical change instead of being treated as separate changes. + * + * @param changes - Array of individual text document changes to be grouped + * @returns Array of TextDocumentChangeGroup objects, each containing related changes and their combined line range + * + * The predicate used for grouping checks if: + * - The original ranges of two changes overlap (for modifications/deletions) + * - The inserted range of the first change overlaps with the original range of the second change + * This ensures that changes affecting the same or adjacent lines are grouped together. + */ export function groupChangesForSimilarLinesTogether( changes: TextDocumentChange[] -): GroupedTextDocumentChange[] { +): TextDocumentChangeGroup[] { if (changes.length === 0) { return [] } - const groupedChanges: GroupedTextDocumentChange[] = [] - let currentGroup: TextDocumentChange[] = [changes[0]] - for (let i = 1; i < changes.length; i++) { - const change = changes[i] - const lastChange = currentGroup[currentGroup.length - 1] - if (shouldCombineChanges(lastChange, change)) { - currentGroup.push(change) - } else { - const range = getRangeValues(currentGroup) - groupedChanges.push({ - changes: currentGroup, - changeStartLine: range[0], - changeEndLine: range[1], - }) - currentGroup = [change] + const groupedChanges = groupConsecutiveItemsByPredicate( + changes, + (lastChange: TextDocumentChange, change: TextDocumentChange) => { + return ( + doesLinesOverlapForRanges(lastChange.change.range, change.change.range) || + doesLinesOverlapForRanges(lastChange.insertedRange, change.change.range) + ) } - } - if (currentGroup.length > 0) { - const range = getRangeValues(currentGroup) - groupedChanges.push({ + ) + return groupedChanges.map(currentGroup => { + const range = getMinMaxRangeLines(currentGroup) + return { changes: currentGroup, changeStartLine: range[0], changeEndLine: range[1], - }) - } - return groupedChanges -} - - -function getRangeValues(documentChanges: TextDocumentChange[]): [number, number] { - let minRange = getMinRange(documentChanges[0].replacedRange, documentChanges[0].insertedRange) - let maxRange = getMaxRange(documentChanges[0].replacedRange, documentChanges[0].insertedRange) - for (let i = 1; i < documentChanges.length; i++) { - const change = documentChanges[i] - minRange = getMinRange(getMinRange(change.replacedRange, change.insertedRange), minRange) - maxRange = getMaxRange(getMaxRange(change.replacedRange, change.insertedRange), maxRange) + } + }) +} + +/** + * Combines consecutive text document change groups that have non-overlapping line ranges. + * The function can generally be called after `groupChangesForSimilarLinesTogether` to further consolidate changes. + * + * This function takes an array of `TextDocumentChangeGroup` objects and merges consecutive groups + * where their line ranges do not overlap. By combining these non-overlapping groups, it creates + * larger groups of changes that can be processed together, even if they affect different parts + * of the document, as long as they occurred consecutively. + * + * @param groupedChanges - Array of `TextDocumentChangeGroup` objects to be combined. + * @returns Array of `TextDocumentChangeGroup` objects where consecutive non-overlapping groups have been merged. + * + * The predicate used for grouping checks if: + * - The line ranges of two groups do not overlap. + * - Specifically, it checks that `a.changeStartLine` to `a.changeEndLine` does not overlap with `b.changeStartLine` to `b.changeEndLine`. + * This ensures that consecutive groups with non-overlapping line ranges are combined together. + */ +export function combineNonOverlappingLinesSchemaTogether( + groupedChanges: TextDocumentChangeGroup[] +): TextDocumentChangeGroup[] { + if (groupedChanges.length === 0) { + return [] } - return [minRange.start.line, maxRange.end.line] -} - -function getMinRange(a: vscode.Range, b: vscode.Range): vscode.Range { - return a.start.isBeforeOrEqual(b.start) ? a : b -} - -function getMaxRange(a: vscode.Range, b: vscode.Range): vscode.Range { - return a.end.isBeforeOrEqual(b.end) ? b : a -} - -function shouldCombineChanges(lastChange: TextDocumentChange, change: TextDocumentChange): boolean { - return ( - doesLinesOverlap(lastChange.replacedRange, change.change.range) || - doesLinesOverlap(lastChange.insertedRange, change.change.range) + const combinedGroups = groupConsecutiveItemsByPredicate( + groupedChanges, + (a: TextDocumentChangeGroup, b: TextDocumentChangeGroup) => { + return !doLineSpansOverlap( + a.changeStartLine, + a.changeEndLine, + b.changeStartLine, + b.changeEndLine + ) + } ) -} - -function doesLinesOverlap(a: vscode.Range, b: vscode.Range): boolean { - return a.start.line <= b.end.line && a.end.line >= b.start.line + return combinedGroups.map(changes => ({ + changes: changes.flatMap(change => change.changes), + changeStartLine: Math.min(...changes.map(change => change.changeStartLine)), + changeEndLine: Math.max(...changes.map(change => change.changeEndLine)), + })) +} + +function getMinMaxRangeLines(documentChanges: TextDocumentChange[]): [number, number] { + let minLine = Number.POSITIVE_INFINITY + let maxLine = Number.NEGATIVE_INFINITY + for (const change of documentChanges) { + const ranges = [change.change.range, change.insertedRange] + for (const range of ranges) { + minLine = Math.min(minLine, range.start.line) + maxLine = Math.max(maxLine, range.end.line) + } + } + return [minLine, maxLine] +} + +/** + * Utility function to combine consecutive items in an array based on a predicate. + */ +export function groupConsecutiveItemsByPredicate( + items: T[], + shouldGroup: (a: T, b: T) => boolean +): T[][] { + return items.reduce((groups, item) => { + if (groups.length === 0) { + groups.push([item]) + } else { + const lastGroup = groups[groups.length - 1] + const lastItem = lastGroup[lastGroup.length - 1] + if (shouldGroup(lastItem, item)) { + lastGroup.push(item) + } else { + groups.push([item]) + } + } + return groups + }, []) } export function computeDiffWithLineNumbers( @@ -134,45 +169,6 @@ export function computeDiffWithLineNumbers( return gitDiff } -export function combineDiffHunksFromSimilarFile(hunks: DiffHunk[]): DiffHunk[] { - if (hunks.length === 0) { - return [] - } - const combinedHunks: DiffHunk[] = [] - let currentHunkList: DiffHunk[] = [hunks[0]] - for (let i = 1; i < hunks.length; i++) { - const hunk = hunks[i] - const lastHunk = currentHunkList[currentHunkList.length - 1] - if (shouldCombineHunks(hunk, lastHunk)) { - currentHunkList.push(hunk) - } else { - combinedHunks.push(combineMultipleHunks(currentHunkList)) - currentHunkList = [hunk] - } - } - if (currentHunkList.length > 0) { - combinedHunks.push(combineMultipleHunks(currentHunkList)) - } - return combinedHunks -} - -function combineMultipleHunks(hunks: DiffHunk[]): DiffHunk { - const lastestTime = Math.max(...hunks.map(h => h.latestEditTimestamp)) - const diffs = PromptString.join( - hunks.map(h => h.diff), - ps`\nthen\n` - ) - return { - uri: hunks[0].uri, - latestEditTimestamp: lastestTime, - diff: diffs, - } -} - -function shouldCombineHunks(hunk1: DiffHunk, hunk2: DiffHunk): boolean { - return hunk1.uri.toString() === hunk2.uri.toString() -} - export function getDiffStringForHunkWithLineNumbers(hunk: Diff.Hunk): string { const lines = [] let oldLineNumber = hunk.oldStart @@ -209,13 +205,15 @@ export function applyTextDocumentChanges( return content } -export function getNewContentAfterApplyingRange( - oldContent: string, - change: vscode.TextDocumentContentChangeEvent -): string { - return ( - oldContent.slice(0, change.rangeOffset) + - change.text + - oldContent.slice(change.rangeOffset + change.rangeLength) - ) +export function doesLinesOverlapForRanges(a: vscode.Range, b: vscode.Range): boolean { + return doLineSpansOverlap(a.start.line, a.end.line, b.start.line, b.end.line) +} + +function doLineSpansOverlap( + firstStart: number, + firstEnd: number, + secondStart: number, + secondEnd: number +): boolean { + return firstStart <= secondEnd && firstEnd >= secondStart } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts index 544f14964157..2b68da39c6dd 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts @@ -2,6 +2,7 @@ import path from 'node:path' import { type PromptString, contextFiltersProvider } from '@sourcegraph/cody-shared' import type { AutocompleteContextSnippet } from '@sourcegraph/cody-shared' import * as vscode from 'vscode' +import { getPositionAfterTextInsertion } from '../../../text-processing/utils' import type { ContextRetriever, ContextRetrieverOptions } from '../../../types' import { RetrieverIdentifier, type ShouldUseContextParams, shouldBeUsedAsContext } from '../../utils' import { @@ -11,10 +12,7 @@ import { type TextDocumentChange, createDiffStrategy, } from './recent-edits-diff-helpers/base' -import { - applyTextDocumentChanges, - getNewContentAfterApplyingRange, -} from './recent-edits-diff-helpers/utils' +import { applyTextDocumentChanges } from './recent-edits-diff-helpers/utils' interface TrackedDocument { content: string @@ -98,7 +96,7 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever const diffs: DiffAcrossDocuments[] = [] const diffPromises = Array.from(this.trackedDocuments.entries()).map( async ([uri, trackedDocument]) => { - if (trackedDocument.changes.length===0) { + if (trackedDocument.changes.length === 0) { return null } const diffHunks = await this.getDiff(vscode.Uri.parse(uri)) @@ -166,29 +164,15 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever } const now = Date.now() - const oldCursorPosition = event.contentChanges[0].range.end for (const change of event.contentChanges) { - const oldContent = - trackedDocument.changes.length > 0 - ? trackedDocument.changes[trackedDocument.changes.length - 1].newContent - : trackedDocument.content - const newCursorPosition = calculateNewCursorPositions(change, oldCursorPosition) - const newContent = getNewContentAfterApplyingRange(oldContent, change) - const insertedRange = calculateInsertedRangeInDocumentBasedOnChange( - oldContent, - newContent, - change + const insertedRange = new vscode.Range( + change.range.start, + getPositionAfterTextInsertion(change.range.start, change.text) ) - trackedDocument.changes.push({ timestamp: now, - oldCursorPosition, - newCursorPosition, - oldContent, - newContent, - replacedRange: change.range, - insertedRange, change, + insertedRange, }) } this.reconcileOutdatedChanges() @@ -261,58 +245,3 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever } } } - -export function calculateInsertedRangeInDocumentBasedOnChange( - oldContent: string, - newContent: string, - change: vscode.TextDocumentContentChangeEvent -): vscode.Range { - // Function calculates the updated range for the new document based on the change. - const startOffset = change.rangeOffset - const endOffset = startOffset + change.text.length - - const startPosition = getPositionAt(newContent, startOffset) - const endPosition = getPositionAt(newContent, endOffset) - - return new vscode.Range(startPosition, endPosition) -} - -// Helper function to convert an offset to a Position (line and character) -function getPositionAt(content: string, offset: number): vscode.Position { - let line = 0 - let character = 0 - let i = 0 - - while (i < offset) { - if (content[i] === '\n') { - line++ - character = 0 - } else { - character++ - } - i++ - } - - return new vscode.Position(line, character) -} - -export function calculateNewCursorPositions( - change: vscode.TextDocumentContentChangeEvent, - oldCursorPosition: vscode.Position -): vscode.Position { - // Starting position of the change - const start = change.range.start - - // Inserted text and its lines - const insertedText = change.text - const insertedLines = insertedText.split('\n') - let newCursorPosition: vscode.Position - if (insertedLines.length === 1) { - newCursorPosition = new vscode.Position(start.line, start.character + insertedText.length) - } else { - const newLineCount = insertedLines.length - 1 - const lastLineLength = insertedLines[insertedLines.length - 1].length - newCursorPosition = new vscode.Position(start.line + newLineCount, lastLineLength) - } - return newCursorPosition -} From 19a1295985f3c3bca9f49700e0da620f1a3a1c9d Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Sat, 23 Nov 2024 04:12:42 +0530 Subject: [PATCH 05/18] checkpoint --- .../recent-edits-diff-helpers/helper.test.ts | 78 +++++++++++++++ .../recent-edits-diff-helpers/helper.ts | 98 +++++++++++++++++++ .../recent-edits-diff-helpers/utils-new.ts | 76 -------------- .../recent-edits-diff-helpers/utils.test.ts | 86 ++++++++++++++-- 4 files changed, 254 insertions(+), 84 deletions(-) create mode 100644 vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts create mode 100644 vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts delete mode 100644 vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils-new.ts diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts new file mode 100644 index 000000000000..6840462eead5 --- /dev/null +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it } from 'vitest' +import { parseTextAndGenerateChangeEvents } from './helper'; +import {applyTextDocumentChanges} from './utils'; + +describe('parseTextAndGenerateChangeEvents', () => { + + const testChanges = ( + params: { + text: string, + expectedOriginalString: string, + expectedChanges: string[], + } + ) => { + const { text, expectedOriginalString, expectedChanges } = params; + const { originalText, changeEvents } = parseTextAndGenerateChangeEvents(text); + expect(originalText).to.equal(expectedOriginalString); + expect(changeEvents.length).to.equal(expectedChanges.length); + for(let i=0; i { + const text = 'This is a test string.' + const expectedOriginalString = 'This is a string.' + const expectedChanges = [ + 'This is a test string.' + ] + testChanges({ text, expectedOriginalString, expectedChanges }); + }); + + it('should handle delete markers correctly', () => { + const text = 'This is a sample string.' + const expectedOriginalString = 'This is a sample string.' + const expectedChanges = [ + 'This is a string.' + ] + testChanges({ text, expectedOriginalString, expectedChanges }); + }); + + + it('should handle replace markers correctly', () => { + const text = 'Please replaceswap this word.' + const expectedOriginalString = 'Please replace this word.' + const expectedChanges = [ + 'Please swap this word.' + ] + testChanges({ text, expectedOriginalString, expectedChanges }); + }); + + it('should handle multiple markers correctly', () => { + const text = 'start and middle unnecessary text swap change end.'; + const expectedOriginalString = 'start middle unnecessary text swap end.'; + const expectedChanges = [ + 'start and middle unnecessary text swap end.', + 'start and middle text swap end.', + 'start and middle text change end.' + ] + testChanges({ text, expectedOriginalString, expectedChanges }); + }); + + it('should handle text without markers correctly', () => { + const text = 'This is plain text.'; + const expectedOriginalString = 'This is plain text.'; + const expectedChanges: string[] = [] + testChanges({ text, expectedOriginalString, expectedChanges }); + }); + + it('should ignore unmatched markers', () => { + const inputText = 'Unmatched markers insert text without closing.'; + const { originalText, changeEvents } = parseTextAndGenerateChangeEvents(inputText); + + expect(originalText).to.equal('Unmatched markers insert text without closing.'); + expect(changeEvents).to.have.lengthOf(0); + }); +}); diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts new file mode 100644 index 000000000000..1779b3f3766b --- /dev/null +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts @@ -0,0 +1,98 @@ +import * as vscode from 'vscode'; +import {getPositionAfterTextInsertion} from '../../../../text-processing/utils'; + +/** + * Parses the input text containing markers and generates the corresponding + * TextDocumentContentChangeEvent events. It also returns the original text + * after processing the markers. + * + * Markers: + * - `text`: Insert `text` at the position of ``. + * - `text`: Delete `text` starting from the position of ``. + * - `text1text2`: Replace `text1` with `text2` starting from the position of ``. + * + * @param text The input text containing markers. + * @returns An object containing the original text and the array of change events. + */ +export function parseTextAndGenerateChangeEvents( + text: string +): { originalText: string; changeEvents: vscode.TextDocumentContentChangeEvent[] } { + const changeEvents: vscode.TextDocumentContentChangeEvent[] = []; + let originalText = ''; + let currentText = ''; + let currentOffset = 0; + + const regex = /(.*?)<\/I>|(.*?)<\/D>|(.*?)(.*?)<\/R>/gs; + let match: RegExpExecArray | null; + + while ((match = regex.exec(text)) !== null) { + const [fullMatch, insertText, deleteText, replaceText1, replaceText2] = match; + const matchIndex = match.index; + + const textBeforeMarker = text.substring(currentOffset, matchIndex); + originalText += textBeforeMarker; + currentText += textBeforeMarker; + + const position = getPositionAt(currentText, currentText.length); + + if (insertText !== undefined) { + changeEvents.push({ + range: new vscode.Range(position, position), + rangeOffset: currentText.length, + rangeLength: 0, + text: insertText, + }); + currentText += insertText; + } else if (deleteText !== undefined) { + const deleteStartPosition = getPositionAt(currentText, currentText.length); + const deleteEndPosition = getPositionAfterTextInsertion(deleteStartPosition, deleteText); + const deleteRange = new vscode.Range( + deleteStartPosition, + deleteEndPosition + ); + changeEvents.push({ + range: deleteRange, + rangeOffset: currentText.length, + rangeLength: deleteText.length, + text: '', + }); + originalText += deleteText; + } else if (replaceText1 !== undefined && replaceText2 !== undefined) { + const replaceStartPosition = getPositionAt(currentText, currentText.length); + const replaceEndPosition = getPositionAfterTextInsertion(replaceStartPosition, replaceText1); + const replaceRange = new vscode.Range( + replaceStartPosition, + replaceEndPosition + ); + changeEvents.push({ + range: replaceRange, + rangeOffset: currentText.length, + rangeLength: replaceText1.length, + text: replaceText2, + }); + + currentText += replaceText2; + originalText += replaceText1; + } + + currentOffset = matchIndex + fullMatch.length; + } + const remainingText = text.substring(currentOffset); + originalText += remainingText; + currentText += remainingText; + return { originalText, changeEvents }; +} + +/** + * Calculates the Position in the text at the given offset. + * + * @param text The text content. + * @param offset The offset in the text. + * @returns The Position corresponding to the offset. + */ +function getPositionAt(text: string, offset: number): vscode.Position { + const lines = text.substring(0, offset).split(/\r\n|\r|\n/); + const line = lines.length - 1; + const character = lines[line].length; + return new vscode.Position(line, character); +} diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils-new.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils-new.ts deleted file mode 100644 index d80ae824e205..000000000000 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils-new.ts +++ /dev/null @@ -1,76 +0,0 @@ -import * as vscode from 'vscode'; -import {PromptString} from '@sourcegraph/cody-shared'; -import {displayPath} from '@sourcegraph/cody-shared/src/editor/displayPath'; -import {structuredPatch} from 'diff'; - -export function computeDiffWithLineNumbers( - uri: vscode.Uri, - originalContent: string, - modifiedContent: string, - numContextLines: number -): PromptString { - const hunkDiffs = [] - const filename = displayPath(uri) - const patch = structuredPatch( - `a/${filename}`, - `b/${filename}`, - originalContent, - modifiedContent, - '', - '', - { context: numContextLines } - ) - for (const hunk of patch.hunks) { - const diffString = getDiffStringForHunkWithLineNumbers(hunk) - hunkDiffs.push(diffString) - } - const gitDiff = PromptString.fromStructuredGitDiff(uri, hunkDiffs.join('\nthen\n')) - return gitDiff -} - -export function getDiffStringForHunkWithLineNumbers(hunk: Diff.Hunk): string { - const lines = [] - let oldLineNumber = hunk.oldStart - let newLineNumber = hunk.newStart - for (const line of hunk.lines) { - if (line.length === 0) { - continue - } - if (line[0] === '-') { - lines.push(`${oldLineNumber}${line[0]}| ${line.slice(1)}`) - oldLineNumber++ - } else if (line[0] === '+') { - lines.push(`${newLineNumber}${line[0]}| ${line.slice(1)}`) - newLineNumber++ - } else if (line[0] === ' ') { - lines.push(`${newLineNumber}${line[0]}| ${line.slice(1)}`) - oldLineNumber++ - newLineNumber++ - } - } - return lines.join('\n') -} - -export function applyTextDocumentChanges( - content: string, - changes: vscode.TextDocumentContentChangeEvent[] -): string { - for (const change of changes) { - content = - content.slice(0, change.rangeOffset) + - change.text + - content.slice(change.rangeOffset + change.rangeLength) - } - return content -} - -export function getNewContentAfterApplyingRange( - oldContent: string, - change: vscode.TextDocumentContentChangeEvent -): string { - return ( - oldContent.slice(0, change.rangeOffset) + - change.text + - oldContent.slice(change.rangeOffset + change.rangeLength) - ) -} diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts index 87f93590cc46..808153ce3322 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts @@ -1,13 +1,83 @@ import { PromptString } from '@sourcegraph/cody-shared' -import { describe, expect, it } from 'vitest' -import type * as vscode from 'vscode' -import { - applyTextDocumentChanges, - computeDiffWithLineNumbers, - doesLinesOverlapForRanges, - groupConsecutiveItemsByPredicate, -} from './utils' +import { describe, expect, it, should } from 'vitest' +import * as vscode from 'vscode' +import {applyTextDocumentChanges, computeDiffWithLineNumbers, doesLinesOverlapForRanges, groupConsecutiveItemsByPredicate, groupChangesForSimilarLinesTogether, TextDocumentChangeGroup} from './utils'; import dedent from 'dedent' +import {documentAndPosition} from '../../../../test-helpers'; +import {TextDocumentChange} from './base'; +import {getPositionAfterTextInsertion} from '../../../../text-processing/utils'; +import {createGitDiff} from '../../../../../../../lib/shared/src/editor/create-git-diff'; + + +/** + * Generates a sequence of text document change events for inserting text at a specific position. + * @param document The VS Code text document to insert text into + * @param position The position in the document where text should be inserted + * @param textToAdd Array of text strings to insert sequentially + * @returns Array of TextDocumentChange objects representing the insertion changes + */ +function generateChangeEventsForInsertionAtPosition( + document: vscode.TextDocument, + position: vscode.Position, + textToAdd: string[], +): TextDocumentChange[] { + const changeEvents: TextDocumentChange[] = []; + let currentPosition = position; + let currentOffset = document.offsetAt(position); + for (let i = 0; i < textToAdd.length; i++) { + const text = textToAdd[i]; + const change = { + range: new vscode.Range(currentPosition, currentPosition), + rangeLength: 0, + text: text, + rangeOffset: currentOffset + }; + // Update current position to be after the inserted text + currentPosition = getPositionAfterTextInsertion(currentPosition, text); + currentOffset += text.length; + changeEvents.push({ + timestamp: Date.now(), + change: change, + insertedRange: new vscode.Range(currentPosition, currentPosition), + }); + } + return changeEvents; +}; + +function getDiffsForContentChanges(oldContent: string, groupedChanges: TextDocumentChangeGroup[]): string[] { + const diffList: string[] = []; + let currentContent = oldContent; + for (const changeGroup of groupedChanges) { + const newContent = applyTextDocumentChanges(currentContent, changeGroup.changes.map(change => change.change)); + const diff = createGitDiff('test.ts', currentContent, newContent) + diffList.push(diff) + currentContent = newContent; + } + return diffList; +} + + +describe('documentChangeEventGroupings', () => { + it('should group changes for similar lines together on addition', () => { + const { document, position } = documentAndPosition('console.â–ˆ\n') + const textToAdd = 'log("Hello, world!");\nconsole.log("done")' + const changeEvents = generateChangeEventsForInsertionAtPosition(document, position, textToAdd.split('')); + const result = groupChangesForSimilarLinesTogether(changeEvents); + expect(result.length).toBe(1) + expect(result[0].changes.length).toBe(textToAdd.split('').length) + const diffs = getDiffsForContentChanges(document.getText(), result); + expect(diffs.length).toBe(1) + expect(diffs[0].split('\n').slice(3).join('\n')).toMatchInlineSnapshot(` + "-console. + +console.log("Hello, world!"); + +console.log("done") + " + `) + }) + + + +}) describe('applyTextDocumentChanges', () => { const createChange = ( From 082fa9e75bca51b6d1089786d21513879e00ab25 Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Sat, 23 Nov 2024 05:01:38 +0530 Subject: [PATCH 06/18] basic test case structure --- .../recent-edits-diff-helpers/helper.test.ts | 132 +++++++++++++++++- .../recent-edits-diff-helpers/helper.ts | 76 +++++++--- .../recent-edits-diff-helpers/utils.test.ts | 87 +++--------- 3 files changed, 206 insertions(+), 89 deletions(-) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts index 6840462eead5..46c13f626ef4 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' -import { parseTextAndGenerateChangeEvents } from './helper'; +import { parseTextAndGenerateChangeEvents, getPositionAt } from './helper'; import {applyTextDocumentChanges} from './utils'; +import dedent from 'dedent' describe('parseTextAndGenerateChangeEvents', () => { @@ -75,4 +76,133 @@ describe('parseTextAndGenerateChangeEvents', () => { expect(originalText).to.equal('Unmatched markers insert text without closing.'); expect(changeEvents).to.have.lengthOf(0); }); + + + it('should handle complex multi-line text with mixed markers', () => { + const text = dedent` + First line inserted text + Second line to be deleted + Third line oldThird line new + Fourth line addition + Fifth line addition + Sixth line to delete + End of text. + ` + const expectedOriginalString = dedent` + First line + Second line to be deleted + Third line old + Sixth line to delete + End of text. + ` + const expectedChanges = [ + dedent` + First line inserted text + Second line to be deleted + Third line old + Sixth line to delete + End of text. + `, + dedent` + First line inserted text + Second line + Third line old + Sixth line to delete + End of text. + `, + dedent` + First line inserted text + Second line + Third line new + Sixth line to delete + End of text. + `, + dedent` + First line inserted text + Second line + Third line new + Fourth line addition + Fifth line addition + Sixth line to delete + End of text. + `, + dedent` + First line inserted text + Second line + Third line new + Fourth line addition + Fifth line addition + End of text. + `, + ]; + testChanges({ text, expectedOriginalString, expectedChanges }); + }); + }); + + +describe('getPositionAt', () => { + it('should return position at offset 0', () => { + const content = 'Hello, world!' + const offset = 0 + const position = getPositionAt(content, offset) + expect(position.line).to.equal(0) + expect(position.character).to.equal(0) + }) + + it('should return correct position in single-line content', () => { + const content = 'Hello, world!' + const offset = 7 + const position = getPositionAt(content, offset) + expect(position.line).to.equal(0) + expect(position.character).to.equal(7) + }) + + it('should return correct position at the end of content', () => { + const content = 'Hello, world!' + const offset = content.length + const position = getPositionAt(content, offset) + expect(position.line).to.equal(0) + expect(position.character).to.equal(content.length) + }) + + it('should return correct position in multi-line content', () => { + const content = 'Line 1\nLine 2\nLine 3' + const offset = content.indexOf('Line 2') + const position = getPositionAt(content, offset) + expect(position.line).to.equal(1) + expect(position.character).to.equal(0) + }) + + it('should handle offsets at line breaks', () => { + const content = 'Line 1\nLine 2\nLine 3' + const offset = content.indexOf('\n') + 1 // Position after the first line break + const position = getPositionAt(content, offset) + expect(position.line).to.equal(1) + expect(position.character).to.equal(0) + }) + + it('should return correct position for offsets within lines', () => { + const content = 'Line 1\nLine 2\nLine 3' + const offset = content.indexOf('2') + const position = getPositionAt(content, offset) + expect(position.line).to.equal(1) + expect(position.character).to.equal(5) + }) + + it('should handle empty content', () => { + const content = '' + const offset = 0 + const position = getPositionAt(content, offset) + expect(position.line).to.equal(0) + expect(position.character).to.equal(0) + }) + + it('should handle content with carriage returns correctly', () => { + const content = 'Line 1\r\nLine 2\r\nLine 3' + const offset = content.indexOf('Line 3') + const position = getPositionAt(content, offset) + expect(position.line).to.equal(2) + expect(position.character).to.equal(0) + }) +}) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts index 1779b3f3766b..198d7a5b2ab8 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts @@ -1,7 +1,41 @@ import * as vscode from 'vscode'; import {getPositionAfterTextInsertion} from '../../../../text-processing/utils'; +import {applyTextDocumentChanges, TextDocumentChangeGroup} from './utils'; +import {createGitDiff} from '../../../../../../../lib/shared/src/editor/create-git-diff'; +import {TextDocumentChange} from './base'; + +export function getTextDocumentChangesForText(text: string): {originalText: string, changes: TextDocumentChange[]} { + const {originalText, changeEvents} = parseTextAndGenerateChangeEvents(text); + const documentChanges: TextDocumentChange[] = [] + for(const change of changeEvents) { + const insertedRange = new vscode.Range( + change.range.start, + getPositionAfterTextInsertion(change.range.start, change.text) + ) + documentChanges.push({ + timestamp: Date.now(), + change: change, + insertedRange + }) + } + return {originalText, changes: documentChanges} +} + +export function getDiffsForContentChanges(oldContent: string, groupedChanges: TextDocumentChangeGroup[]): string[] { + const diffList: string[] = []; + let currentContent = oldContent; + for (const changeGroup of groupedChanges) { + const newContent = applyTextDocumentChanges(currentContent, changeGroup.changes.map(change => change.change)); + const diff = createGitDiff('test.ts', currentContent, newContent) + diffList.push(diff) + currentContent = newContent; + } + return diffList; +} + /** + * The function is used by the test classes to simulate the text changes in a document text. * Parses the input text containing markers and generates the corresponding * TextDocumentContentChangeEvent events. It also returns the original text * after processing the markers. @@ -44,37 +78,25 @@ export function parseTextAndGenerateChangeEvents( }); currentText += insertText; } else if (deleteText !== undefined) { - const deleteStartPosition = getPositionAt(currentText, currentText.length); - const deleteEndPosition = getPositionAfterTextInsertion(deleteStartPosition, deleteText); - const deleteRange = new vscode.Range( - deleteStartPosition, - deleteEndPosition - ); + const deleteEndPosition = getPositionAfterTextInsertion(position, deleteText); changeEvents.push({ - range: deleteRange, + range: new vscode.Range(position, deleteEndPosition), rangeOffset: currentText.length, rangeLength: deleteText.length, text: '', }); originalText += deleteText; } else if (replaceText1 !== undefined && replaceText2 !== undefined) { - const replaceStartPosition = getPositionAt(currentText, currentText.length); - const replaceEndPosition = getPositionAfterTextInsertion(replaceStartPosition, replaceText1); - const replaceRange = new vscode.Range( - replaceStartPosition, - replaceEndPosition - ); + const replaceEndPosition = getPositionAfterTextInsertion(position, replaceText1); changeEvents.push({ - range: replaceRange, + range: new vscode.Range(position, replaceEndPosition), rangeOffset: currentText.length, rangeLength: replaceText1.length, text: replaceText2, }); - currentText += replaceText2; originalText += replaceText1; } - currentOffset = matchIndex + fullMatch.length; } const remainingText = text.substring(currentOffset); @@ -90,9 +112,21 @@ export function parseTextAndGenerateChangeEvents( * @param offset The offset in the text. * @returns The Position corresponding to the offset. */ -function getPositionAt(text: string, offset: number): vscode.Position { - const lines = text.substring(0, offset).split(/\r\n|\r|\n/); - const line = lines.length - 1; - const character = lines[line].length; - return new vscode.Position(line, character); +// Helper function to convert an offset to a Position (line and character) +export function getPositionAt(content: string, offset: number): vscode.Position { + let line = 0 + let character = 0 + let i = 0 + while (i < offset) { + if (content[i] === '\n') { + line++ + character = 0 + } else { + character++ + } + i++ + } + + return new vscode.Position(line, character) } + diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts index 808153ce3322..2dca4cd0ebe6 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts @@ -1,82 +1,35 @@ import { PromptString } from '@sourcegraph/cody-shared' -import { describe, expect, it, should } from 'vitest' +import { describe, expect, it } from 'vitest' import * as vscode from 'vscode' -import {applyTextDocumentChanges, computeDiffWithLineNumbers, doesLinesOverlapForRanges, groupConsecutiveItemsByPredicate, groupChangesForSimilarLinesTogether, TextDocumentChangeGroup} from './utils'; +import {applyTextDocumentChanges, computeDiffWithLineNumbers, doesLinesOverlapForRanges, groupConsecutiveItemsByPredicate, groupChangesForSimilarLinesTogether} from './utils'; import dedent from 'dedent' -import {documentAndPosition} from '../../../../test-helpers'; -import {TextDocumentChange} from './base'; -import {getPositionAfterTextInsertion} from '../../../../text-processing/utils'; -import {createGitDiff} from '../../../../../../../lib/shared/src/editor/create-git-diff'; - - -/** - * Generates a sequence of text document change events for inserting text at a specific position. - * @param document The VS Code text document to insert text into - * @param position The position in the document where text should be inserted - * @param textToAdd Array of text strings to insert sequentially - * @returns Array of TextDocumentChange objects representing the insertion changes - */ -function generateChangeEventsForInsertionAtPosition( - document: vscode.TextDocument, - position: vscode.Position, - textToAdd: string[], -): TextDocumentChange[] { - const changeEvents: TextDocumentChange[] = []; - let currentPosition = position; - let currentOffset = document.offsetAt(position); - for (let i = 0; i < textToAdd.length; i++) { - const text = textToAdd[i]; - const change = { - range: new vscode.Range(currentPosition, currentPosition), - rangeLength: 0, - text: text, - rangeOffset: currentOffset - }; - // Update current position to be after the inserted text - currentPosition = getPositionAfterTextInsertion(currentPosition, text); - currentOffset += text.length; - changeEvents.push({ - timestamp: Date.now(), - change: change, - insertedRange: new vscode.Range(currentPosition, currentPosition), - }); - } - return changeEvents; -}; - -function getDiffsForContentChanges(oldContent: string, groupedChanges: TextDocumentChangeGroup[]): string[] { - const diffList: string[] = []; - let currentContent = oldContent; - for (const changeGroup of groupedChanges) { - const newContent = applyTextDocumentChanges(currentContent, changeGroup.changes.map(change => change.change)); - const diff = createGitDiff('test.ts', currentContent, newContent) - diffList.push(diff) - currentContent = newContent; - } - return diffList; +import {getTextDocumentChangesForText, getDiffsForContentChanges} from './helper'; + + +const removeNewLineStringFromDiff = (text: string) => { + const lines = text.split('\n') + return lines.filter(line => !line.includes('\\ No newline at end of file')).join('\n') } +describe.only('documentChangeEventGroupings', () => { -describe('documentChangeEventGroupings', () => { - it('should group changes for similar lines together on addition', () => { - const { document, position } = documentAndPosition('console.â–ˆ\n') - const textToAdd = 'log("Hello, world!");\nconsole.log("done")' - const changeEvents = generateChangeEventsForInsertionAtPosition(document, position, textToAdd.split('')); - const result = groupChangesForSimilarLinesTogether(changeEvents); + it('basic test case', () => { + const text = dedent` + console.log('Hello, world!'); + console.log('done') + ` + const {originalText, changes} = getTextDocumentChangesForText(text); + const result = groupChangesForSimilarLinesTogether(changes); expect(result.length).toBe(1) - expect(result[0].changes.length).toBe(textToAdd.split('').length) - const diffs = getDiffsForContentChanges(document.getText(), result); + const diffs = getDiffsForContentChanges(originalText, result); expect(diffs.length).toBe(1) - expect(diffs[0].split('\n').slice(3).join('\n')).toMatchInlineSnapshot(` + expect(removeNewLineStringFromDiff(diffs[0]).split('\n').slice(3).join('\n')).toMatchInlineSnapshot(` "-console. - +console.log("Hello, world!"); - +console.log("done") + +console.log('Hello, world!'); + +console.log('done') " `) }) - - - }) describe('applyTextDocumentChanges', () => { From e50b49b2b3fb8b0c2754425b283f3a2a2cdd65e9 Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Sat, 23 Nov 2024 05:15:33 +0530 Subject: [PATCH 07/18] improve marker strategy --- .../recent-edits-diff-helpers/helper.test.ts | 89 +++++++++++++++++++ .../recent-edits-diff-helpers/helper.ts | 25 ++++++ 2 files changed, 114 insertions(+) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts index 46c13f626ef4..7667ae8a7bd0 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest' import { parseTextAndGenerateChangeEvents, getPositionAt } from './helper'; import {applyTextDocumentChanges} from './utils'; +import { processContinousChangesForText } from './helper'; import dedent from 'dedent' describe('parseTextAndGenerateChangeEvents', () => { @@ -138,6 +139,94 @@ describe('parseTextAndGenerateChangeEvents', () => { testChanges({ text, expectedOriginalString, expectedChanges }); }); + it('should handle continuous insert markers correctly', () => { + const text = 'Hello World!' + const expectedOriginalString = 'Hello !' + const expectedChanges = [ + 'Hello W!', + 'Hello Wo!', + 'Hello Wor!', + 'Hello Worl!', + 'Hello World!', + ] + testChanges({ text, expectedOriginalString, expectedChanges }) + }) + + it('should handle continuous delete markers correctly', () => { + const text = 'Delete this text.' + const expectedOriginalString = 'Delete this text.' + const expectedChanges = [ + 'Deletethis text.', + 'Deletehis text.', + 'Deleteis text.', + 'Deletes text.', + 'Delete text.', + ] + testChanges({ text, expectedOriginalString, expectedChanges }) + }) + + it('should handle multiple continuous markers correctly', () => { + const text = 'Hi, this is a sample text.' + const expectedOriginalString = ', this is a sample text.' + const expectedChanges = [ + 'H, this is a sample text.', + 'Hi, this is a sample text.', + 'Hi, this is a ample text.', + 'Hi, this is a mple text.', + 'Hi, this is a ple text.', + 'Hi, this is a le text.', + 'Hi, this is a e text.', + 'Hi, this is a text.', + ] + testChanges({ text, expectedOriginalString, expectedChanges }) + }) + +}); + + + +describe('processContinousChangesForText', () => { + + it('should convert tags into individual tags for each character', () => { + const input = 'Hello World!'; + const expectedOutput = 'Hello World!'; + expect(processContinousChangesForText(input)).toBe(expectedOutput); + }); + + it('should convert tags into individual tags for each character', () => { + const input = 'Delete this text.'; + const expectedOutput = 'Delete this text.'; + expect(processContinousChangesForText(input)).toBe(expectedOutput); + }); + + it('should handle multiple and tags in the input', () => { + const input = 'Hello and Goodbye!'; + const expectedOutput = 'Hello and Goodbye!'; + expect(processContinousChangesForText(input)).toBe(expectedOutput); + }); + + it('should return the same text if there are no or tags', () => { + const input = 'No changes here.'; + expect(processContinousChangesForText(input)).toBe(input); + }); + + it('should handle empty and tags gracefully', () => { + const input = 'Empty and tags.'; + const expectedOutput = 'Empty and tags.'; + expect(processContinousChangesForText(input)).toBe(expectedOutput); + }); + + it('should handle special characters within and tags', () => { + const input = 'Special chars: !@# and 123.'; + const expectedOutput = 'Special chars: !@# and 123.'; + expect(processContinousChangesForText(input)).toBe(expectedOutput); + }); + + it('should handle consecutive and tags without text in between', () => { + const input = 'ABC123'; + const expectedOutput = 'ABC123'; + expect(processContinousChangesForText(input)).toBe(expectedOutput); + }); }); diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts index 198d7a5b2ab8..bcbc64c5d5ed 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts @@ -44,6 +44,8 @@ export function getDiffsForContentChanges(oldContent: string, groupedChanges: Te * - `text`: Insert `text` at the position of ``. * - `text`: Delete `text` starting from the position of ``. * - `text1text2`: Replace `text1` with `text2` starting from the position of ``. + * - `text`: Creates a seperate insert change for each character in `text`. + * - `text`: Creates a seperate delete change for each character in `text`. * * @param text The input text containing markers. * @returns An object containing the original text and the array of change events. @@ -51,6 +53,8 @@ export function getDiffsForContentChanges(oldContent: string, groupedChanges: Te export function parseTextAndGenerateChangeEvents( text: string ): { originalText: string; changeEvents: vscode.TextDocumentContentChangeEvent[] } { + text = processContinousChangesForText(text); + const changeEvents: vscode.TextDocumentContentChangeEvent[] = []; let originalText = ''; let currentText = ''; @@ -105,6 +109,27 @@ export function parseTextAndGenerateChangeEvents( return { originalText, changeEvents }; } +/** + * Processes continuous changes in text by converting continuous insertion and deletion markers + * into individual character markers. + * + * @param text The input text containing and markers + * @returns The processed text with individual and markers for each character + */ +export function processContinousChangesForText(text: string): string { + // Replace ... with individual ... markers for each character + text = text.replace(/(.*?)<\/IC>/gs, (_, content) => { + return content.split('').map((char: string) => `${char}`).join(''); + }); + + // Replace ... with individual ... markers for each character + text = text.replace(/(.*?)<\/DC>/gs, (_, content) => { + return content.split('').map((char: string) => `${char}`).join(''); + }); + + return text; +} + /** * Calculates the Position in the text at the given offset. * From 3a1f1d1a3f01dcad9d5dbc4e02f98f1f389ec8da Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Sat, 23 Nov 2024 06:14:19 +0530 Subject: [PATCH 08/18] diff --- .../recent-edits-diff-helpers/helper.test.ts | 171 ++++++++---------- .../recent-edits-diff-helpers/helper.ts | 122 +++++++------ .../recent-edits-diff-helpers/utils.test.ts | 104 ++++++++--- 3 files changed, 224 insertions(+), 173 deletions(-) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts index 7667ae8a7bd0..b9e0ec834ba0 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.test.ts @@ -1,83 +1,75 @@ -import { describe, expect, it } from 'vitest' -import { parseTextAndGenerateChangeEvents, getPositionAt } from './helper'; -import {applyTextDocumentChanges} from './utils'; -import { processContinousChangesForText } from './helper'; import dedent from 'dedent' +import { describe, expect, it } from 'vitest' +import { getPositionAt, parseTextAndGenerateChangeEvents } from './helper' +import { processContinousChangesForText } from './helper' +import { applyTextDocumentChanges } from './utils' describe('parseTextAndGenerateChangeEvents', () => { - - const testChanges = ( - params: { - text: string, - expectedOriginalString: string, - expectedChanges: string[], - } - ) => { - const { text, expectedOriginalString, expectedChanges } = params; - const { originalText, changeEvents } = parseTextAndGenerateChangeEvents(text); - expect(originalText).to.equal(expectedOriginalString); - expect(changeEvents.length).to.equal(expectedChanges.length); - for(let i=0; i { + const { text, expectedOriginalString, expectedChanges } = params + const { originalText, changeEvents } = parseTextAndGenerateChangeEvents(text) + expect(originalText).to.equal(expectedOriginalString) + expect(changeEvents.length).to.equal(expectedChanges.length) + for (let i = 0; i < changeEvents.length; i++) { + const changes = changeEvents.slice(0, i + 1) const newContent = applyTextDocumentChanges(originalText, changes) - expect(newContent).to.equal(expectedChanges[i], `Failed at index ${i}. Expected "${expectedChanges[i]}" but got "${newContent}"`); + expect(newContent).to.equal( + expectedChanges[i], + `Failed at index ${i}. Expected "${expectedChanges[i]}" but got "${newContent}"` + ) } } it('should handle insert markers correctly', () => { const text = 'This is a test string.' const expectedOriginalString = 'This is a string.' - const expectedChanges = [ - 'This is a test string.' - ] - testChanges({ text, expectedOriginalString, expectedChanges }); - }); + const expectedChanges = ['This is a test string.'] + testChanges({ text, expectedOriginalString, expectedChanges }) + }) it('should handle delete markers correctly', () => { const text = 'This is a sample string.' const expectedOriginalString = 'This is a sample string.' - const expectedChanges = [ - 'This is a string.' - ] - testChanges({ text, expectedOriginalString, expectedChanges }); - }); - + const expectedChanges = ['This is a string.'] + testChanges({ text, expectedOriginalString, expectedChanges }) + }) it('should handle replace markers correctly', () => { const text = 'Please replaceswap this word.' const expectedOriginalString = 'Please replace this word.' - const expectedChanges = [ - 'Please swap this word.' - ] - testChanges({ text, expectedOriginalString, expectedChanges }); - }); + const expectedChanges = ['Please swap this word.'] + testChanges({ text, expectedOriginalString, expectedChanges }) + }) it('should handle multiple markers correctly', () => { - const text = 'start and middle unnecessary text swap change end.'; - const expectedOriginalString = 'start middle unnecessary text swap end.'; + const text = 'start and middle unnecessary text swap change end.' + const expectedOriginalString = 'start middle unnecessary text swap end.' const expectedChanges = [ 'start and middle unnecessary text swap end.', 'start and middle text swap end.', - 'start and middle text change end.' + 'start and middle text change end.', ] - testChanges({ text, expectedOriginalString, expectedChanges }); - }); + testChanges({ text, expectedOriginalString, expectedChanges }) + }) it('should handle text without markers correctly', () => { - const text = 'This is plain text.'; - const expectedOriginalString = 'This is plain text.'; + const text = 'This is plain text.' + const expectedOriginalString = 'This is plain text.' const expectedChanges: string[] = [] - testChanges({ text, expectedOriginalString, expectedChanges }); - }); + testChanges({ text, expectedOriginalString, expectedChanges }) + }) it('should ignore unmatched markers', () => { - const inputText = 'Unmatched markers insert text without closing.'; - const { originalText, changeEvents } = parseTextAndGenerateChangeEvents(inputText); - - expect(originalText).to.equal('Unmatched markers insert text without closing.'); - expect(changeEvents).to.have.lengthOf(0); - }); + const inputText = 'Unmatched markers insert text without closing.' + const { originalText, changeEvents } = parseTextAndGenerateChangeEvents(inputText) + expect(originalText).to.equal('Unmatched markers insert text without closing.') + expect(changeEvents).to.have.lengthOf(0) + }) it('should handle complex multi-line text with mixed markers', () => { const text = dedent` @@ -135,20 +127,14 @@ describe('parseTextAndGenerateChangeEvents', () => { Fifth line addition End of text. `, - ]; - testChanges({ text, expectedOriginalString, expectedChanges }); - }); + ] + testChanges({ text, expectedOriginalString, expectedChanges }) + }) it('should handle continuous insert markers correctly', () => { const text = 'Hello World!' const expectedOriginalString = 'Hello !' - const expectedChanges = [ - 'Hello W!', - 'Hello Wo!', - 'Hello Wor!', - 'Hello Worl!', - 'Hello World!', - ] + const expectedChanges = ['Hello W!', 'Hello Wo!', 'Hello Wor!', 'Hello Worl!', 'Hello World!'] testChanges({ text, expectedOriginalString, expectedChanges }) }) @@ -180,55 +166,52 @@ describe('parseTextAndGenerateChangeEvents', () => { ] testChanges({ text, expectedOriginalString, expectedChanges }) }) - -}); - - +}) describe('processContinousChangesForText', () => { - it('should convert tags into individual tags for each character', () => { - const input = 'Hello World!'; - const expectedOutput = 'Hello World!'; - expect(processContinousChangesForText(input)).toBe(expectedOutput); - }); + const input = 'Hello World!' + const expectedOutput = 'Hello World!' + expect(processContinousChangesForText(input)).toBe(expectedOutput) + }) it('should convert tags into individual tags for each character', () => { - const input = 'Delete this text.'; - const expectedOutput = 'Delete this text.'; - expect(processContinousChangesForText(input)).toBe(expectedOutput); - }); + const input = 'Delete this text.' + const expectedOutput = 'Delete this text.' + expect(processContinousChangesForText(input)).toBe(expectedOutput) + }) it('should handle multiple and tags in the input', () => { - const input = 'Hello and Goodbye!'; - const expectedOutput = 'Hello and Goodbye!'; - expect(processContinousChangesForText(input)).toBe(expectedOutput); - }); + const input = 'Hello and Goodbye!' + const expectedOutput = + 'Hello and Goodbye!' + expect(processContinousChangesForText(input)).toBe(expectedOutput) + }) it('should return the same text if there are no or tags', () => { - const input = 'No changes here.'; - expect(processContinousChangesForText(input)).toBe(input); - }); + const input = 'No changes here.' + expect(processContinousChangesForText(input)).toBe(input) + }) it('should handle empty and tags gracefully', () => { - const input = 'Empty and tags.'; - const expectedOutput = 'Empty and tags.'; - expect(processContinousChangesForText(input)).toBe(expectedOutput); - }); + const input = 'Empty and tags.' + const expectedOutput = 'Empty and tags.' + expect(processContinousChangesForText(input)).toBe(expectedOutput) + }) it('should handle special characters within and tags', () => { - const input = 'Special chars: !@# and 123.'; - const expectedOutput = 'Special chars: !@# and 123.'; - expect(processContinousChangesForText(input)).toBe(expectedOutput); - }); + const input = 'Special chars: !@# and 123.' + const expectedOutput = + 'Special chars: !@# and 123.' + expect(processContinousChangesForText(input)).toBe(expectedOutput) + }) it('should handle consecutive and tags without text in between', () => { - const input = 'ABC123'; - const expectedOutput = 'ABC123'; - expect(processContinousChangesForText(input)).toBe(expectedOutput); - }); -}); - + const input = 'ABC123' + const expectedOutput = 'ABC123' + expect(processContinousChangesForText(input)).toBe(expectedOutput) + }) +}) describe('getPositionAt', () => { it('should return position at offset 0', () => { diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts index bcbc64c5d5ed..3d08c22b4016 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts @@ -1,13 +1,16 @@ -import * as vscode from 'vscode'; -import {getPositionAfterTextInsertion} from '../../../../text-processing/utils'; -import {applyTextDocumentChanges, TextDocumentChangeGroup} from './utils'; -import {createGitDiff} from '../../../../../../../lib/shared/src/editor/create-git-diff'; -import {TextDocumentChange} from './base'; - -export function getTextDocumentChangesForText(text: string): {originalText: string, changes: TextDocumentChange[]} { - const {originalText, changeEvents} = parseTextAndGenerateChangeEvents(text); +import * as vscode from 'vscode' +import { createGitDiff } from '../../../../../../../lib/shared/src/editor/create-git-diff' +import { getPositionAfterTextInsertion } from '../../../../text-processing/utils' +import type { TextDocumentChange } from './base' +import { type TextDocumentChangeGroup, applyTextDocumentChanges } from './utils' + +export function getTextDocumentChangesForText(text: string): { + originalText: string + changes: TextDocumentChange[] +} { + const { originalText, changeEvents } = parseTextAndGenerateChangeEvents(text) const documentChanges: TextDocumentChange[] = [] - for(const change of changeEvents) { + for (const change of changeEvents) { const insertedRange = new vscode.Range( change.range.start, getPositionAfterTextInsertion(change.range.start, change.text) @@ -15,25 +18,30 @@ export function getTextDocumentChangesForText(text: string): {originalText: stri documentChanges.push({ timestamp: Date.now(), change: change, - insertedRange + insertedRange, }) } - return {originalText, changes: documentChanges} + return { originalText, changes: documentChanges } } -export function getDiffsForContentChanges(oldContent: string, groupedChanges: TextDocumentChangeGroup[]): string[] { - const diffList: string[] = []; - let currentContent = oldContent; +export function getDiffsForContentChanges( + oldContent: string, + groupedChanges: TextDocumentChangeGroup[] +): string[] { + const diffList: string[] = [] + let currentContent = oldContent for (const changeGroup of groupedChanges) { - const newContent = applyTextDocumentChanges(currentContent, changeGroup.changes.map(change => change.change)); + const newContent = applyTextDocumentChanges( + currentContent, + changeGroup.changes.map(change => change.change) + ) const diff = createGitDiff('test.ts', currentContent, newContent) diffList.push(diff) - currentContent = newContent; + currentContent = newContent } - return diffList; + return diffList } - /** * The function is used by the test classes to simulate the text changes in a document text. * Parses the input text containing markers and generates the corresponding @@ -50,28 +58,29 @@ export function getDiffsForContentChanges(oldContent: string, groupedChanges: Te * @param text The input text containing markers. * @returns An object containing the original text and the array of change events. */ -export function parseTextAndGenerateChangeEvents( - text: string -): { originalText: string; changeEvents: vscode.TextDocumentContentChangeEvent[] } { - text = processContinousChangesForText(text); +export function parseTextAndGenerateChangeEvents(text: string): { + originalText: string + changeEvents: vscode.TextDocumentContentChangeEvent[] +} { + text = processContinousChangesForText(text) - const changeEvents: vscode.TextDocumentContentChangeEvent[] = []; - let originalText = ''; - let currentText = ''; - let currentOffset = 0; + const changeEvents: vscode.TextDocumentContentChangeEvent[] = [] + let originalText = '' + let currentText = '' + let currentOffset = 0 - const regex = /(.*?)<\/I>|(.*?)<\/D>|(.*?)(.*?)<\/R>/gs; - let match: RegExpExecArray | null; + const regex = /(.*?)<\/I>|(.*?)<\/D>|(.*?)(.*?)<\/R>/gs + let match: RegExpExecArray | null while ((match = regex.exec(text)) !== null) { - const [fullMatch, insertText, deleteText, replaceText1, replaceText2] = match; - const matchIndex = match.index; + const [fullMatch, insertText, deleteText, replaceText1, replaceText2] = match + const matchIndex = match.index - const textBeforeMarker = text.substring(currentOffset, matchIndex); - originalText += textBeforeMarker; - currentText += textBeforeMarker; + const textBeforeMarker = text.substring(currentOffset, matchIndex) + originalText += textBeforeMarker + currentText += textBeforeMarker - const position = getPositionAt(currentText, currentText.length); + const position = getPositionAt(currentText, currentText.length) if (insertText !== undefined) { changeEvents.push({ @@ -79,34 +88,34 @@ export function parseTextAndGenerateChangeEvents( rangeOffset: currentText.length, rangeLength: 0, text: insertText, - }); - currentText += insertText; + }) + currentText += insertText } else if (deleteText !== undefined) { - const deleteEndPosition = getPositionAfterTextInsertion(position, deleteText); + const deleteEndPosition = getPositionAfterTextInsertion(position, deleteText) changeEvents.push({ range: new vscode.Range(position, deleteEndPosition), rangeOffset: currentText.length, rangeLength: deleteText.length, text: '', - }); - originalText += deleteText; + }) + originalText += deleteText } else if (replaceText1 !== undefined && replaceText2 !== undefined) { - const replaceEndPosition = getPositionAfterTextInsertion(position, replaceText1); + const replaceEndPosition = getPositionAfterTextInsertion(position, replaceText1) changeEvents.push({ range: new vscode.Range(position, replaceEndPosition), rangeOffset: currentText.length, rangeLength: replaceText1.length, text: replaceText2, - }); - currentText += replaceText2; - originalText += replaceText1; + }) + currentText += replaceText2 + originalText += replaceText1 } - currentOffset = matchIndex + fullMatch.length; + currentOffset = matchIndex + fullMatch.length } - const remainingText = text.substring(currentOffset); - originalText += remainingText; - currentText += remainingText; - return { originalText, changeEvents }; + const remainingText = text.substring(currentOffset) + originalText += remainingText + currentText += remainingText + return { originalText, changeEvents } } /** @@ -119,15 +128,21 @@ export function parseTextAndGenerateChangeEvents( export function processContinousChangesForText(text: string): string { // Replace ... with individual ... markers for each character text = text.replace(/(.*?)<\/IC>/gs, (_, content) => { - return content.split('').map((char: string) => `${char}`).join(''); - }); + return content + .split('') + .map((char: string) => `${char}`) + .join('') + }) // Replace ... with individual ... markers for each character text = text.replace(/(.*?)<\/DC>/gs, (_, content) => { - return content.split('').map((char: string) => `${char}`).join(''); - }); + return content + .split('') + .map((char: string) => `${char}`) + .join('') + }) - return text; + return text } /** @@ -154,4 +169,3 @@ export function getPositionAt(content: string, offset: number): vscode.Position return new vscode.Position(line, character) } - diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts index 2dca4cd0ebe6..7efac8cfe845 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts @@ -1,29 +1,86 @@ import { PromptString } from '@sourcegraph/cody-shared' -import { describe, expect, it } from 'vitest' -import * as vscode from 'vscode' -import {applyTextDocumentChanges, computeDiffWithLineNumbers, doesLinesOverlapForRanges, groupConsecutiveItemsByPredicate, groupChangesForSimilarLinesTogether} from './utils'; import dedent from 'dedent' -import {getTextDocumentChangesForText, getDiffsForContentChanges} from './helper'; - - -const removeNewLineStringFromDiff = (text: string) => { +import { describe, expect, it } from 'vitest' +import type * as vscode from 'vscode' +import { getDiffsForContentChanges, getTextDocumentChangesForText } from './helper' +import { + applyTextDocumentChanges, + computeDiffWithLineNumbers, + doesLinesOverlapForRanges, + groupChangesForSimilarLinesTogether, + groupConsecutiveItemsByPredicate, +} from './utils' + +const processComputedDiff = (text: string) => { const lines = text.split('\n') - return lines.filter(line => !line.includes('\\ No newline at end of file')).join('\n') + const updatedText = lines.filter(line => !line.includes('\\ No newline at end of file')).join('\n') + return updatedText.split('\n').slice(3).join('\n') } -describe.only('documentChangeEventGroupings', () => { +describe('groupChangesForSimilarLinesTogether', () => { + + it('seperate line changes for non-continous changes on different lines', () => { + const text = dedent` + console.log('Hello, world!'); + data = 'check' + const a = 5; + ` + const { originalText, changes } = getTextDocumentChangesForText(text) + const result = groupChangesForSimilarLinesTogether(changes) + expect(result.length).toBe(3) + const diffs = getDiffsForContentChanges(originalText, result) + expect(processComputedDiff(diffs[0])).toMatchInlineSnapshot(` + "-console. + +console.log('Hello, world!'); + data = + const + " + `) + expect(processComputedDiff(diffs[1])).toMatchInlineSnapshot(` + " console.log('Hello, world!'); + -data = + +data = 'check' + const + " + `) + expect(processComputedDiff(diffs[2])).toMatchInlineSnapshot(` + " console.log('Hello, world!'); + data = 'check' + -const + +const a = 5; + " + `) + }) + + it('same line changes with non-continous character typing', () => { + const text = dedent` + console.log('Hello, world!'); + console.log('done') + const a = 5; + ` + const { originalText, changes } = getTextDocumentChangesForText(text) + const result = groupChangesForSimilarLinesTogether(changes) + expect(result.length).toBe(1) + const diffs = getDiffsForContentChanges(originalText, result) + expect(processComputedDiff(diffs[0])).toMatchInlineSnapshot(` + "-console.log + +console.log('Hello, world!'); + +console.log('done') + +const a = 5; + " + `) + }) - it('basic test case', () => { + it('continous character typing by the user', () => { const text = dedent` - console.log('Hello, world!'); - console.log('done') + console.log('Hello, world!'); + console.log('done') ` - const {originalText, changes} = getTextDocumentChangesForText(text); - const result = groupChangesForSimilarLinesTogether(changes); + const { originalText, changes } = getTextDocumentChangesForText(text) + const result = groupChangesForSimilarLinesTogether(changes) expect(result.length).toBe(1) - const diffs = getDiffsForContentChanges(originalText, result); - expect(diffs.length).toBe(1) - expect(removeNewLineStringFromDiff(diffs[0]).split('\n').slice(3).join('\n')).toMatchInlineSnapshot(` + const diffs = getDiffsForContentChanges(originalText, result) + expect(processComputedDiff(diffs[0])).toMatchInlineSnapshot(` "-console. +console.log('Hello, world!'); +console.log('done') @@ -191,10 +248,11 @@ describe('groupConsecutiveItemsByPredicate', () => { }) describe('computeDiffWithLineNumbers', () => { - const createTestUri = () => ({ - fsPath: '/path/to/file.ts', - toString: () => '/path/to/file.ts', - } as vscode.Uri) + const createTestUri = () => + ({ + fsPath: '/path/to/file.ts', + toString: () => '/path/to/file.ts', + }) as vscode.Uri const assertDiffResult = (result: any, expectedSnapshot: string) => { expect(result).toBeInstanceOf(PromptString) @@ -289,7 +347,3 @@ describe('computeDiffWithLineNumbers', () => { ) }) }) - - - - From 670e3c878b29a7561e91c491f9b9c6ac9017e05a Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Sat, 23 Nov 2024 07:16:53 +0530 Subject: [PATCH 09/18] diff --- .../recent-edits-diff-helpers/utils.test.ts | 98 ++++++++++++------- .../recent-edits-diff-helpers/utils.ts | 12 +-- 2 files changed, 71 insertions(+), 39 deletions(-) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts index 7efac8cfe845..2cdb911eda57 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts @@ -6,7 +6,6 @@ import { getDiffsForContentChanges, getTextDocumentChangesForText } from './help import { applyTextDocumentChanges, computeDiffWithLineNumbers, - doesLinesOverlapForRanges, groupChangesForSimilarLinesTogether, groupConsecutiveItemsByPredicate, } from './utils' @@ -19,6 +18,71 @@ const processComputedDiff = (text: string) => { describe('groupChangesForSimilarLinesTogether', () => { + it('handles multiple deletions across different lines', () => { + const text = dedent` + const a = 5; + console.log('test'); + const data = 5; + function test() { + return true; + } + ` + const { originalText, changes } = getTextDocumentChangesForText(text) + const result = groupChangesForSimilarLinesTogether(changes) + expect(result.length).toBe(2) + const diffs = getDiffsForContentChanges(originalText, result) + expect(processComputedDiff(diffs[0])).toMatchInlineSnapshot(` + " const a = 5; + -console.log('test'); + const data = 5; + function test() { + return true; + } + " + `) + expect(processComputedDiff(diffs[1])).toMatchInlineSnapshot(` + " const a = 5; + const data = 5; + -function test() { + - return true; + -} + " + `) + }) + + it('handles interleaved insertions and deletions', () => { + const text = dedent` + letconst x = 5; + varlet y = 10; + console.log(x +x * y); + ` + const { originalText, changes } = getTextDocumentChangesForText(text) + const result = groupChangesForSimilarLinesTogether(changes) + expect(result.length).toBe(3) + const diffs = getDiffsForContentChanges(originalText, result) + expect(processComputedDiff(diffs[0])).toMatchInlineSnapshot(` + "-let x = 5; + +const x = 5; + var y = 10; + console.log(x + y); + " + `) + }) + + it('handles overlapping multi-line changes', () => { + const text = dedent` + function test() { + const x = 5; + if (true) { + console.log(x); + } + } + ` + const { changes } = getTextDocumentChangesForText(text) + const result = groupChangesForSimilarLinesTogether(changes) + expect(result.length).toBe(2) + }) + it('seperate line changes for non-continous changes on different lines', () => { const text = dedent` console.log('Hello, world!'); @@ -144,38 +208,6 @@ describe('applyTextDocumentChanges', () => { }) }) -describe('doesLinesOverlapForRanges', () => { - const createRange = (startLine: number, endLine: number): vscode.Range => - ({ - start: { line: startLine, character: 0 }, - end: { line: endLine, character: 0 }, - }) as vscode.Range - - it('should detect overlapping ranges', () => { - const rangeA = createRange(1, 5) - const rangeB = createRange(3, 7) - expect(doesLinesOverlapForRanges(rangeA, rangeB)).toBe(true) - }) - - it('should detect adjacent non-overlapping ranges', () => { - const rangeA = createRange(1, 3) - const rangeB = createRange(4, 6) - expect(doesLinesOverlapForRanges(rangeA, rangeB)).toBe(false) - }) - - it('should detect contained ranges', () => { - const rangeA = createRange(1, 10) - const rangeB = createRange(3, 5) - expect(doesLinesOverlapForRanges(rangeA, rangeB)).toBe(true) - }) - - it('should detect ranges touching at endpoints', () => { - const rangeA = createRange(1, 3) - const rangeB = createRange(3, 5) - expect(doesLinesOverlapForRanges(rangeA, rangeB)).toBe(true) - }) -}) - describe('groupConsecutiveItemsByPredicate', () => { it('should return empty array when given an empty array', () => { const result = groupConsecutiveItemsByPredicate([], (a, b) => a === b) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts index f566e4c387c5..164ac23618e5 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts @@ -52,8 +52,12 @@ export function groupChangesForSimilarLinesTogether( changes, (lastChange: TextDocumentChange, change: TextDocumentChange) => { return ( - doesLinesOverlapForRanges(lastChange.change.range, change.change.range) || - doesLinesOverlapForRanges(lastChange.insertedRange, change.change.range) + doLineSpansOverlap( + lastChange.insertedRange.start.line, + lastChange.insertedRange.end.line, + change.change.range.start.line, + change.change.range.end.line + ) ) } ) @@ -205,10 +209,6 @@ export function applyTextDocumentChanges( return content } -export function doesLinesOverlapForRanges(a: vscode.Range, b: vscode.Range): boolean { - return doLineSpansOverlap(a.start.line, a.end.line, b.start.line, b.end.line) -} - function doLineSpansOverlap( firstStart: number, firstEnd: number, From b7f39a7e8fd469927d3301ed4a66c95425acf262 Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Sat, 23 Nov 2024 20:12:04 +0530 Subject: [PATCH 10/18] logic fix and test cases fix --- .editorconfig | 2 +- vscode/src/chat/chat-view/chat-helpers.ts | 8 +++ .../auotedit-short-term-diff.ts | 39 +++++++++++++- .../recent-edits-diff-helpers/utils.test.ts | 52 +++++++++++++++---- .../recent-edits-diff-helpers/utils.ts | 51 +++++++++--------- 5 files changed, 114 insertions(+), 38 deletions(-) diff --git a/.editorconfig b/.editorconfig index 4d582a0f31fc..69dfbc863399 100644 --- a/.editorconfig +++ b/.editorconfig @@ -4,7 +4,7 @@ root = true insert_final_newline = true end_of_line = lf charset = utf-8 -trim_trailing_whitespace = true +trim_trailing_whitespace = false indent_style = space indent_size = 4 diff --git a/vscode/src/chat/chat-view/chat-helpers.ts b/vscode/src/chat/chat-view/chat-helpers.ts index dcc22f013b83..866290776043 100644 --- a/vscode/src/chat/chat-view/chat-helpers.ts +++ b/vscode/src/chat/chat-view/chat-helpers.ts @@ -13,3 +13,11 @@ export function getChatPanelTitle(lastHumanText?: string, truncateTitle = true): // truncate title that is too long return text.length > 25 ? `${text.slice(0, 25).trim()}...` : text } + +export function chatHelper(hello: string): string { + return hello +} + +export function chatHelper2(message: string): string { + return message +} diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts index 326127b1202c..5b175c92f84a 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts @@ -22,10 +22,19 @@ export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDi public getDiffHunks(input: DiffCalculationInput): DiffHunk[] { const rawChanges = groupChangesForSimilarLinesTogether(input.changes) + const rawDiffHunks = this.getDiffHunksFromGroupedChanges(input, rawChanges) const changes = combineNonOverlappingLinesSchemaTogether(rawChanges) - this.logGroupedChanges(input.uri, input.oldContent, changes) - const allDiffHunks: DiffHunk[] = [] + const combinedDiffHunks = this.getDiffHunksFromGroupedChanges(input, changes) + + this.logRawDataPoints(input.uri.toString(), input.oldContent, rawDiffHunks, combinedDiffHunks) + return combinedDiffHunks + } + private getDiffHunksFromGroupedChanges( + input: DiffCalculationInput, + changes: TextDocumentChangeGroup[] + ): DiffHunk[] { + const allDiffHunks: DiffHunk[] = [] let oldContent = input.oldContent for (const changeList of changes) { const [diffHunk, newContent] = this.getDiffHunksForChanges( @@ -40,6 +49,32 @@ export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDi return allDiffHunks } + private logRawDataPoints(uri: string, oldContent: string, rawDiffHunks: DiffHunk[], combinedDiffHunks: DiffHunk[]) { + const dirPath = '/Users/hiteshsagtani/Desktop/raw-diff-logs' + const fileName = uri.split('/').pop()?.split('.')[0] || 'document' + const logPath = uri.replace(/[^/\\]+$/, `${fileName}_raw.jsonl`) + const finalLogPath = path.join(dirPath, path.basename(logPath)) + const fs = require('fs') + + // Create directory if it doesn't exist + if (!fs.existsSync(dirPath)) { + fs.mkdirSync(dirPath, { recursive: true }) + } + + const logData = { + uri: uri.toString(), + oldContent, + rawDiffHunks, + combinedDiffHunks + } + // Append to file if it exists, create if it doesn't + fs.appendFileSync( + finalLogPath, + JSON.stringify(logData) + '\n', + { encoding: 'utf8' } + ) + } + private logGroupedChanges(uri: vscode.Uri, oldContent: string, changes: TextDocumentChangeGroup[]) { const fileName = uri.fsPath.split('/').pop()?.split('.')[0] || 'document' const logPath = uri.fsPath.replace(/[^/\\]+$/, `${fileName}_grouped.json`) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts index 2cdb911eda57..3f1979cdca20 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts @@ -3,12 +3,7 @@ import dedent from 'dedent' import { describe, expect, it } from 'vitest' import type * as vscode from 'vscode' import { getDiffsForContentChanges, getTextDocumentChangesForText } from './helper' -import { - applyTextDocumentChanges, - computeDiffWithLineNumbers, - groupChangesForSimilarLinesTogether, - groupConsecutiveItemsByPredicate, -} from './utils' +import {applyTextDocumentChanges, computeDiffWithLineNumbers, groupChangesForSimilarLinesTogether, groupConsecutiveItemsByPredicate, combineNonOverlappingLinesSchemaTogether} from './utils'; const processComputedDiff = (text: string) => { const lines = text.split('\n') @@ -16,7 +11,7 @@ const processComputedDiff = (text: string) => { return updatedText.split('\n').slice(3).join('\n') } -describe('groupChangesForSimilarLinesTogether', () => { +describe('groupChangesForLines', () => { it('handles multiple deletions across different lines', () => { const text = dedent` @@ -48,6 +43,19 @@ describe('groupChangesForSimilarLinesTogether', () => { -} " `) + const combinedChanges = combineNonOverlappingLinesSchemaTogether(result) + expect(combinedChanges.length).toBe(1) + const combinedDiffs = getDiffsForContentChanges(originalText, combinedChanges) + expect(processComputedDiff(combinedDiffs[0])).toMatchInlineSnapshot(` + " const a = 5; + -console.log('test'); + const data = 5; + -function test() { + - return true; + -} + " + `) + }) it('handles interleaved insertions and deletions', () => { @@ -74,13 +82,27 @@ describe('groupChangesForSimilarLinesTogether', () => { function test() { const x = 5; if (true) { - console.log(x); + console.log(x); } } ` - const { changes } = getTextDocumentChangesForText(text) + const { originalText, changes } = getTextDocumentChangesForText(text) const result = groupChangesForSimilarLinesTogether(changes) expect(result.length).toBe(2) + const combinedChanges = combineNonOverlappingLinesSchemaTogether(result) + expect(combinedChanges.length).toBe(1) + const combinedDiffs = getDiffsForContentChanges(originalText, combinedChanges) + expect(processComputedDiff(combinedDiffs[0])).toMatchInlineSnapshot(` + " function test() { + - + - + + const x = 5; + + if (true) { + + console.log(x); + + } + } + " + `) }) it('seperate line changes for non-continous changes on different lines', () => { @@ -114,6 +136,18 @@ describe('groupChangesForSimilarLinesTogether', () => { +const a = 5; " `) + const combinedChanges = combineNonOverlappingLinesSchemaTogether(result) + expect(combinedChanges.length).toBe(1) + const combinedDiffs = getDiffsForContentChanges(originalText, combinedChanges) + expect(processComputedDiff(combinedDiffs[0])).toMatchInlineSnapshot(` + "-console. + -data = + -const + +console.log('Hello, world!'); + +data = 'check' + +const a = 5; + " + `) }) it('same line changes with non-continous character typing', () => { diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts index 164ac23618e5..25831bd4f4ca 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts @@ -1,11 +1,11 @@ import { PromptString } from '@sourcegraph/cody-shared' import { displayPath } from '@sourcegraph/cody-shared/src/editor/displayPath' import { structuredPatch } from 'diff' -import type * as vscode from 'vscode' +import * as vscode from 'vscode' import type { TextDocumentChange } from './base' /** - * Represents a group of text document changes with their line range information. + * Represents a group of text document changes with their range information. * The grouped changes are consecutive changes made in the document that should be treated as a single entity when computing diffs. * * @example @@ -17,14 +17,14 @@ export interface TextDocumentChangeGroup { changes: TextDocumentChange[] /** - * The starting line number of the changes in this group + * The union of the inserted ranges of all changes in this group */ - changeStartLine: number + insertRange: vscode.Range /** - * The ending line number of the changes in this group + * The union of the replace ranges of all changes in this group */ - changeEndLine: number + replaceRange: vscode.Range } /** @@ -62,11 +62,10 @@ export function groupChangesForSimilarLinesTogether( } ) return groupedChanges.map(currentGroup => { - const range = getMinMaxRangeLines(currentGroup) return { changes: currentGroup, - changeStartLine: range[0], - changeEndLine: range[1], + insertRange: getRangeUnion(currentGroup.map(change => change.insertedRange)), + replaceRange: getRangeUnion(currentGroup.map(change => change.change.range)), } }) } @@ -96,33 +95,33 @@ export function combineNonOverlappingLinesSchemaTogether( } const combinedGroups = groupConsecutiveItemsByPredicate( groupedChanges, - (a: TextDocumentChangeGroup, b: TextDocumentChangeGroup) => { + (lastChange: TextDocumentChangeGroup, change: TextDocumentChangeGroup) => { return !doLineSpansOverlap( - a.changeStartLine, - a.changeEndLine, - b.changeStartLine, - b.changeEndLine + lastChange.insertRange.start.line, + lastChange.insertRange.end.line, + change.replaceRange.start.line, + change.replaceRange.end.line ) } ) return combinedGroups.map(changes => ({ changes: changes.flatMap(change => change.changes), - changeStartLine: Math.min(...changes.map(change => change.changeStartLine)), - changeEndLine: Math.max(...changes.map(change => change.changeEndLine)), + insertRange: getRangeUnion(changes.map(change => change.insertRange)), + replaceRange: getRangeUnion(changes.map(change => change.replaceRange)), })) } -function getMinMaxRangeLines(documentChanges: TextDocumentChange[]): [number, number] { - let minLine = Number.POSITIVE_INFINITY - let maxLine = Number.NEGATIVE_INFINITY - for (const change of documentChanges) { - const ranges = [change.change.range, change.insertedRange] - for (const range of ranges) { - minLine = Math.min(minLine, range.start.line) - maxLine = Math.max(maxLine, range.end.line) - } +function getRangeUnion(ranges: vscode.Range[]): vscode.Range { + if (ranges.length === 0) { + throw new Error('Cannot get union of empty ranges') + } + let start = ranges[0].start + let end = ranges[0].end + for (const range of ranges) { + start = start.isBefore(range.start) ? start : range.start + end = end.isAfter(range.end) ? end : range.end } - return [minLine, maxLine] + return new vscode.Range(start, end) } /** From 9da6dc4edc9d80d6c30cd194f817347148f47793 Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Sat, 23 Nov 2024 21:18:18 +0530 Subject: [PATCH 11/18] cleanup --- .editorconfig | 2 +- vscode/src/autoedits/autoedits-provider.ts | 10 +- vscode/src/autoedits/prompt-utils.ts | 9 +- vscode/src/chat/chat-view/chat-helpers.ts | 8 -- .../auotedit-short-term-diff.test.ts | 45 ------- .../auotedit-short-term-diff.ts | 40 +++---- .../recent-edits-diff-helpers/helper.ts | 5 +- .../recent-edits-diff-helpers/utils.test.ts | 36 +++--- .../recent-edits-diff-helpers/utils.ts | 111 +++++++++--------- .../recent-edits-retriever.ts | 10 -- 10 files changed, 98 insertions(+), 178 deletions(-) delete mode 100644 vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.test.ts diff --git a/.editorconfig b/.editorconfig index 69dfbc863399..4d582a0f31fc 100644 --- a/.editorconfig +++ b/.editorconfig @@ -4,7 +4,7 @@ root = true insert_final_newline = true end_of_line = lf charset = utf-8 -trim_trailing_whitespace = false +trim_trailing_whitespace = true indent_style = space indent_size = 4 diff --git a/vscode/src/autoedits/autoedits-provider.ts b/vscode/src/autoedits/autoedits-provider.ts index 7fc96f062758..dd605a47247d 100644 --- a/vscode/src/autoedits/autoedits-provider.ts +++ b/vscode/src/autoedits/autoedits-provider.ts @@ -233,7 +233,7 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v document.lineAt(position).range.end ) ) - // autoeditsLogger.logDebug('Autocomplete Inline Response: ', autocompleteResponse) + autoeditsLogger.logDebug('Autocomplete Inline Response: ', autocompleteResponse) return [inlineCompletionItem] } return null @@ -303,10 +303,10 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v suffix: codeToReplaceData.suffixInArea + codeToReplaceData.suffixAfterArea, }) ) { - // autoeditsLogger.logDebug( - // 'Autoedits', - // 'Skipping autoedit - predicted text already exists in suffix' - // ) + autoeditsLogger.logDebug( + 'Autoedits', + 'Skipping autoedit - predicted text already exists in suffix' + ) return } await this.rendererManager.showEdit({ diff --git a/vscode/src/autoedits/prompt-utils.ts b/vscode/src/autoedits/prompt-utils.ts index 2e35db2a865c..4287e2c40c0a 100644 --- a/vscode/src/autoedits/prompt-utils.ts +++ b/vscode/src/autoedits/prompt-utils.ts @@ -140,8 +140,7 @@ ${recentCopyPrompt} ${areaPrompt} ${FINAL_USER_PROMPT} ` - // autoeditsLogger.logDebug('AutoEdits', 'Prompt\n', finalPrompt) - autoeditsLogger.logDebug('AutoEdits', 'Diff Values\n', recentEditsPrompt) + autoeditsLogger.logDebug('AutoEdits', 'Prompt\n', finalPrompt) return { codeToReplace: codeToReplace, prompt: finalPrompt, @@ -334,7 +333,7 @@ export function getRecentEditsPrompt(contextItems: AutocompleteContextSnippet[]) return ps`` } const recentEditsPrompts = recentEdits.map(item => - getContextPromptForDiffPrompt( + getContextPromptWithPath( PromptString.fromDisplayPath(item.uri), PromptString.fromAutocompleteContextSnippet(item).content ) @@ -456,7 +455,3 @@ function getContextItemsForIdentifier( function getContextPromptWithPath(filePath: PromptString, content: PromptString): PromptString { return ps`(\`${filePath}\`)\n\n${content}\n` } - -function getContextPromptForDiffPrompt(filePath: PromptString, content: PromptString): PromptString { - return ps`${filePath}\n${content}` -} diff --git a/vscode/src/chat/chat-view/chat-helpers.ts b/vscode/src/chat/chat-view/chat-helpers.ts index 866290776043..dcc22f013b83 100644 --- a/vscode/src/chat/chat-view/chat-helpers.ts +++ b/vscode/src/chat/chat-view/chat-helpers.ts @@ -13,11 +13,3 @@ export function getChatPanelTitle(lastHumanText?: string, truncateTitle = true): // truncate title that is too long return text.length > 25 ? `${text.slice(0, 25).trim()}...` : text } - -export function chatHelper(hello: string): string { - return hello -} - -export function chatHelper2(message: string): string { - return message -} diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.test.ts deleted file mode 100644 index 0e23f6a26aec..000000000000 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -// import { describe, expect, it } from 'vitest' -// import { Uri } from 'vscode' -// import { range } from '../../../../../testutils/textDocument' -// import { AutoeditWithShortTermDiffStrategy } from './auotedit-short-term-diff' -// import type { TextDocumentChange } from './base' - -// describe('AutoeditWithShortTermDiffStrategy', () => { -// const strategy = new AutoeditWithShortTermDiffStrategy() -// const mockUri = Uri.parse('file:///test.txt') - -// const createChange = (timestamp: number, oldText: string, text: string) => ({ -// timestamp, -// change: { -// range: range(0, 0, 0, 0), -// text, -// rangeLength: oldText.length, -// rangeOffset: 0, -// }, -// }) - -// it('should divide changes into short-term and long-term windows', () => { -// const now = Date.now() -// const initialContent = 'initial content' -// const changes: TextDocumentChange[] = [ -// createChange(now - 10000, initialContent, 'change 1'), -// createChange(now - 2000, 'change 1', 'change 2'), -// ] - -// const hunks = strategy.getDiffHunks({ -// uri: mockUri, -// oldContent: initialContent, -// changes, -// }) - -// expect(hunks).toHaveLength(2) -// expect(hunks[0].diff.toString()).toMatchInlineSnapshot(` -// "1-| initial content -// 1+| change 1" -// `) -// expect(hunks[1].diff.toString()).toMatchInlineSnapshot(` -// "1-| change 1 -// 1+| change 2" -// `) -// }) -// }) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts index 5b175c92f84a..a223530b5899 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts @@ -1,3 +1,4 @@ +import fs from 'node:fs' import path from 'node:path' import type * as vscode from 'vscode' import type { @@ -9,9 +10,10 @@ import type { import { applyTextDocumentChanges, computeDiffWithLineNumbers, - groupChangesForSimilarLinesTogether, + groupNonOverlappingChangeGroups, + groupOverlappingDocumentChanges, } from './utils' -import { type TextDocumentChangeGroup, combineNonOverlappingLinesSchemaTogether } from './utils' +import type { TextDocumentChangeGroup } from './utils' /** * Generates a single unified diff patch that combines all changes @@ -21,11 +23,10 @@ export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDi private shortTermContextLines = 0 public getDiffHunks(input: DiffCalculationInput): DiffHunk[] { - const rawChanges = groupChangesForSimilarLinesTogether(input.changes) + const rawChanges = groupOverlappingDocumentChanges(input.changes) const rawDiffHunks = this.getDiffHunksFromGroupedChanges(input, rawChanges) - const changes = combineNonOverlappingLinesSchemaTogether(rawChanges) + const changes = groupNonOverlappingChangeGroups(rawChanges) const combinedDiffHunks = this.getDiffHunksFromGroupedChanges(input, changes) - this.logRawDataPoints(input.uri.toString(), input.oldContent, rawDiffHunks, combinedDiffHunks) return combinedDiffHunks } @@ -49,12 +50,16 @@ export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDi return allDiffHunks } - private logRawDataPoints(uri: string, oldContent: string, rawDiffHunks: DiffHunk[], combinedDiffHunks: DiffHunk[]) { + private logRawDataPoints( + uri: string, + oldContent: string, + rawDiffHunks: DiffHunk[], + combinedDiffHunks: DiffHunk[] + ) { const dirPath = '/Users/hiteshsagtani/Desktop/raw-diff-logs' const fileName = uri.split('/').pop()?.split('.')[0] || 'document' const logPath = uri.replace(/[^/\\]+$/, `${fileName}_raw.jsonl`) const finalLogPath = path.join(dirPath, path.basename(logPath)) - const fs = require('fs') // Create directory if it doesn't exist if (!fs.existsSync(dirPath)) { @@ -65,27 +70,10 @@ export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDi uri: uri.toString(), oldContent, rawDiffHunks, - combinedDiffHunks + combinedDiffHunks, } // Append to file if it exists, create if it doesn't - fs.appendFileSync( - finalLogPath, - JSON.stringify(logData) + '\n', - { encoding: 'utf8' } - ) - } - - private logGroupedChanges(uri: vscode.Uri, oldContent: string, changes: TextDocumentChangeGroup[]) { - const fileName = uri.fsPath.split('/').pop()?.split('.')[0] || 'document' - const logPath = uri.fsPath.replace(/[^/\\]+$/, `${fileName}_grouped.json`) - const finalLogPath = path.join('/Users/hiteshsagtani/Desktop/diff-logs', path.basename(logPath)) - const fs = require('fs') - const logData = { - uri: uri.toString(), - oldContent: oldContent, - changes: changes.map(c => c.changes), - } - fs.writeFileSync(finalLogPath, JSON.stringify(logData, null, 2)) + fs.appendFileSync(finalLogPath, JSON.stringify(logData) + '\n', { encoding: 'utf8' }) } private getDiffHunksForChanges( diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts index 3d08c22b4016..4a0e5a09fc78 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/helper.ts @@ -68,11 +68,11 @@ export function parseTextAndGenerateChangeEvents(text: string): { let originalText = '' let currentText = '' let currentOffset = 0 - const regex = /(.*?)<\/I>|(.*?)<\/D>|(.*?)(.*?)<\/R>/gs let match: RegExpExecArray | null - while ((match = regex.exec(text)) !== null) { + match = regex.exec(text) + while (match !== null) { const [fullMatch, insertText, deleteText, replaceText1, replaceText2] = match const matchIndex = match.index @@ -111,6 +111,7 @@ export function parseTextAndGenerateChangeEvents(text: string): { originalText += replaceText1 } currentOffset = matchIndex + fullMatch.length + match = regex.exec(text) } const remainingText = text.substring(currentOffset) originalText += remainingText diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts index 3f1979cdca20..bc2a138e2b80 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts @@ -3,7 +3,13 @@ import dedent from 'dedent' import { describe, expect, it } from 'vitest' import type * as vscode from 'vscode' import { getDiffsForContentChanges, getTextDocumentChangesForText } from './helper' -import {applyTextDocumentChanges, computeDiffWithLineNumbers, groupChangesForSimilarLinesTogether, groupConsecutiveItemsByPredicate, combineNonOverlappingLinesSchemaTogether} from './utils'; +import { + applyTextDocumentChanges, + computeDiffWithLineNumbers, + groupConsecutiveItemsByPredicate, + groupNonOverlappingChangeGroups, + groupOverlappingDocumentChanges, +} from './utils' const processComputedDiff = (text: string) => { const lines = text.split('\n') @@ -12,7 +18,6 @@ const processComputedDiff = (text: string) => { } describe('groupChangesForLines', () => { - it('handles multiple deletions across different lines', () => { const text = dedent` const a = 5; @@ -23,7 +28,7 @@ describe('groupChangesForLines', () => { } ` const { originalText, changes } = getTextDocumentChangesForText(text) - const result = groupChangesForSimilarLinesTogether(changes) + const result = groupOverlappingDocumentChanges(changes) expect(result.length).toBe(2) const diffs = getDiffsForContentChanges(originalText, result) expect(processComputedDiff(diffs[0])).toMatchInlineSnapshot(` @@ -43,7 +48,7 @@ describe('groupChangesForLines', () => { -} " `) - const combinedChanges = combineNonOverlappingLinesSchemaTogether(result) + const combinedChanges = groupNonOverlappingChangeGroups(result) expect(combinedChanges.length).toBe(1) const combinedDiffs = getDiffsForContentChanges(originalText, combinedChanges) expect(processComputedDiff(combinedDiffs[0])).toMatchInlineSnapshot(` @@ -55,7 +60,6 @@ describe('groupChangesForLines', () => { -} " `) - }) it('handles interleaved insertions and deletions', () => { @@ -65,7 +69,7 @@ describe('groupChangesForLines', () => { console.log(x +x * y); ` const { originalText, changes } = getTextDocumentChangesForText(text) - const result = groupChangesForSimilarLinesTogether(changes) + const result = groupOverlappingDocumentChanges(changes) expect(result.length).toBe(3) const diffs = getDiffsForContentChanges(originalText, result) expect(processComputedDiff(diffs[0])).toMatchInlineSnapshot(` @@ -80,22 +84,22 @@ describe('groupChangesForLines', () => { it('handles overlapping multi-line changes', () => { const text = dedent` function test() { - const x = 5; + const x = 5; if (true) { - console.log(x); + console.log(x); } } ` const { originalText, changes } = getTextDocumentChangesForText(text) - const result = groupChangesForSimilarLinesTogether(changes) + const result = groupOverlappingDocumentChanges(changes) expect(result.length).toBe(2) - const combinedChanges = combineNonOverlappingLinesSchemaTogether(result) + const combinedChanges = groupNonOverlappingChangeGroups(result) expect(combinedChanges.length).toBe(1) const combinedDiffs = getDiffsForContentChanges(originalText, combinedChanges) expect(processComputedDiff(combinedDiffs[0])).toMatchInlineSnapshot(` " function test() { - - - - + - + - + const x = 5; + if (true) { + console.log(x); @@ -112,7 +116,7 @@ describe('groupChangesForLines', () => { const a = 5; ` const { originalText, changes } = getTextDocumentChangesForText(text) - const result = groupChangesForSimilarLinesTogether(changes) + const result = groupOverlappingDocumentChanges(changes) expect(result.length).toBe(3) const diffs = getDiffsForContentChanges(originalText, result) expect(processComputedDiff(diffs[0])).toMatchInlineSnapshot(` @@ -136,7 +140,7 @@ describe('groupChangesForLines', () => { +const a = 5; " `) - const combinedChanges = combineNonOverlappingLinesSchemaTogether(result) + const combinedChanges = groupNonOverlappingChangeGroups(result) expect(combinedChanges.length).toBe(1) const combinedDiffs = getDiffsForContentChanges(originalText, combinedChanges) expect(processComputedDiff(combinedDiffs[0])).toMatchInlineSnapshot(` @@ -157,7 +161,7 @@ describe('groupChangesForLines', () => { const a = 5; ` const { originalText, changes } = getTextDocumentChangesForText(text) - const result = groupChangesForSimilarLinesTogether(changes) + const result = groupOverlappingDocumentChanges(changes) expect(result.length).toBe(1) const diffs = getDiffsForContentChanges(originalText, result) expect(processComputedDiff(diffs[0])).toMatchInlineSnapshot(` @@ -175,7 +179,7 @@ describe('groupChangesForLines', () => { console.log('done') ` const { originalText, changes } = getTextDocumentChangesForText(text) - const result = groupChangesForSimilarLinesTogether(changes) + const result = groupOverlappingDocumentChanges(changes) expect(result.length).toBe(1) const diffs = getDiffsForContentChanges(originalText, result) expect(processComputedDiff(diffs[0])).toMatchInlineSnapshot(` diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts index 25831bd4f4ca..ce45c056e083 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts @@ -19,96 +19,91 @@ export interface TextDocumentChangeGroup { /** * The union of the inserted ranges of all changes in this group */ - insertRange: vscode.Range + insertedRange: vscode.Range /** * The union of the replace ranges of all changes in this group */ - replaceRange: vscode.Range + replacementRange: vscode.Range } /** * Groups consecutive text document changes together based on line overlap. * This function helps create more meaningful diffs by combining related changes that occur on overlapping lines. * - * For example, when a user types multiple characters or performs multiple edits in the same area of text, + * For example, when a user types multiple characters or performs multiple edits in the same lines of text, * these changes are grouped together as a single logical change instead of being treated as separate changes. * - * @param changes - Array of individual text document changes to be grouped + * @param documentChanges - Array of individual text document changes to be grouped * @returns Array of TextDocumentChangeGroup objects, each containing related changes and their combined line range - * - * The predicate used for grouping checks if: - * - The original ranges of two changes overlap (for modifications/deletions) - * - The inserted range of the first change overlaps with the original range of the second change - * This ensures that changes affecting the same or adjacent lines are grouped together. */ -export function groupChangesForSimilarLinesTogether( - changes: TextDocumentChange[] +export function groupOverlappingDocumentChanges( + documentChanges: TextDocumentChange[] ): TextDocumentChangeGroup[] { - if (changes.length === 0) { - return [] - } - const groupedChanges = groupConsecutiveItemsByPredicate( - changes, - (lastChange: TextDocumentChange, change: TextDocumentChange) => { - return ( - doLineSpansOverlap( - lastChange.insertedRange.start.line, - lastChange.insertedRange.end.line, - change.change.range.start.line, - change.change.range.end.line - ) - ) - } - ) - return groupedChanges.map(currentGroup => { - return { - changes: currentGroup, - insertRange: getRangeUnion(currentGroup.map(change => change.insertedRange)), - replaceRange: getRangeUnion(currentGroup.map(change => change.change.range)), - } + return mergeDocumentChanges({ + items: documentChanges.map(change => ({ + insertedRange: change.insertedRange, + replacementRange: change.change.range, + originalChange: change, + })), + mergePredicate: (a, b) => doLineSpansOverlap(a.start.line, a.end.line, b.start.line, b.end.line), + getChanges: item => [item.originalChange], }) } /** * Combines consecutive text document change groups that have non-overlapping line ranges. - * The function can generally be called after `groupChangesForSimilarLinesTogether` to further consolidate changes. + * The function can generally be called after `groupOverlappingDocumentChanges` to further consolidate changes. * * This function takes an array of `TextDocumentChangeGroup` objects and merges consecutive groups * where their line ranges do not overlap. By combining these non-overlapping groups, it creates * larger groups of changes that can be processed together, even if they affect different parts - * of the document, as long as they occurred consecutively. + * of the document. * * @param groupedChanges - Array of `TextDocumentChangeGroup` objects to be combined. * @returns Array of `TextDocumentChangeGroup` objects where consecutive non-overlapping groups have been merged. - * - * The predicate used for grouping checks if: - * - The line ranges of two groups do not overlap. - * - Specifically, it checks that `a.changeStartLine` to `a.changeEndLine` does not overlap with `b.changeStartLine` to `b.changeEndLine`. - * This ensures that consecutive groups with non-overlapping line ranges are combined together. */ -export function combineNonOverlappingLinesSchemaTogether( +export function groupNonOverlappingChangeGroups( groupedChanges: TextDocumentChangeGroup[] ): TextDocumentChangeGroup[] { - if (groupedChanges.length === 0) { + return mergeDocumentChanges({ + items: groupedChanges, + mergePredicate: (a, b) => + !doLineSpansOverlap(a.start.line, a.end.line, b.start.line, b.end.line), + getChanges: group => group.changes, + }) +} + +/** + * Merges document changes based on a predicate and extracts changes using a provided function. + * + * @param items - Array of objects containing insertedRange and replacementRange properties + * @param mergePredicate - Function that determines if two ranges should be merged + * @param getChanges - Function that extracts TextDocumentChange array from an item + * @returns Array of TextDocumentChangeGroup objects containing merged changes and their ranges + */ +function mergeDocumentChanges< + T extends { insertedRange: vscode.Range; replacementRange: vscode.Range }, +>(args: { + items: T[] + mergePredicate: (a: vscode.Range, b: vscode.Range) => boolean + getChanges: (item: T) => TextDocumentChange[] +}): TextDocumentChangeGroup[] { + if (args.items.length === 0) { return [] } - const combinedGroups = groupConsecutiveItemsByPredicate( - groupedChanges, - (lastChange: TextDocumentChangeGroup, change: TextDocumentChangeGroup) => { - return !doLineSpansOverlap( - lastChange.insertRange.start.line, - lastChange.insertRange.end.line, - change.replaceRange.start.line, - change.replaceRange.end.line - ) - } - ) - return combinedGroups.map(changes => ({ - changes: changes.flatMap(change => change.changes), - insertRange: getRangeUnion(changes.map(change => change.insertRange)), - replaceRange: getRangeUnion(changes.map(change => change.replaceRange)), - })) + + const mergedGroups = groupConsecutiveItemsByPredicate(args.items, (lastItem, currentItem) => { + return args.mergePredicate(lastItem.insertedRange, currentItem.replacementRange) + }) + + return mergedGroups + .filter(group => group.length > 0) + .map(group => ({ + changes: group.flatMap(item => args.getChanges(item)), + insertedRange: getRangeUnion(group.map(item => item.insertedRange)), + replacementRange: getRangeUnion(group.map(item => item.replacementRange)), + })) } function getRangeUnion(ranges: vscode.Range[]): vscode.Range { diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts index 2b68da39c6dd..0e2a2737c0ce 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts @@ -1,4 +1,3 @@ -import path from 'node:path' import { type PromptString, contextFiltersProvider } from '@sourcegraph/cody-shared' import type { AutocompleteContextSnippet } from '@sourcegraph/cody-shared' import * as vscode from 'vscode' @@ -176,15 +175,6 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever }) } this.reconcileOutdatedChanges() - this.logTextDocument(trackedDocument) - } - - private logTextDocument(trackedDocument: TrackedDocument): void { - const fileName = trackedDocument.uri.fsPath.split('/').pop()?.split('.')[0] || 'document' - const logPath = trackedDocument.uri.fsPath.replace(/[^/\\]+$/, `${fileName}.json`) - const finalLogPath = path.join('/Users/hiteshsagtani/Desktop/diff-logs', path.basename(logPath)) - const fs = require('fs') - fs.writeFileSync(finalLogPath, JSON.stringify(trackedDocument, null, 2)) } private onDidOpenTextDocument(document: vscode.TextDocument): void { From b77a743408b267b2a832c72612bda8852710dc43 Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Sun, 24 Nov 2024 02:07:19 +0530 Subject: [PATCH 12/18] add long term and short term diff in autoedit prompt --- vscode/src/autoedits/prompt-utils.test.ts | 1 + vscode/src/autoedits/prompt-utils.ts | 69 +++++++++-- .../auotedit-short-term-diff.ts | 114 +++++++----------- .../autoedit.short-term-diff.test.ts | 46 +++++++ .../recent-edits-diff-helpers/base.ts | 7 ++ .../recent-edits-diff-helpers/unified-diff.ts | 35 ++---- .../recent-edits-diff-helpers/utils.ts | 40 +++++- 7 files changed, 202 insertions(+), 110 deletions(-) create mode 100644 vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/autoedit.short-term-diff.test.ts diff --git a/vscode/src/autoedits/prompt-utils.test.ts b/vscode/src/autoedits/prompt-utils.test.ts index 0f93109acd7e..70b85c80aa4e 100644 --- a/vscode/src/autoedits/prompt-utils.test.ts +++ b/vscode/src/autoedits/prompt-utils.test.ts @@ -384,6 +384,7 @@ line 64 + Now, continue where I left off and finish my change by rewriting "code_to_rewrite": ` expect(prompt.toString()).toEqual(expectedPrompt) diff --git a/vscode/src/autoedits/prompt-utils.ts b/vscode/src/autoedits/prompt-utils.ts index 4287e2c40c0a..2ac25287af81 100644 --- a/vscode/src/autoedits/prompt-utils.ts +++ b/vscode/src/autoedits/prompt-utils.ts @@ -77,6 +77,11 @@ interface CurrentFileContext { range: vscode.Range } +interface RecentEditPromptComponents { + longTermDiff: PromptString + shortTermDiff: PromptString +} + // Helper function to get prompt in some format export function getBaseUserPrompt( docContext: DocumentContext, @@ -107,10 +112,8 @@ export function getBaseUserPrompt( getRecentlyViewedSnippetsPrompt ) - const recentEditsPrompt = getPromptForTheContextSource( - contextItemMapping.get(RetrieverIdentifier.RecentEditsRetriever) || [], - RECENT_EDITS_INSTRUCTION, - getRecentEditsPrompt + const recentEditsPromptComponents = getRecentEditsPromptComponents( + contextItemMapping.get(RetrieverIdentifier.RecentEditsRetriever) || [] ) const lintErrorsPrompt = getPromptForTheContextSource( @@ -134,10 +137,11 @@ export function getBaseUserPrompt( ${jaccardSimilarityPrompt} ${recentViewsPrompt} ${CURRENT_FILE_INSTRUCTION}${fileWithMarkerPrompt} -${recentEditsPrompt} +${recentEditsPromptComponents.longTermDiff} ${lintErrorsPrompt} ${recentCopyPrompt} ${areaPrompt} +${recentEditsPromptComponents.shortTermDiff} ${FINAL_USER_PROMPT} ` autoeditsLogger.logDebug('AutoEdits', 'Prompt\n', finalPrompt) @@ -323,24 +327,61 @@ ${RECENT_COPY_TAG_CLOSE} ` } -export function getRecentEditsPrompt(contextItems: AutocompleteContextSnippet[]): PromptString { +export function getRecentEditsPromptComponents( + contextItems: AutocompleteContextSnippet[] +): RecentEditPromptComponents { const recentEdits = getContextItemsForIdentifier( contextItems, RetrieverIdentifier.RecentEditsRetriever ) recentEdits.reverse() - if (recentEdits.length === 0) { + let shortTermDiff: PromptString = ps`` + let longTermDiff: PromptString = ps`` + if (recentEdits.length > 0) { + shortTermDiff = getRecentEditPrompt([recentEdits.at(-1)!]) + } + if (recentEdits.length > 1) { + const longTermDiffPrompt = getRecentEditPromptLongTermDiffComponent(recentEdits.slice(0, -1)) + longTermDiff = ps`${RECENT_EDITS_INSTRUCTION} +${longTermDiffPrompt} +` + } + return { + shortTermDiff, + longTermDiff, + } +} + +function getRecentEditPromptLongTermDiffComponent(context: AutocompleteContextSnippet[]): PromptString { + if (context.length === 0) { return ps`` } - const recentEditsPrompts = recentEdits.map(item => - getContextPromptWithPath( + const prompts = context.map(item => + getContextPromptForDiffWithPath( PromptString.fromDisplayPath(item.uri), PromptString.fromAutocompleteContextSnippet(item).content ) ) - const recentEditsPrompt = PromptString.join(recentEditsPrompts, ps`\n`) - return ps`${RECENT_EDITS_TAG_OPEN} -${recentEditsPrompt} + return ps` +${RECENT_EDITS_TAG_OPEN} +${PromptString.join(prompts, ps`\n`)} +${RECENT_EDITS_TAG_CLOSE} +` +} + +function getRecentEditPrompt(contextItems: AutocompleteContextSnippet[]): PromptString { + if (contextItems.length === 0) { + return ps`` + } + const prompts = contextItems.map(item => + getContextPromptForDiffWithPath( + PromptString.fromDisplayPath(item.uri), + PromptString.fromAutocompleteContextSnippet(item).content + ) + ) + return ps` +${RECENT_EDITS_TAG_OPEN} +${PromptString.join(prompts, ps`\n`)} ${RECENT_EDITS_TAG_CLOSE} ` } @@ -455,3 +496,7 @@ function getContextItemsForIdentifier( function getContextPromptWithPath(filePath: PromptString, content: PromptString): PromptString { return ps`(\`${filePath}\`)\n\n${content}\n` } + +function getContextPromptForDiffWithPath(filePath: PromptString, content: PromptString): PromptString { + return ps`${filePath}\n${content}` +} diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts index a223530b5899..a22a6a661020 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts @@ -1,98 +1,70 @@ -import fs from 'node:fs' -import path from 'node:path' -import type * as vscode from 'vscode' import type { DiffCalculationInput, DiffHunk, RecentEditsRetrieverDiffStrategy, TextDocumentChange, } from './base' +import { groupOverlappingDocumentChanges } from './utils' import { - applyTextDocumentChanges, - computeDiffWithLineNumbers, - groupNonOverlappingChangeGroups, - groupOverlappingDocumentChanges, + type TextDocumentChangeGroup, + getDiffHunkFromUnifiedPatch, + getUnifiedDiffHunkFromTextDocumentChange, } from './utils' -import type { TextDocumentChangeGroup } from './utils' /** * Generates a single unified diff patch that combines all changes * made to a document into one consolidated view. */ export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDiffStrategy { + private longTermContextLines = 3 private shortTermContextLines = 0 public getDiffHunks(input: DiffCalculationInput): DiffHunk[] { const rawChanges = groupOverlappingDocumentChanges(input.changes) - const rawDiffHunks = this.getDiffHunksFromGroupedChanges(input, rawChanges) - const changes = groupNonOverlappingChangeGroups(rawChanges) - const combinedDiffHunks = this.getDiffHunksFromGroupedChanges(input, changes) - this.logRawDataPoints(input.uri.toString(), input.oldContent, rawDiffHunks, combinedDiffHunks) - return combinedDiffHunks - } + const { shortTermChanges, longTermChanges } = + this.divideChangesIntoShortTermAndLongTerm(rawChanges) - private getDiffHunksFromGroupedChanges( - input: DiffCalculationInput, - changes: TextDocumentChangeGroup[] - ): DiffHunk[] { - const allDiffHunks: DiffHunk[] = [] - let oldContent = input.oldContent - for (const changeList of changes) { - const [diffHunk, newContent] = this.getDiffHunksForChanges( - input.uri, - oldContent, - changeList.changes, - this.shortTermContextLines - ) - oldContent = newContent - allDiffHunks.push(diffHunk) - } - return allDiffHunks + const longTermPatch = getUnifiedDiffHunkFromTextDocumentChange({ + uri: input.uri, + oldContent: input.oldContent, + changes: longTermChanges, + addLineNumbersForDiff: true, + contextLines: this.longTermContextLines, + }) + const shortTermPatch = getUnifiedDiffHunkFromTextDocumentChange({ + uri: input.uri, + oldContent: longTermPatch?.newContent || input.oldContent, + changes: shortTermChanges, + addLineNumbersForDiff: true, + contextLines: this.shortTermContextLines, + }) + return [ + getDiffHunkFromUnifiedPatch(shortTermPatch), + getDiffHunkFromUnifiedPatch(longTermPatch), + ].filter((hunk): hunk is DiffHunk => hunk !== undefined) } - private logRawDataPoints( - uri: string, - oldContent: string, - rawDiffHunks: DiffHunk[], - combinedDiffHunks: DiffHunk[] - ) { - const dirPath = '/Users/hiteshsagtani/Desktop/raw-diff-logs' - const fileName = uri.split('/').pop()?.split('.')[0] || 'document' - const logPath = uri.replace(/[^/\\]+$/, `${fileName}_raw.jsonl`) - const finalLogPath = path.join(dirPath, path.basename(logPath)) - - // Create directory if it doesn't exist - if (!fs.existsSync(dirPath)) { - fs.mkdirSync(dirPath, { recursive: true }) + private divideChangesIntoShortTermAndLongTerm(changes: TextDocumentChangeGroup[]): { + shortTermChanges: TextDocumentChange[] + longTermChanges: TextDocumentChange[] + } { + if (changes.length <= 1) { + return { + shortTermChanges: this.convertTextDocumentChangeGroupToTextDocumentChange(changes), + longTermChanges: [], + } } - - const logData = { - uri: uri.toString(), - oldContent, - rawDiffHunks, - combinedDiffHunks, + return { + shortTermChanges: this.convertTextDocumentChangeGroupToTextDocumentChange(changes.slice(-1)), + longTermChanges: this.convertTextDocumentChangeGroupToTextDocumentChange( + changes.slice(0, -1) + ), } - // Append to file if it exists, create if it doesn't - fs.appendFileSync(finalLogPath, JSON.stringify(logData) + '\n', { encoding: 'utf8' }) } - private getDiffHunksForChanges( - uri: vscode.Uri, - oldContent: string, - changes: TextDocumentChange[], - numContextLines: number - ): [DiffHunk, string] { - const newContent = applyTextDocumentChanges( - oldContent, - changes.map(c => c.change) - ) - const gitDiff = computeDiffWithLineNumbers(uri, oldContent, newContent, numContextLines) - const diffHunk = { - uri, - leastEditTimestamp: Math.min(...changes.map(c => c.timestamp)), - latestEditTimestamp: Math.max(...changes.map(c => c.timestamp)), - diff: gitDiff, - } - return [diffHunk, newContent] + private convertTextDocumentChangeGroupToTextDocumentChange( + changeGroup: TextDocumentChangeGroup[] + ): TextDocumentChange[] { + return changeGroup.flatMap(group => group.changes) } } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/autoedit.short-term-diff.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/autoedit.short-term-diff.test.ts new file mode 100644 index 000000000000..32b1b9633739 --- /dev/null +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/autoedit.short-term-diff.test.ts @@ -0,0 +1,46 @@ +import dedent from 'dedent' +import { describe, expect, it } from 'vitest' +import * as vscode from 'vscode' +import { AutoeditWithShortTermDiffStrategy } from './auotedit-short-term-diff' +import { getTextDocumentChangesForText } from './helper' + +const processComputedDiff = (text: string): string => { + const lines = text.split('\n') + const updatedText = lines.filter(line => !line.includes('\\ No newline at end of file')).join('\n') + return updatedText +} + +describe('AutoeditWithShortTermDiffStrategy', () => { + const strategy = new AutoeditWithShortTermDiffStrategy() + + it('handles multiple changes across different lines', () => { + const text = dedent` + letconst x = 5; + varlet y = 10; + console.log('break'); + letconst z = 5; + console.log(x +x * y); + ` + const { originalText, changes } = getTextDocumentChangesForText(text) + const diffs = strategy.getDiffHunks({ + uri: vscode.Uri.parse('file://test.ts'), + oldContent: originalText, + changes, + }) + expect(diffs.length).toBe(2) + expect(processComputedDiff(diffs[0].diff.toString())).toMatchInlineSnapshot(` + "5-| console.log(x + y); + 5+| console.log(x * y);" + `) + expect(processComputedDiff(diffs[1].diff.toString())).toMatchInlineSnapshot(` + "1-| let x = 5; + 2-| var y = 10; + 1+| const x = 5; + 2+| let y = 10; + 3 | console.log('break'); + 4-| let z = 5; + 4+| const z = 5; + 5 | console.log(x + y);" + `) + }) +}) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts index f29b755bbfcd..dd081a5944ad 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts @@ -63,3 +63,10 @@ export interface DiffHunk { latestEditTimestamp: number diff: PromptString } + +export interface UnifiedPatchResponse { + uri: vscode.Uri + newContent: string + diff: PromptString + latestEditTimestamp: number +} diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts index 2bd3de7c2d0e..4ad9177505a3 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts @@ -1,6 +1,5 @@ -import { PromptString } from '@sourcegraph/cody-shared' import type { DiffCalculationInput, DiffHunk, RecentEditsRetrieverDiffStrategy } from './base' -import { applyTextDocumentChanges, computeDiffWithLineNumbers } from './utils' +import { getUnifiedDiffHunkFromTextDocumentChange } from './utils' interface UnifiedDiffStrategyOptions { addLineNumbers: boolean @@ -19,29 +18,13 @@ export class UnifiedDiffStrategy implements RecentEditsRetrieverDiffStrategy { } public getDiffHunks(input: DiffCalculationInput): DiffHunk[] { - const newContent = applyTextDocumentChanges( - input.oldContent, - input.changes.map(c => c.change) - ) - const diff = this.getDiffForUnifiedStrategy(input, newContent) - return [ - { - uri: input.uri, - diff, - latestEditTimestamp: Math.max(...input.changes.map(c => c.timestamp)), - }, - ] - } - - private getDiffForUnifiedStrategy(input: DiffCalculationInput, newContent: string): PromptString { - if (this.addLineNumbers) { - return computeDiffWithLineNumbers( - input.uri, - input.oldContent, - newContent, - this.numContextLines - ) - } - return PromptString.fromGitDiff(input.uri, input.oldContent, newContent) + const diffHunk = getUnifiedDiffHunkFromTextDocumentChange({ + uri: input.uri, + oldContent: input.oldContent, + changes: input.changes, + addLineNumbersForDiff: this.addLineNumbers, + contextLines: this.numContextLines, + }) + return diffHunk ? [diffHunk] : [] } } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts index ce45c056e083..0a3fe2ec33a6 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts @@ -2,7 +2,7 @@ import { PromptString } from '@sourcegraph/cody-shared' import { displayPath } from '@sourcegraph/cody-shared/src/editor/displayPath' import { structuredPatch } from 'diff' import * as vscode from 'vscode' -import type { TextDocumentChange } from './base' +import type { DiffHunk, TextDocumentChange, UnifiedPatchResponse } from './base' /** * Represents a group of text document changes with their range information. @@ -190,6 +190,44 @@ export function getDiffStringForHunkWithLineNumbers(hunk: Diff.Hunk): string { return lines.join('\n') } +export function getUnifiedDiffHunkFromTextDocumentChange(params: { + uri: vscode.Uri + oldContent: string + changes: TextDocumentChange[] + addLineNumbersForDiff: boolean + contextLines: number +}): UnifiedPatchResponse | undefined { + if (params.changes.length === 0) { + return undefined + } + const newContent = applyTextDocumentChanges( + params.oldContent, + params.changes.map(c => c.change) + ) + const diff = params.addLineNumbersForDiff + ? computeDiffWithLineNumbers(params.uri, params.oldContent, newContent, params.contextLines) + : PromptString.fromGitDiff(params.uri, params.oldContent, newContent) + + return { + uri: params.uri, + newContent, + diff, + latestEditTimestamp: Math.max(...params.changes.map(c => c.timestamp)), + } +} + +export function getDiffHunkFromUnifiedPatch( + unifiedPatch: UnifiedPatchResponse | undefined +): DiffHunk | undefined { + return unifiedPatch + ? { + uri: unifiedPatch.uri, + latestEditTimestamp: unifiedPatch.latestEditTimestamp, + diff: unifiedPatch.diff, + } + : undefined +} + export function applyTextDocumentChanges( content: string, changes: vscode.TextDocumentContentChangeEvent[] From eeb7eee8eb009e921b1c962144e6618af26349a5 Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Sun, 24 Nov 2024 03:40:20 +0530 Subject: [PATCH 13/18] diff strategies --- vscode/src/completions/analytics-logger.ts | 1 + .../context/context-data-logging.ts | 14 +- .../completions/context/context-strategy.ts | 26 ++-- .../autoedit.short-term-diff.test.ts | 46 ------- .../recent-edits-diff-helpers/base.ts | 41 +----- .../line-level-diff.test.ts | 103 +++++++++++++++ .../line-level-diff.ts | 57 ++++++++ .../two-stage-unified-diff.test.ts | 125 ++++++++++++++++++ ...term-diff.ts => two-stage-unified-diff.ts} | 6 +- .../recent-edits-diff-helpers/unified-diff.ts | 6 + .../recent-edits-retriever.test.ts | 4 +- .../recent-edits-retriever.ts | 56 +++++--- .../supercompletion-provider.ts | 4 +- 13 files changed, 362 insertions(+), 127 deletions(-) delete mode 100644 vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/autoedit.short-term-diff.test.ts create mode 100644 vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.test.ts create mode 100644 vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts create mode 100644 vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.test.ts rename vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/{auotedit-short-term-diff.ts => two-stage-unified-diff.ts} (93%) diff --git a/vscode/src/completions/analytics-logger.ts b/vscode/src/completions/analytics-logger.ts index 8c7b1974481c..c96925210d4d 100644 --- a/vscode/src/completions/analytics-logger.ts +++ b/vscode/src/completions/analytics-logger.ts @@ -803,6 +803,7 @@ function suggestionDocumentDiffTracker( const documentText = document.getText(trackingRange) const persistenceTimeoutList = [ + 10 * 1000, // 10 seconds 20 * 1000, // 20 seconds 60 * 1000, // 60 seconds 120 * 1000, // 120 seconds diff --git a/vscode/src/completions/context/context-data-logging.ts b/vscode/src/completions/context/context-data-logging.ts index 7e56fe6a7acb..3655d2c6d649 100644 --- a/vscode/src/completions/context/context-data-logging.ts +++ b/vscode/src/completions/context/context-data-logging.ts @@ -13,14 +13,15 @@ import type { RetrievedContextResults } from './completions-context-ranker' import { JaccardSimilarityRetriever } from './retrievers/jaccard-similarity/jaccard-similarity-retriever' import { DiagnosticsRetriever } from './retrievers/recent-user-actions/diagnostics-retriever' import { RecentCopyRetriever } from './retrievers/recent-user-actions/recent-copy' -import { RecentEditsRetrieverDiffStrategyIdentifier } from './retrievers/recent-user-actions/recent-edits-diff-helpers/base' +import { LineLevelDiffStrategy } from './retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff' +import { TwoStageUnifiedDiffStrategy } from './retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff' import { RecentEditsRetriever } from './retrievers/recent-user-actions/recent-edits-retriever' import { RecentViewPortRetriever } from './retrievers/recent-user-actions/recent-view-port' import { RetrieverIdentifier } from './utils' interface RetrieverConfig { identifier: RetrieverIdentifier - maxSnippets: number + maxSnippets?: number } export class ContextRetrieverDataCollection implements vscode.Disposable { @@ -31,7 +32,7 @@ export class ContextRetrieverDataCollection implements vscode.Disposable { private gitMetadataInstance = GitHubDotComRepoMetadata.getInstance() private readonly retrieverConfigs: RetrieverConfig[] = [ - { identifier: RetrieverIdentifier.RecentEditsRetriever, maxSnippets: 15 }, + { identifier: RetrieverIdentifier.RecentEditsRetriever }, { identifier: RetrieverIdentifier.DiagnosticsRetriever, maxSnippets: 15 }, { identifier: RetrieverIdentifier.RecentViewPortRetriever, maxSnippets: 10 }, ] @@ -105,8 +106,11 @@ export class ContextRetrieverDataCollection implements vscode.Disposable { case RetrieverIdentifier.RecentEditsRetriever: return new RecentEditsRetriever({ maxAgeMs: 10 * 60 * 1000, - diffStrategyIdentifier: - RecentEditsRetrieverDiffStrategyIdentifier.UnifiedDiffWithLineNumbers, + diffStrategyList: [ + new TwoStageUnifiedDiffStrategy(), + new LineLevelDiffStrategy({ shouldGroupNonOverlappingLines: true }), + new LineLevelDiffStrategy({ shouldGroupNonOverlappingLines: false }), + ], }) case RetrieverIdentifier.DiagnosticsRetriever: return new DiagnosticsRetriever({ diff --git a/vscode/src/completions/context/context-strategy.ts b/vscode/src/completions/context/context-strategy.ts index 61cbe0dbcdfb..b07d1b7e4817 100644 --- a/vscode/src/completions/context/context-strategy.ts +++ b/vscode/src/completions/context/context-strategy.ts @@ -11,7 +11,8 @@ import { JaccardSimilarityRetriever } from './retrievers/jaccard-similarity/jacc import { LspLightRetriever } from './retrievers/lsp-light/lsp-light-retriever' import { DiagnosticsRetriever } from './retrievers/recent-user-actions/diagnostics-retriever' import { RecentCopyRetriever } from './retrievers/recent-user-actions/recent-copy' -import { RecentEditsRetrieverDiffStrategyIdentifier } from './retrievers/recent-user-actions/recent-edits-diff-helpers/base' +import { TwoStageUnifiedDiffStrategy } from './retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff' +import { UnifiedDiffStrategy } from './retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff' import { RecentEditsRetriever } from './retrievers/recent-user-actions/recent-edits-retriever' import { RecentViewPortRetriever } from './retrievers/recent-user-actions/recent-view-port' import { loadTscRetriever } from './retrievers/tsc/load-tsc-retriever' @@ -55,8 +56,9 @@ export class DefaultContextStrategyFactory implements ContextStrategyFactory { this.allLocalRetrievers = [ new RecentEditsRetriever({ maxAgeMs: 60 * 1000, - diffStrategyIdentifier: - RecentEditsRetrieverDiffStrategyIdentifier.UnifiedDiff, + diffStrategyList: [ + new UnifiedDiffStrategy({ addLineNumbers: false }), + ], }), ] break @@ -64,8 +66,9 @@ export class DefaultContextStrategyFactory implements ContextStrategyFactory { this.allLocalRetrievers = [ new RecentEditsRetriever({ maxAgeMs: 60 * 1000, - diffStrategyIdentifier: - RecentEditsRetrieverDiffStrategyIdentifier.UnifiedDiff, + diffStrategyList: [ + new UnifiedDiffStrategy({ addLineNumbers: false }), + ], }), ] break @@ -73,8 +76,9 @@ export class DefaultContextStrategyFactory implements ContextStrategyFactory { this.allLocalRetrievers = [ new RecentEditsRetriever({ maxAgeMs: 60 * 5 * 1000, - diffStrategyIdentifier: - RecentEditsRetrieverDiffStrategyIdentifier.UnifiedDiff, + diffStrategyList: [ + new UnifiedDiffStrategy({ addLineNumbers: false }), + ], }), ] break @@ -82,8 +86,9 @@ export class DefaultContextStrategyFactory implements ContextStrategyFactory { this.allLocalRetrievers = [ new RecentEditsRetriever({ maxAgeMs: 60 * 1000, - diffStrategyIdentifier: - RecentEditsRetrieverDiffStrategyIdentifier.UnifiedDiff, + diffStrategyList: [ + new UnifiedDiffStrategy({ addLineNumbers: false }), + ], }), new JaccardSimilarityRetriever(), ] @@ -127,8 +132,7 @@ export class DefaultContextStrategyFactory implements ContextStrategyFactory { this.allLocalRetrievers = [ new RecentEditsRetriever({ maxAgeMs: 10 * 60 * 1000, - diffStrategyIdentifier: - RecentEditsRetrieverDiffStrategyIdentifier.AutoeditWithShortTermDiff, + diffStrategyList: [new TwoStageUnifiedDiffStrategy()], }), new DiagnosticsRetriever({ contextLines: 0, diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/autoedit.short-term-diff.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/autoedit.short-term-diff.test.ts deleted file mode 100644 index 32b1b9633739..000000000000 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/autoedit.short-term-diff.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import dedent from 'dedent' -import { describe, expect, it } from 'vitest' -import * as vscode from 'vscode' -import { AutoeditWithShortTermDiffStrategy } from './auotedit-short-term-diff' -import { getTextDocumentChangesForText } from './helper' - -const processComputedDiff = (text: string): string => { - const lines = text.split('\n') - const updatedText = lines.filter(line => !line.includes('\\ No newline at end of file')).join('\n') - return updatedText -} - -describe('AutoeditWithShortTermDiffStrategy', () => { - const strategy = new AutoeditWithShortTermDiffStrategy() - - it('handles multiple changes across different lines', () => { - const text = dedent` - letconst x = 5; - varlet y = 10; - console.log('break'); - letconst z = 5; - console.log(x +x * y); - ` - const { originalText, changes } = getTextDocumentChangesForText(text) - const diffs = strategy.getDiffHunks({ - uri: vscode.Uri.parse('file://test.ts'), - oldContent: originalText, - changes, - }) - expect(diffs.length).toBe(2) - expect(processComputedDiff(diffs[0].diff.toString())).toMatchInlineSnapshot(` - "5-| console.log(x + y); - 5+| console.log(x * y);" - `) - expect(processComputedDiff(diffs[1].diff.toString())).toMatchInlineSnapshot(` - "1-| let x = 5; - 2-| var y = 10; - 1+| const x = 5; - 2+| let y = 10; - 3 | console.log('break'); - 4-| let z = 5; - 4+| const z = 5; - 5 | console.log(x + y);" - `) - }) -}) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts index dd081a5944ad..e69ad9a165f5 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts @@ -1,48 +1,9 @@ import type { PromptString } from '@sourcegraph/cody-shared' import type * as vscode from 'vscode' -import { AutoeditWithShortTermDiffStrategy } from './auotedit-short-term-diff' -import { UnifiedDiffStrategy } from './unified-diff' - -/** - * Identifiers for the different diff strategies. - */ -export enum RecentEditsRetrieverDiffStrategyIdentifier { - /** - * Unified diff strategy that shows changes in a single patch. - */ - UnifiedDiff = 'unified-diff', - /** - * Unified diff strategy that shows changes in a single patch. - */ - UnifiedDiffWithLineNumbers = 'unified-diff-with-line-numbers', - /** - * Diff Strategy to use a seperate short term diff used by `auto-edits`. - */ - AutoeditWithShortTermDiff = 'autoedit-with-short-term-diff', -} - -/** - * Creates a new instance of a diff strategy based on the provided identifier. - * @param identifier The identifier of the diff strategy to create. - * @returns A new instance of the diff strategy. - */ -export function createDiffStrategy( - identifier: RecentEditsRetrieverDiffStrategyIdentifier -): RecentEditsRetrieverDiffStrategy { - switch (identifier) { - case RecentEditsRetrieverDiffStrategyIdentifier.UnifiedDiff: - return new UnifiedDiffStrategy({ addLineNumbers: false }) - case RecentEditsRetrieverDiffStrategyIdentifier.UnifiedDiffWithLineNumbers: - return new UnifiedDiffStrategy({ addLineNumbers: true }) - case RecentEditsRetrieverDiffStrategyIdentifier.AutoeditWithShortTermDiff: - return new AutoeditWithShortTermDiffStrategy() - default: - throw new Error(`Unknown diff strategy identifier: ${identifier}`) - } -} export interface RecentEditsRetrieverDiffStrategy { getDiffHunks(input: DiffCalculationInput): DiffHunk[] + getDiffStrategyName(): string } export interface TextDocumentChange { diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.test.ts new file mode 100644 index 000000000000..5e391a433406 --- /dev/null +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.test.ts @@ -0,0 +1,103 @@ +import dedent from 'dedent' +import { describe, expect, it } from 'vitest' +import * as vscode from 'vscode' +import { getTextDocumentChangesForText } from './helper' +import { LineLevelDiffStrategy } from './line-level-diff' + +const processComputedDiff = (text: string): string => { + const lines = text.split('\n') + const updatedText = lines.filter(line => !line.includes('\\ No newline at end of file')).join('\n') + return updatedText +} + +describe('LineLevelDiffStrategy', () => { + describe('with non-overlapping lines grouping enabled', () => { + const strategy = new LineLevelDiffStrategy({ shouldGroupNonOverlappingLines: true }) + + it('handles multiple line changes with grouping', () => { + const text = dedent` + letconst x = 5; + console.log('break'); + letconst y = 10; + ` + const { originalText, changes } = getTextDocumentChangesForText(text) + const diffs = strategy.getDiffHunks({ + uri: vscode.Uri.parse('file://test.ts'), + oldContent: originalText, + changes, + }) + expect(diffs.length).toBe(1) + expect(processComputedDiff(diffs[0].diff.toString())).toMatchInlineSnapshot(` + "1-| let x = 5; + 1+| const x = 5; + 2 | console.log('break'); + 3-| let y = 10; + 3+| const y = 10;" + `) + }) + + it('handles single line change', () => { + const text = dedent` + const x = 5; + varlet y = 10; + console.log('test'); + ` + const { originalText, changes } = getTextDocumentChangesForText(text) + const diffs = strategy.getDiffHunks({ + uri: vscode.Uri.parse('file://test.ts'), + oldContent: originalText, + changes, + }) + expect(diffs.length).toBe(1) + expect(processComputedDiff(diffs[0].diff.toString())).toMatchInlineSnapshot(` + "1 | const x = 5; + 2-| var y = 10; + 2+| let y = 10; + 3 | console.log('test');" + `) + }) + }) + + describe('with non-overlapping lines grouping disabled', () => { + const strategy = new LineLevelDiffStrategy({ shouldGroupNonOverlappingLines: false }) + + it('handles multiple separate changes without grouping', () => { + const text = dedent` + letconst x = 5; + console.log('break'); + letconst y = 10; + ` + const { originalText, changes } = getTextDocumentChangesForText(text) + const diffs = strategy.getDiffHunks({ + uri: vscode.Uri.parse('file://test.ts'), + oldContent: originalText, + changes, + }) + expect(diffs.length).toBe(2) + expect(processComputedDiff(diffs[0].diff.toString())).toMatchInlineSnapshot(` + "1-| let x = 5; + 1+| const x = 5; + 2 | console.log('break'); + 3 | let y = 10;" + `) + expect(processComputedDiff(diffs[1].diff.toString())).toMatchInlineSnapshot(` + "1 | const x = 5; + 2 | console.log('break'); + 3-| let y = 10; + 3+| const y = 10;" + `) + }) + }) + + it('returns correct strategy name', () => { + const strategyWithGrouping = new LineLevelDiffStrategy({ shouldGroupNonOverlappingLines: true }) + expect(strategyWithGrouping.getDiffStrategyName()).toBe('line-level-diff-non-overlap-lines-true') + + const strategyWithoutGrouping = new LineLevelDiffStrategy({ + shouldGroupNonOverlappingLines: false, + }) + expect(strategyWithoutGrouping.getDiffStrategyName()).toBe( + 'line-level-diff-non-overlap-lines-false' + ) + }) +}) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts new file mode 100644 index 000000000000..7fe5f35c369b --- /dev/null +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts @@ -0,0 +1,57 @@ +import type { DiffCalculationInput, DiffHunk, RecentEditsRetrieverDiffStrategy } from './base' +import { groupNonOverlappingChangeGroups, groupOverlappingDocumentChanges } from './utils' +import { + type TextDocumentChangeGroup, + getDiffHunkFromUnifiedPatch, + getUnifiedDiffHunkFromTextDocumentChange, +} from './utils' + +interface StrategyOptions { + shouldGroupNonOverlappingLines: boolean +} + +export class LineLevelDiffStrategy implements RecentEditsRetrieverDiffStrategy { + private contextLines = 3 + private shouldGroupNonOverlappingLines: boolean + + constructor(options: StrategyOptions) { + this.shouldGroupNonOverlappingLines = options.shouldGroupNonOverlappingLines + } + + public getDiffHunks(input: DiffCalculationInput): DiffHunk[] { + const groupedChanges = this.getLineLevelChanges(input) + const diffHunks: DiffHunk[] = [] + let oldContent = input.oldContent + for (const groupedChange of groupedChanges) { + const patch = getUnifiedDiffHunkFromTextDocumentChange({ + uri: input.uri, + oldContent: oldContent, + changes: groupedChange.changes, + addLineNumbersForDiff: true, + contextLines: this.contextLines, + }) + if (patch) { + const hunk = getDiffHunkFromUnifiedPatch(patch) + if (hunk) { + diffHunks.push(hunk) + } + oldContent = patch.newContent + } + } + return diffHunks + } + + private getLineLevelChanges(input: DiffCalculationInput): TextDocumentChangeGroup[] { + const changes = groupOverlappingDocumentChanges(input.changes) + if (!this.shouldGroupNonOverlappingLines) { + return changes + } + return groupNonOverlappingChangeGroups(changes) + } + + public getDiffStrategyName(): string { + return `line-level-diff-${ + this.shouldGroupNonOverlappingLines ? 'non-overlap-lines-true' : 'non-overlap-lines-false' + }` + } +} diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.test.ts new file mode 100644 index 000000000000..410599b3726a --- /dev/null +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.test.ts @@ -0,0 +1,125 @@ +import dedent from 'dedent' +import { describe, expect, it } from 'vitest' +import * as vscode from 'vscode' +import { getTextDocumentChangesForText } from './helper' +import { TwoStageUnifiedDiffStrategy } from './two-stage-unified-diff' + +const processComputedDiff = (text: string): string => { + const lines = text.split('\n') + const updatedText = lines.filter(line => !line.includes('\\ No newline at end of file')).join('\n') + return updatedText +} + +describe('AutoeditWithShortTermDiffStrategy', () => { + const strategy = new TwoStageUnifiedDiffStrategy() + + it('handles multiple changes across different lines', () => { + const text = dedent` + letconst x = 5; + varlet y = 10; + console.log('break'); + letconst z = 5; + console.log(x +x * y); + ` + const { originalText, changes } = getTextDocumentChangesForText(text) + const diffs = strategy.getDiffHunks({ + uri: vscode.Uri.parse('file://test.ts'), + oldContent: originalText, + changes, + }) + expect(diffs.length).toBe(2) + expect(processComputedDiff(diffs[0].diff.toString())).toMatchInlineSnapshot(` + "5-| console.log(x + y); + 5+| console.log(x * y);" + `) + expect(processComputedDiff(diffs[1].diff.toString())).toMatchInlineSnapshot(` + "1-| let x = 5; + 2-| var y = 10; + 1+| const x = 5; + 2+| let y = 10; + 3 | console.log('break'); + 4-| let z = 5; + 4+| const z = 5; + 5 | console.log(x + y);" + `) + }) + + it('handles case with no changes', () => { + const text = dedent` + const x = 5; + let y = 10; + console.log('break'); + ` + const { originalText, changes } = getTextDocumentChangesForText(text) + const diffs = strategy.getDiffHunks({ + uri: vscode.Uri.parse('file://test.ts'), + oldContent: originalText, + changes, + }) + expect(diffs.length).toBe(0) + }) + + it('handles single change', () => { + const text = dedent` + const x = 5; + varlet y = 10; + console.log('break'); + ` + const { originalText, changes } = getTextDocumentChangesForText(text) + const diffs = strategy.getDiffHunks({ + uri: vscode.Uri.parse('file://test.ts'), + oldContent: originalText, + changes, + }) + expect(diffs.length).toBe(1) + expect(processComputedDiff(diffs[0].diff.toString())).toMatchInlineSnapshot(` + "2-| var y = 10; + 2+| let y = 10;" + `) + }) + + it('handles changes at file boundaries', () => { + const text = dedent` + // First line added\nconst x = 5; + let y = 10; + console.log('break');\nfinal line removed + ` + const { originalText, changes } = getTextDocumentChangesForText(text) + const diffs = strategy.getDiffHunks({ + uri: vscode.Uri.parse('file://test.ts'), + oldContent: originalText, + changes, + }) + expect(diffs.length).toBe(2) + expect(processComputedDiff(diffs[0].diff.toString())).toMatchInlineSnapshot(` + "4-| console.log('break'); + 5-| final line removed + 4+| console.log('break');" + `) + expect(processComputedDiff(diffs[1].diff.toString())).toMatchInlineSnapshot(` + "1+| // First line added + 2 | const x = 5; + 3 | let y = 10; + 4 | console.log('break');" + `) + }) + + it('handles multiple adjacent changes', () => { + const text = dedent` + const x = 5; + varlet y = 1020; + console.log('break'); + ` + const { originalText, changes } = getTextDocumentChangesForText(text) + const diffs = strategy.getDiffHunks({ + uri: vscode.Uri.parse('file://test.ts'), + oldContent: originalText, + changes, + }) + expect(diffs.length).toBe(1) + expect(processComputedDiff(diffs[0].diff.toString())).toMatchInlineSnapshot(` + "2-| var y = 10; + 2+| let y = 20;" + `) + }) +}) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.ts similarity index 93% rename from vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts rename to vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.ts index a22a6a661020..19a4dd6f5458 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/auotedit-short-term-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.ts @@ -15,7 +15,7 @@ import { * Generates a single unified diff patch that combines all changes * made to a document into one consolidated view. */ -export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDiffStrategy { +export class TwoStageUnifiedDiffStrategy implements RecentEditsRetrieverDiffStrategy { private longTermContextLines = 3 private shortTermContextLines = 0 @@ -67,4 +67,8 @@ export class AutoeditWithShortTermDiffStrategy implements RecentEditsRetrieverDi ): TextDocumentChange[] { return changeGroup.flatMap(group => group.changes) } + + public getDiffStrategyName(): string { + return 'two-stage-unified-diff-strategy' + } } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts index 4ad9177505a3..1f5ab77caea7 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts @@ -27,4 +27,10 @@ export class UnifiedDiffStrategy implements RecentEditsRetrieverDiffStrategy { }) return diffHunk ? [diffHunk] : [] } + + public getDiffStrategyName(): string { + return `unified-diff-strategy-${ + this.addLineNumbers ? 'with-line-numbers' : 'without-line-numbers' + }` + } } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts index ea96715e749b..e1f1552ad7f4 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts @@ -4,7 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type * as vscode from 'vscode' import { range } from '../../../../testutils/textDocument' import { document } from '../../../test-helpers' -import { RecentEditsRetrieverDiffStrategyIdentifier } from './recent-edits-diff-helpers/base' +import { UnifiedDiffStrategy } from './recent-edits-diff-helpers/unified-diff' import { RecentEditsRetriever } from './recent-edits-retriever' const FIVE_MINUTES = 5 * 60 * 1000 @@ -25,7 +25,7 @@ describe('RecentEditsRetriever', () => { retriever = new RecentEditsRetriever( { maxAgeMs: FIVE_MINUTES, - diffStrategyIdentifier: RecentEditsRetrieverDiffStrategyIdentifier.UnifiedDiff, + diffStrategyList: [new UnifiedDiffStrategy({ addLineNumbers: false })], }, { onDidChangeTextDocument(listener) { diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts index 0e2a2737c0ce..797a78d2001b 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts @@ -4,12 +4,10 @@ import * as vscode from 'vscode' import { getPositionAfterTextInsertion } from '../../../text-processing/utils' import type { ContextRetriever, ContextRetrieverOptions } from '../../../types' import { RetrieverIdentifier, type ShouldUseContextParams, shouldBeUsedAsContext } from '../../utils' -import { - type DiffHunk, - type RecentEditsRetrieverDiffStrategy, - type RecentEditsRetrieverDiffStrategyIdentifier, - type TextDocumentChange, - createDiffStrategy, +import type { + DiffHunk, + RecentEditsRetrieverDiffStrategy, + TextDocumentChange, } from './recent-edits-diff-helpers/base' import { applyTextDocumentChanges } from './recent-edits-diff-helpers/utils' @@ -20,9 +18,13 @@ interface TrackedDocument { changes: TextDocumentChange[] } +interface DiffHunkWithStrategy extends DiffHunk { + diffStrategyName: string +} + export interface RecentEditsRetrieverOptions { maxAgeMs: number - diffStrategyIdentifier: RecentEditsRetrieverDiffStrategyIdentifier + diffStrategyList: RecentEditsRetrieverDiffStrategy[] } interface DiffAcrossDocuments { @@ -30,6 +32,7 @@ interface DiffAcrossDocuments { uri: vscode.Uri languageId: string latestChangeTimestamp: number + diffStrategyName: string } export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever { @@ -39,8 +42,7 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever public identifier = RetrieverIdentifier.RecentEditsRetriever private disposables: vscode.Disposable[] = [] private readonly maxAgeMs: number - private readonly diffStrategyIdentifier: RecentEditsRetrieverDiffStrategyIdentifier - private readonly diffStrategy: RecentEditsRetrieverDiffStrategy + private readonly diffStrategyList: RecentEditsRetrieverDiffStrategy[] constructor( options: RecentEditsRetrieverOptions, @@ -50,8 +52,7 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever > = vscode.workspace ) { this.maxAgeMs = options.maxAgeMs - this.diffStrategyIdentifier = options.diffStrategyIdentifier - this.diffStrategy = createDiffStrategy(this.diffStrategyIdentifier) + this.diffStrategyList = options.diffStrategyList // Track the already open documents when editor was opened for (const document of vscode.workspace.textDocuments) { @@ -68,8 +69,13 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever public async retrieve(options: ContextRetrieverOptions): Promise { const rawDiffs = await this.getDiffAcrossDocuments() const diffs = this.filterCandidateDiffs(rawDiffs, options.document) - // Heuristics ordering by timestamp, taking the most recent diffs first. - diffs.sort((a, b) => b.latestChangeTimestamp - a.latestChangeTimestamp) + // Sort first by strategy name and then by timestamp + diffs.sort((a, b) => { + if (a.diffStrategyName !== b.diffStrategyName) { + return a.diffStrategyName.localeCompare(b.diffStrategyName) + } + return b.latestChangeTimestamp - a.latestChangeTimestamp + }) const autocompleteContextSnippets = [] for (const diff of diffs) { @@ -80,7 +86,7 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever content, metadata: { timeSinceActionMs: Date.now() - diff.latestChangeTimestamp, - recentEditsRetrieverDiffStrategy: this.diffStrategyIdentifier, + recentEditsRetrieverDiffStrategy: diff.diffStrategyName, }, } satisfies Omit autocompleteContextSnippets.push(autocompleteSnippet) @@ -105,6 +111,7 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever uri: trackedDocument.uri, languageId: trackedDocument.languageId, latestChangeTimestamp: diffHunk.latestEditTimestamp, + diffStrategyName: diffHunk.diffStrategyName, })) } return null @@ -135,7 +142,7 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever return filterCandidateDiffs } - public async getDiff(uri: vscode.Uri): Promise { + public async getDiff(uri: vscode.Uri): Promise { if (await contextFiltersProvider.isUriIgnored(uri)) { return null } @@ -144,11 +151,20 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever if (!trackedDocument) { return null } - const diffHunks = this.diffStrategy.getDiffHunks({ - uri: trackedDocument.uri, - oldContent: trackedDocument.content, - changes: trackedDocument.changes, - }) + const diffHunks: DiffHunkWithStrategy[] = [] + for (const diffStrategy of this.diffStrategyList) { + const hunks = diffStrategy.getDiffHunks({ + uri: trackedDocument.uri, + oldContent: trackedDocument.content, + changes: trackedDocument.changes, + }) + for (const hunk of hunks) { + diffHunks.push({ + ...hunk, + diffStrategyName: diffStrategy.getDiffStrategyName(), + }) + } + } return diffHunks } diff --git a/vscode/src/supercompletions/supercompletion-provider.ts b/vscode/src/supercompletions/supercompletion-provider.ts index 01008c030271..68feaf418729 100644 --- a/vscode/src/supercompletions/supercompletion-provider.ts +++ b/vscode/src/supercompletions/supercompletion-provider.ts @@ -1,6 +1,6 @@ import type { ChatClient } from '@sourcegraph/cody-shared' import * as vscode from 'vscode' -import { RecentEditsRetrieverDiffStrategyIdentifier } from '../completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base' +import { UnifiedDiffStrategy } from '../completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff' import { RecentEditsRetriever } from '../completions/context/retrievers/recent-user-actions/recent-edits-retriever' import type { CodyStatusBar } from '../services/StatusBar' import { type Supercompletion, getSupercompletions } from './get-supercompletion' @@ -31,7 +31,7 @@ export class SupercompletionProvider implements vscode.Disposable { this.recentEditsRetriever = new RecentEditsRetriever( { maxAgeMs: EDIT_HISTORY_TIMEOUT, - diffStrategyIdentifier: RecentEditsRetrieverDiffStrategyIdentifier.UnifiedDiff, + diffStrategyList: [new UnifiedDiffStrategy({ addLineNumbers: false })], }, workspace ) From a51b77c11dc630e83fd6542a49fce1d63313d5a0 Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Sun, 24 Nov 2024 03:53:08 +0530 Subject: [PATCH 14/18] augment test case --- .../recent-edits-retriever.test.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts index e1f1552ad7f4..238bff869dba 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts @@ -4,6 +4,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type * as vscode from 'vscode' import { range } from '../../../../testutils/textDocument' import { document } from '../../../test-helpers' +import { LineLevelDiffStrategy } from './recent-edits-diff-helpers/line-level-diff' import { UnifiedDiffStrategy } from './recent-edits-diff-helpers/unified-diff' import { RecentEditsRetriever } from './recent-edits-retriever' @@ -25,7 +26,10 @@ describe('RecentEditsRetriever', () => { retriever = new RecentEditsRetriever( { maxAgeMs: FIVE_MINUTES, - diffStrategyList: [new UnifiedDiffStrategy({ addLineNumbers: false })], + diffStrategyList: [ + new UnifiedDiffStrategy({ addLineNumbers: false }), + new LineLevelDiffStrategy({ shouldGroupNonOverlappingLines: false }), + ], }, { onDidChangeTextDocument(listener) { @@ -116,7 +120,7 @@ describe('RecentEditsRetriever', () => { const diffHunks = await retriever.getDiff(testDocument.uri) expect(diffHunks).not.toBeNull() - expect(diffHunks?.length).toBe(1) + expect(diffHunks?.length).toBe(3) expect( diffHunks?.[0].diff?.toString().split('\n').slice(2).join('\n') ).toMatchInlineSnapshot(` @@ -162,7 +166,7 @@ describe('RecentEditsRetriever', () => { const diffHunks = await retriever.getDiff(testDocument.uri) expect(diffHunks).not.toBeNull() - expect(diffHunks?.length).toBe(1) + expect(diffHunks?.length).toBe(2) expect( diffHunks?.[0].diff?.toString().split('\n').slice(2).join('\n') ).toMatchInlineSnapshot(` @@ -203,7 +207,7 @@ describe('RecentEditsRetriever', () => { const diffHunks = await retriever.getDiff(newUri) expect(diffHunks).not.toBeNull() - expect(diffHunks?.length).toBe(1) + expect(diffHunks?.length).toBe(2) expect( diffHunks?.[0].diff?.toString().split('\n').slice(2).join('\n') ).toMatchInlineSnapshot(` From be81dbcc864b3e6fd85d38f6de928f221afdaabe Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Sun, 24 Nov 2024 05:23:09 +0530 Subject: [PATCH 15/18] add line level strategies --- .../line-level-diff.test.ts | 23 +++++---- .../line-level-diff.ts | 46 ++++++++++++----- .../two-stage-unified-diff.ts | 46 ++++------------- .../recent-edits-diff-helpers/utils.ts | 51 ++++++++++--------- 4 files changed, 82 insertions(+), 84 deletions(-) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.test.ts index 5e391a433406..1d06ae63cb6b 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.test.ts @@ -26,14 +26,19 @@ describe('LineLevelDiffStrategy', () => { oldContent: originalText, changes, }) - expect(diffs.length).toBe(1) - expect(processComputedDiff(diffs[0].diff.toString())).toMatchInlineSnapshot(` - "1-| let x = 5; - 1+| const x = 5; - 2 | console.log('break'); - 3-| let y = 10; - 3+| const y = 10;" + expect(diffs.length).toBe(2) + expect(processComputedDiff(diffs[1].diff.toString())).toMatchInlineSnapshot(` + "1-| let x = 5; + 1+| const x = 5; + 2 | console.log('break'); + 3 | let y = 10;" `) + expect(processComputedDiff(diffs[0].diff.toString())).toMatchInlineSnapshot(` + "1 | const x = 5; + 2 | console.log('break'); + 3-| let y = 10; + 3+| const y = 10;" + `) }) it('handles single line change', () => { @@ -74,13 +79,13 @@ describe('LineLevelDiffStrategy', () => { changes, }) expect(diffs.length).toBe(2) - expect(processComputedDiff(diffs[0].diff.toString())).toMatchInlineSnapshot(` + expect(processComputedDiff(diffs[1].diff.toString())).toMatchInlineSnapshot(` "1-| let x = 5; 1+| const x = 5; 2 | console.log('break'); 3 | let y = 10;" `) - expect(processComputedDiff(diffs[1].diff.toString())).toMatchInlineSnapshot(` + expect(processComputedDiff(diffs[0].diff.toString())).toMatchInlineSnapshot(` "1 | const x = 5; 2 | console.log('break'); 3-| let y = 10; diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts index 7fe5f35c369b..61f23d7a79f0 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts @@ -1,7 +1,9 @@ +import type * as vscode from 'vscode' import type { DiffCalculationInput, DiffHunk, RecentEditsRetrieverDiffStrategy } from './base' import { groupNonOverlappingChangeGroups, groupOverlappingDocumentChanges } from './utils' import { type TextDocumentChangeGroup, + divideGroupedChangesIntoShortTermAndLongTerm, getDiffHunkFromUnifiedPatch, getUnifiedDiffHunkFromTextDocumentChange, } from './utils' @@ -20,23 +22,37 @@ export class LineLevelDiffStrategy implements RecentEditsRetrieverDiffStrategy { public getDiffHunks(input: DiffCalculationInput): DiffHunk[] { const groupedChanges = this.getLineLevelChanges(input) + const diffHunks = this.getDiffHunksForGroupedChanges({ + uri: input.uri, + oldContent: input.oldContent, + groupedChanges, + contextLines: this.contextLines, + addLineNumbersForDiff: true, + }).filter(diffHunk => diffHunk.diff.toString() !== '') + diffHunks.reverse() + return diffHunks + } + + private getDiffHunksForGroupedChanges(params: { + uri: vscode.Uri + oldContent: string + groupedChanges: TextDocumentChangeGroup[] + contextLines: number + addLineNumbersForDiff: boolean + }): DiffHunk[] { + let currentContent = params.oldContent const diffHunks: DiffHunk[] = [] - let oldContent = input.oldContent - for (const groupedChange of groupedChanges) { + for (const groupedChange of params.groupedChanges) { const patch = getUnifiedDiffHunkFromTextDocumentChange({ - uri: input.uri, - oldContent: oldContent, + uri: params.uri, + oldContent: currentContent, changes: groupedChange.changes, - addLineNumbersForDiff: true, - contextLines: this.contextLines, + addLineNumbersForDiff: params.addLineNumbersForDiff, + contextLines: params.contextLines, }) - if (patch) { - const hunk = getDiffHunkFromUnifiedPatch(patch) - if (hunk) { - diffHunks.push(hunk) - } - oldContent = patch.newContent - } + const hunk = getDiffHunkFromUnifiedPatch(patch) + diffHunks.push(hunk) + currentContent = patch.newContent } return diffHunks } @@ -46,7 +62,9 @@ export class LineLevelDiffStrategy implements RecentEditsRetrieverDiffStrategy { if (!this.shouldGroupNonOverlappingLines) { return changes } - return groupNonOverlappingChangeGroups(changes) + let { shortTermChanges, longTermChanges } = divideGroupedChangesIntoShortTermAndLongTerm(changes) + longTermChanges = groupNonOverlappingChangeGroups(longTermChanges) + return [...longTermChanges, ...shortTermChanges] } public getDiffStrategyName(): string { diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.ts index 19a4dd6f5458..495d49239426 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.ts @@ -1,12 +1,7 @@ -import type { - DiffCalculationInput, - DiffHunk, - RecentEditsRetrieverDiffStrategy, - TextDocumentChange, -} from './base' +import type { DiffCalculationInput, DiffHunk, RecentEditsRetrieverDiffStrategy } from './base' import { groupOverlappingDocumentChanges } from './utils' import { - type TextDocumentChangeGroup, + divideGroupedChangesIntoShortTermAndLongTerm, getDiffHunkFromUnifiedPatch, getUnifiedDiffHunkFromTextDocumentChange, } from './utils' @@ -22,50 +17,27 @@ export class TwoStageUnifiedDiffStrategy implements RecentEditsRetrieverDiffStra public getDiffHunks(input: DiffCalculationInput): DiffHunk[] { const rawChanges = groupOverlappingDocumentChanges(input.changes) const { shortTermChanges, longTermChanges } = - this.divideChangesIntoShortTermAndLongTerm(rawChanges) + divideGroupedChangesIntoShortTermAndLongTerm(rawChanges) const longTermPatch = getUnifiedDiffHunkFromTextDocumentChange({ uri: input.uri, oldContent: input.oldContent, - changes: longTermChanges, + changes: longTermChanges.flatMap(c => c.changes), addLineNumbersForDiff: true, contextLines: this.longTermContextLines, }) const shortTermPatch = getUnifiedDiffHunkFromTextDocumentChange({ uri: input.uri, - oldContent: longTermPatch?.newContent || input.oldContent, - changes: shortTermChanges, + oldContent: longTermPatch.newContent, + changes: shortTermChanges.flatMap(c => c.changes), addLineNumbersForDiff: true, contextLines: this.shortTermContextLines, }) - return [ + const diffs = [ getDiffHunkFromUnifiedPatch(shortTermPatch), getDiffHunkFromUnifiedPatch(longTermPatch), - ].filter((hunk): hunk is DiffHunk => hunk !== undefined) - } - - private divideChangesIntoShortTermAndLongTerm(changes: TextDocumentChangeGroup[]): { - shortTermChanges: TextDocumentChange[] - longTermChanges: TextDocumentChange[] - } { - if (changes.length <= 1) { - return { - shortTermChanges: this.convertTextDocumentChangeGroupToTextDocumentChange(changes), - longTermChanges: [], - } - } - return { - shortTermChanges: this.convertTextDocumentChangeGroupToTextDocumentChange(changes.slice(-1)), - longTermChanges: this.convertTextDocumentChangeGroupToTextDocumentChange( - changes.slice(0, -1) - ), - } - } - - private convertTextDocumentChangeGroupToTextDocumentChange( - changeGroup: TextDocumentChangeGroup[] - ): TextDocumentChange[] { - return changeGroup.flatMap(group => group.changes) + ].filter(diff => diff.diff.length > 0) + return diffs } public getDiffStrategyName(): string { diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts index 0a3fe2ec33a6..57f7771242ba 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts @@ -46,7 +46,7 @@ export function groupOverlappingDocumentChanges( replacementRange: change.change.range, originalChange: change, })), - mergePredicate: (a, b) => doLineSpansOverlap(a.start.line, a.end.line, b.start.line, b.end.line), + mergePredicate: (a, b) => doLineOverlapForRanges(a, b), getChanges: item => [item.originalChange], }) } @@ -68,8 +68,7 @@ export function groupNonOverlappingChangeGroups( ): TextDocumentChangeGroup[] { return mergeDocumentChanges({ items: groupedChanges, - mergePredicate: (a, b) => - !doLineSpansOverlap(a.start.line, a.end.line, b.start.line, b.end.line), + mergePredicate: (a, b) => !doLineOverlapForRanges(a, b), getChanges: group => group.changes, }) } @@ -196,10 +195,7 @@ export function getUnifiedDiffHunkFromTextDocumentChange(params: { changes: TextDocumentChange[] addLineNumbersForDiff: boolean contextLines: number -}): UnifiedPatchResponse | undefined { - if (params.changes.length === 0) { - return undefined - } +}): UnifiedPatchResponse { const newContent = applyTextDocumentChanges( params.oldContent, params.changes.map(c => c.change) @@ -216,16 +212,28 @@ export function getUnifiedDiffHunkFromTextDocumentChange(params: { } } -export function getDiffHunkFromUnifiedPatch( - unifiedPatch: UnifiedPatchResponse | undefined -): DiffHunk | undefined { - return unifiedPatch - ? { - uri: unifiedPatch.uri, - latestEditTimestamp: unifiedPatch.latestEditTimestamp, - diff: unifiedPatch.diff, - } - : undefined +export function divideGroupedChangesIntoShortTermAndLongTerm(changes: TextDocumentChangeGroup[]): { + shortTermChanges: TextDocumentChangeGroup[] + longTermChanges: TextDocumentChangeGroup[] +} { + if (changes.length <= 1) { + return { + shortTermChanges: changes, + longTermChanges: [], + } + } + return { + shortTermChanges: changes.slice(-1), + longTermChanges: changes.slice(0, -1), + } +} + +export function getDiffHunkFromUnifiedPatch(unifiedPatch: UnifiedPatchResponse): DiffHunk { + return { + uri: unifiedPatch.uri, + latestEditTimestamp: unifiedPatch.latestEditTimestamp, + diff: unifiedPatch.diff, + } } export function applyTextDocumentChanges( @@ -241,11 +249,6 @@ export function applyTextDocumentChanges( return content } -function doLineSpansOverlap( - firstStart: number, - firstEnd: number, - secondStart: number, - secondEnd: number -): boolean { - return firstStart <= secondEnd && firstEnd >= secondStart +function doLineOverlapForRanges(a: vscode.Range, b: vscode.Range): boolean { + return a.start.line <= b.end.line && a.end.line >= b.start.line } From 8ee99a27eec74fb8e24ea639f3616ca689c48899 Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Sun, 24 Nov 2024 23:46:04 +0530 Subject: [PATCH 16/18] use common timestamp for different context candidates --- vscode/src/completions/context/context-data-logging.ts | 1 + .../retrievers/recent-user-actions/recent-edits-retriever.ts | 3 ++- .../retrievers/recent-user-actions/recent-view-port.ts | 5 ++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/vscode/src/completions/context/context-data-logging.ts b/vscode/src/completions/context/context-data-logging.ts index 3655d2c6d649..9966c33feca9 100644 --- a/vscode/src/completions/context/context-data-logging.ts +++ b/vscode/src/completions/context/context-data-logging.ts @@ -35,6 +35,7 @@ export class ContextRetrieverDataCollection implements vscode.Disposable { { identifier: RetrieverIdentifier.RecentEditsRetriever }, { identifier: RetrieverIdentifier.DiagnosticsRetriever, maxSnippets: 15 }, { identifier: RetrieverIdentifier.RecentViewPortRetriever, maxSnippets: 10 }, + { identifier: RetrieverIdentifier.JaccardSimilarityRetriever, maxSnippets: 5 }, ] constructor() { diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts index 797a78d2001b..88fc78ff1c3f 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts @@ -78,6 +78,7 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever }) const autocompleteContextSnippets = [] + const retrievalTriggerTime = Date.now() for (const diff of diffs) { const content = diff.diff.toString() const autocompleteSnippet = { @@ -85,7 +86,7 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever identifier: this.identifier, content, metadata: { - timeSinceActionMs: Date.now() - diff.latestChangeTimestamp, + timeSinceActionMs: retrievalTriggerTime - diff.latestChangeTimestamp, recentEditsRetrieverDiffStrategy: diff.diffStrategyName, }, } satisfies Omit diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-view-port.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-view-port.ts index 1a155f2cdc89..53ff47c2b3aa 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-view-port.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-view-port.ts @@ -48,11 +48,10 @@ export class RecentViewPortRetriever implements vscode.Disposable, ContextRetrie public async retrieve({ document }: ContextRetrieverOptions): Promise { const sortedViewPorts = this.getValidViewPorts(document) - + const retrievalTriggerTime = Date.now() const snippetPromises = sortedViewPorts.map(async viewPort => { const document = await vscode.workspace.openTextDocument(viewPort.uri) const content = document.getText(viewPort.visibleRange) - return { uri: viewPort.uri, content, @@ -60,7 +59,7 @@ export class RecentViewPortRetriever implements vscode.Disposable, ContextRetrie endLine: viewPort.visibleRange.end.line, identifier: this.identifier, metadata: { - timeSinceActionMs: Date.now() - viewPort.lastAccessTimestamp, + timeSinceActionMs: retrievalTriggerTime - viewPort.lastAccessTimestamp, }, } }) From 20f35053d13e6b0011115808a46c54f81c0f3a90 Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Mon, 25 Nov 2024 06:05:09 +0530 Subject: [PATCH 17/18] use time based diff logic --- .../context/context-data-logging.ts | 39 ++++- .../completions/context/context-strategy.ts | 9 +- .../line-level-diff.test.ts | 47 +++-- .../line-level-diff.ts | 35 ++-- .../two-stage-unified-diff.test.ts | 35 +++- .../two-stage-unified-diff.ts | 21 ++- .../recent-edits-diff-helpers/utils.test.ts | 162 +++++++++++++++++- .../recent-edits-diff-helpers/utils.ts | 35 +++- .../recent-edits-retriever.test.ts | 7 +- 9 files changed, 327 insertions(+), 63 deletions(-) diff --git a/vscode/src/completions/context/context-data-logging.ts b/vscode/src/completions/context/context-data-logging.ts index 9966c33feca9..42267d7c5ae6 100644 --- a/vscode/src/completions/context/context-data-logging.ts +++ b/vscode/src/completions/context/context-data-logging.ts @@ -35,7 +35,6 @@ export class ContextRetrieverDataCollection implements vscode.Disposable { { identifier: RetrieverIdentifier.RecentEditsRetriever }, { identifier: RetrieverIdentifier.DiagnosticsRetriever, maxSnippets: 15 }, { identifier: RetrieverIdentifier.RecentViewPortRetriever, maxSnippets: 10 }, - { identifier: RetrieverIdentifier.JaccardSimilarityRetriever, maxSnippets: 5 }, ] constructor() { @@ -108,9 +107,41 @@ export class ContextRetrieverDataCollection implements vscode.Disposable { return new RecentEditsRetriever({ maxAgeMs: 10 * 60 * 1000, diffStrategyList: [ - new TwoStageUnifiedDiffStrategy(), - new LineLevelDiffStrategy({ shouldGroupNonOverlappingLines: true }), - new LineLevelDiffStrategy({ shouldGroupNonOverlappingLines: false }), + // Only use the last event as a short term diff. + new TwoStageUnifiedDiffStrategy({ + longTermContextLines: 3, + shortTermContextLines: 3, + minShortTermEvents: 1, + minShortTermTimeMs: 0, + }), + // Use atleast last 30 seconds of edits as short term diff + new TwoStageUnifiedDiffStrategy({ + longTermContextLines: 3, + shortTermContextLines: 3, + minShortTermEvents: 1, + minShortTermTimeMs: 30 * 1000, // 30 seconds + }), + // Use non-overlapping lines combination for long term diffs. + new LineLevelDiffStrategy({ + contextLines: 3, + longTermDiffCombinationStrategy: 'lines-based', + minShortTermEvents: 1, + minShortTermTimeMs: 30 * 1000, // 30 seconds + }), + // Use unified diff for long term changes, and line based diff for short term changes. + new LineLevelDiffStrategy({ + contextLines: 3, + longTermDiffCombinationStrategy: 'unified-diff', + minShortTermEvents: 1, + minShortTermTimeMs: 2 * 60 * 1000, // 2 minutes + }), + // Use raw line based changes for all the diff calculation. + new LineLevelDiffStrategy({ + contextLines: 3, + longTermDiffCombinationStrategy: undefined, + minShortTermEvents: 1, + minShortTermTimeMs: 0, + }), ], }) case RetrieverIdentifier.DiagnosticsRetriever: diff --git a/vscode/src/completions/context/context-strategy.ts b/vscode/src/completions/context/context-strategy.ts index b07d1b7e4817..8497a87fbe6f 100644 --- a/vscode/src/completions/context/context-strategy.ts +++ b/vscode/src/completions/context/context-strategy.ts @@ -132,7 +132,14 @@ export class DefaultContextStrategyFactory implements ContextStrategyFactory { this.allLocalRetrievers = [ new RecentEditsRetriever({ maxAgeMs: 10 * 60 * 1000, - diffStrategyList: [new TwoStageUnifiedDiffStrategy()], + diffStrategyList: [ + new TwoStageUnifiedDiffStrategy({ + longTermContextLines: 3, + shortTermContextLines: 0, + minShortTermEvents: 1, + minShortTermTimeMs: 0, + }), + ], }), new DiagnosticsRetriever({ contextLines: 0, diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.test.ts index 1d06ae63cb6b..05b24661c817 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.test.ts @@ -1,8 +1,8 @@ import dedent from 'dedent' -import { describe, expect, it } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' import * as vscode from 'vscode' import { getTextDocumentChangesForText } from './helper' -import { LineLevelDiffStrategy } from './line-level-diff' +import { LineLevelDiffStrategy, type LineLevelStrategyOptions } from './line-level-diff' const processComputedDiff = (text: string): string => { const lines = text.split('\n') @@ -11,8 +11,29 @@ const processComputedDiff = (text: string): string => { } describe('LineLevelDiffStrategy', () => { + const getTextDocumentChanges = (text: string) => { + const { originalText, changes } = getTextDocumentChangesForText(text) + // Advance the time to simulate Date.now() at a later time compared to when the changes were made + vi.advanceTimersByTime(1) + return { + originalText, + changes, + } + } + + const getStrategyOptions = (shouldGroupNonOverlappingLines: boolean): LineLevelStrategyOptions => ({ + contextLines: 3, + longTermDiffCombinationStrategy: shouldGroupNonOverlappingLines ? 'lines-based' : undefined, + minShortTermEvents: 1, + minShortTermTimeMs: 0, + }) + + beforeEach(() => { + vi.useFakeTimers() + }) + describe('with non-overlapping lines grouping enabled', () => { - const strategy = new LineLevelDiffStrategy({ shouldGroupNonOverlappingLines: true }) + const strategy = new LineLevelDiffStrategy(getStrategyOptions(true)) it('handles multiple line changes with grouping', () => { const text = dedent` @@ -20,7 +41,7 @@ describe('LineLevelDiffStrategy', () => { console.log('break'); letconst y = 10; ` - const { originalText, changes } = getTextDocumentChangesForText(text) + const { originalText, changes } = getTextDocumentChanges(text) const diffs = strategy.getDiffHunks({ uri: vscode.Uri.parse('file://test.ts'), oldContent: originalText, @@ -47,7 +68,7 @@ describe('LineLevelDiffStrategy', () => { varlet y = 10; console.log('test'); ` - const { originalText, changes } = getTextDocumentChangesForText(text) + const { originalText, changes } = getTextDocumentChanges(text) const diffs = strategy.getDiffHunks({ uri: vscode.Uri.parse('file://test.ts'), oldContent: originalText, @@ -64,7 +85,7 @@ describe('LineLevelDiffStrategy', () => { }) describe('with non-overlapping lines grouping disabled', () => { - const strategy = new LineLevelDiffStrategy({ shouldGroupNonOverlappingLines: false }) + const strategy = new LineLevelDiffStrategy(getStrategyOptions(false)) it('handles multiple separate changes without grouping', () => { const text = dedent` @@ -72,7 +93,7 @@ describe('LineLevelDiffStrategy', () => { console.log('break'); letconst y = 10; ` - const { originalText, changes } = getTextDocumentChangesForText(text) + const { originalText, changes } = getTextDocumentChanges(text) const diffs = strategy.getDiffHunks({ uri: vscode.Uri.parse('file://test.ts'), oldContent: originalText, @@ -93,16 +114,4 @@ describe('LineLevelDiffStrategy', () => { `) }) }) - - it('returns correct strategy name', () => { - const strategyWithGrouping = new LineLevelDiffStrategy({ shouldGroupNonOverlappingLines: true }) - expect(strategyWithGrouping.getDiffStrategyName()).toBe('line-level-diff-non-overlap-lines-true') - - const strategyWithoutGrouping = new LineLevelDiffStrategy({ - shouldGroupNonOverlappingLines: false, - }) - expect(strategyWithoutGrouping.getDiffStrategyName()).toBe( - 'line-level-diff-non-overlap-lines-false' - ) - }) }) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts index 61f23d7a79f0..0c2d740d6812 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts @@ -3,22 +3,21 @@ import type { DiffCalculationInput, DiffHunk, RecentEditsRetrieverDiffStrategy } import { groupNonOverlappingChangeGroups, groupOverlappingDocumentChanges } from './utils' import { type TextDocumentChangeGroup, + combineTextDocumentGroups, divideGroupedChangesIntoShortTermAndLongTerm, getDiffHunkFromUnifiedPatch, getUnifiedDiffHunkFromTextDocumentChange, } from './utils' -interface StrategyOptions { - shouldGroupNonOverlappingLines: boolean +export interface LineLevelStrategyOptions { + contextLines: number + longTermDiffCombinationStrategy: 'unified-diff' | 'lines-based' | undefined + minShortTermEvents: number + minShortTermTimeMs: number } export class LineLevelDiffStrategy implements RecentEditsRetrieverDiffStrategy { - private contextLines = 3 - private shouldGroupNonOverlappingLines: boolean - - constructor(options: StrategyOptions) { - this.shouldGroupNonOverlappingLines = options.shouldGroupNonOverlappingLines - } + constructor(private readonly options: LineLevelStrategyOptions) {} public getDiffHunks(input: DiffCalculationInput): DiffHunk[] { const groupedChanges = this.getLineLevelChanges(input) @@ -26,7 +25,7 @@ export class LineLevelDiffStrategy implements RecentEditsRetrieverDiffStrategy { uri: input.uri, oldContent: input.oldContent, groupedChanges, - contextLines: this.contextLines, + contextLines: this.options.contextLines, addLineNumbersForDiff: true, }).filter(diffHunk => diffHunk.diff.toString() !== '') diffHunks.reverse() @@ -59,17 +58,23 @@ export class LineLevelDiffStrategy implements RecentEditsRetrieverDiffStrategy { private getLineLevelChanges(input: DiffCalculationInput): TextDocumentChangeGroup[] { const changes = groupOverlappingDocumentChanges(input.changes) - if (!this.shouldGroupNonOverlappingLines) { + if (this.options.longTermDiffCombinationStrategy === undefined) { return changes } - let { shortTermChanges, longTermChanges } = divideGroupedChangesIntoShortTermAndLongTerm(changes) - longTermChanges = groupNonOverlappingChangeGroups(longTermChanges) + let { shortTermChanges, longTermChanges } = divideGroupedChangesIntoShortTermAndLongTerm({ + changes, + minEvents: this.options.minShortTermEvents, + minTimeMs: this.options.minShortTermTimeMs, + }) + if (this.options.longTermDiffCombinationStrategy === 'lines-based') { + longTermChanges = groupNonOverlappingChangeGroups(longTermChanges) + } else if (this.options.longTermDiffCombinationStrategy === 'unified-diff') { + longTermChanges = [combineTextDocumentGroups(longTermChanges)] + } return [...longTermChanges, ...shortTermChanges] } public getDiffStrategyName(): string { - return `line-level-diff-${ - this.shouldGroupNonOverlappingLines ? 'non-overlap-lines-true' : 'non-overlap-lines-false' - }` + return 'line-level-diff' } } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.test.ts index 410599b3726a..1dcf10cc7289 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.test.ts @@ -1,5 +1,5 @@ import dedent from 'dedent' -import { describe, expect, it } from 'vitest' +import { beforeEach, describe, expect, it, vi } from 'vitest' import * as vscode from 'vscode' import { getTextDocumentChangesForText } from './helper' import { TwoStageUnifiedDiffStrategy } from './two-stage-unified-diff' @@ -10,8 +10,27 @@ const processComputedDiff = (text: string): string => { return updatedText } -describe('AutoeditWithShortTermDiffStrategy', () => { - const strategy = new TwoStageUnifiedDiffStrategy() +describe('TwoStageUnifiedDiffStrategy', () => { + const getTextDocumentChanges = (text: string) => { + const { originalText, changes } = getTextDocumentChangesForText(text) + // Advance the time to simulate Date.now() at a later time compared to when the changes were made + vi.advanceTimersByTime(1) + return { + originalText, + changes, + } + } + + const strategy = new TwoStageUnifiedDiffStrategy({ + longTermContextLines: 3, + shortTermContextLines: 0, + minShortTermEvents: 1, + minShortTermTimeMs: 0, + }) + + beforeEach(() => { + vi.useFakeTimers() + }) it('handles multiple changes across different lines', () => { const text = dedent` @@ -21,7 +40,7 @@ describe('AutoeditWithShortTermDiffStrategy', () => { letconst z = 5; console.log(x +x * y); ` - const { originalText, changes } = getTextDocumentChangesForText(text) + const { originalText, changes } = getTextDocumentChanges(text) const diffs = strategy.getDiffHunks({ uri: vscode.Uri.parse('file://test.ts'), oldContent: originalText, @@ -50,7 +69,7 @@ describe('AutoeditWithShortTermDiffStrategy', () => { let y = 10; console.log('break'); ` - const { originalText, changes } = getTextDocumentChangesForText(text) + const { originalText, changes } = getTextDocumentChanges(text) const diffs = strategy.getDiffHunks({ uri: vscode.Uri.parse('file://test.ts'), oldContent: originalText, @@ -65,7 +84,7 @@ describe('AutoeditWithShortTermDiffStrategy', () => { varlet y = 10; console.log('break'); ` - const { originalText, changes } = getTextDocumentChangesForText(text) + const { originalText, changes } = getTextDocumentChanges(text) const diffs = strategy.getDiffHunks({ uri: vscode.Uri.parse('file://test.ts'), oldContent: originalText, @@ -84,7 +103,7 @@ describe('AutoeditWithShortTermDiffStrategy', () => { let y = 10; console.log('break');\nfinal line removed ` - const { originalText, changes } = getTextDocumentChangesForText(text) + const { originalText, changes } = getTextDocumentChanges(text) const diffs = strategy.getDiffHunks({ uri: vscode.Uri.parse('file://test.ts'), oldContent: originalText, @@ -110,7 +129,7 @@ describe('AutoeditWithShortTermDiffStrategy', () => { varlet y = 1020; console.log('break'); ` - const { originalText, changes } = getTextDocumentChangesForText(text) + const { originalText, changes } = getTextDocumentChanges(text) const diffs = strategy.getDiffHunks({ uri: vscode.Uri.parse('file://test.ts'), oldContent: originalText, diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.ts index 495d49239426..271973359fbe 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.ts @@ -6,32 +6,41 @@ import { getUnifiedDiffHunkFromTextDocumentChange, } from './utils' +interface StrategyOptions { + longTermContextLines: number + shortTermContextLines: number + minShortTermEvents: number + minShortTermTimeMs: number +} + /** * Generates a single unified diff patch that combines all changes * made to a document into one consolidated view. */ export class TwoStageUnifiedDiffStrategy implements RecentEditsRetrieverDiffStrategy { - private longTermContextLines = 3 - private shortTermContextLines = 0 + constructor(private readonly options: StrategyOptions) {} public getDiffHunks(input: DiffCalculationInput): DiffHunk[] { const rawChanges = groupOverlappingDocumentChanges(input.changes) - const { shortTermChanges, longTermChanges } = - divideGroupedChangesIntoShortTermAndLongTerm(rawChanges) + const { shortTermChanges, longTermChanges } = divideGroupedChangesIntoShortTermAndLongTerm({ + changes: rawChanges, + minEvents: this.options.minShortTermEvents, + minTimeMs: this.options.minShortTermTimeMs, + }) const longTermPatch = getUnifiedDiffHunkFromTextDocumentChange({ uri: input.uri, oldContent: input.oldContent, changes: longTermChanges.flatMap(c => c.changes), addLineNumbersForDiff: true, - contextLines: this.longTermContextLines, + contextLines: this.options.longTermContextLines, }) const shortTermPatch = getUnifiedDiffHunkFromTextDocumentChange({ uri: input.uri, oldContent: longTermPatch.newContent, changes: shortTermChanges.flatMap(c => c.changes), addLineNumbersForDiff: true, - contextLines: this.shortTermContextLines, + contextLines: this.options.shortTermContextLines, }) const diffs = [ getDiffHunkFromUnifiedPatch(shortTermPatch), diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts index bc2a138e2b80..f921c55717f7 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts @@ -1,11 +1,13 @@ import { PromptString } from '@sourcegraph/cody-shared' import dedent from 'dedent' -import { describe, expect, it } from 'vitest' -import type * as vscode from 'vscode' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import * as vscode from 'vscode' import { getDiffsForContentChanges, getTextDocumentChangesForText } from './helper' import { + type TextDocumentChangeGroup, applyTextDocumentChanges, computeDiffWithLineNumbers, + divideGroupedChangesIntoShortTermAndLongTerm, groupConsecutiveItemsByPredicate, groupNonOverlappingChangeGroups, groupOverlappingDocumentChanges, @@ -417,3 +419,159 @@ describe('computeDiffWithLineNumbers', () => { ) }) }) + +describe('divideGroupedChangesIntoShortTermAndLongTerm', () => { + let changes: TextDocumentChangeGroup[] + + beforeEach(() => { + vi.useFakeTimers() + }) + + const createTestChange = (timeAgo: number): TextDocumentChangeGroup => { + const dummyRange = new vscode.Range(0, 0, 0, 0) + return { + changes: [ + { + timestamp: Date.now() - timeAgo, + change: { + range: dummyRange, + rangeOffset: 0, + rangeLength: 0, + text: '', + }, + insertedRange: dummyRange, + }, + ], + insertedRange: dummyRange, + replacementRange: dummyRange, + } + } + + const createTestChanges = (timeAgos: number[]): TextDocumentChangeGroup[] => + timeAgos.map(timeAgo => createTestChange(timeAgo)) + + const assertDivision = (params: { + timeAgos: number[] + minEvents: number + minTimeMs: number + expectedShortTermCount: number + }) => { + changes = createTestChanges(params.timeAgos) + const { shortTermChanges, longTermChanges } = divideGroupedChangesIntoShortTermAndLongTerm({ + changes, + minEvents: params.minEvents, + minTimeMs: params.minTimeMs, + }) + expect(shortTermChanges).toEqual(changes.slice(changes.length - params.expectedShortTermCount)) + expect(longTermChanges).toEqual(changes.slice(0, changes.length - params.expectedShortTermCount)) + } + + it('should return all changes as shortTermChanges when conditions are not met', () => { + assertDivision({ + timeAgos: [1000, 2000], + minEvents: 3, + minTimeMs: 5000, + expectedShortTermCount: 2, + }) + }) + + it('should divide changes into shortTermChanges and longTermChanges correctly', () => { + assertDivision({ + timeAgos: [20000, 15000, 5000, 1000], + minEvents: 2, + minTimeMs: 10000, + expectedShortTermCount: 2, + }) + }) + + it('should return all changes as longTermChanges when conditions are met early', () => { + assertDivision({ + timeAgos: [30000, 25000, 20000], + minEvents: 0, + minTimeMs: 5000, + expectedShortTermCount: 0, + }) + }) + + it('should handle empty changes array', () => { + const { shortTermChanges, longTermChanges } = divideGroupedChangesIntoShortTermAndLongTerm({ + changes: [], + minEvents: 2, + minTimeMs: 5000, + }) + expect(shortTermChanges).toEqual([]) + expect(longTermChanges).toEqual([]) + }) + + it('should handle single change', () => { + assertDivision({ + timeAgos: [1000], + minEvents: 2, + minTimeMs: 5000, + expectedShortTermCount: 1, + }) + }) + + it('should handle changes with identical timestamps', () => { + assertDivision({ + timeAgos: [1000, 1000, 1000], + minEvents: 2, + minTimeMs: 5000, + expectedShortTermCount: 3, + }) + }) + + it('should handle changes with zero time difference', () => { + assertDivision({ + timeAgos: [0, 0, 0], + minEvents: 0, + minTimeMs: 0, + expectedShortTermCount: 3, + }) + }) + + it('should handle negative minEvents parameter', () => { + assertDivision({ + timeAgos: [3000, 2000, 1000], + minEvents: -1, + minTimeMs: 5000, + expectedShortTermCount: 3, + }) + }) + + it('should handle zero minTimeMs parameter', () => { + assertDivision({ + timeAgos: [3000, 2000, 1000], + minEvents: 2, + minTimeMs: 0, + expectedShortTermCount: 2, + }) + }) + + it('should handle one minEvents', () => { + assertDivision({ + timeAgos: [3000, 2000, 1000], + minEvents: 1, + minTimeMs: 0, + expectedShortTermCount: 1, + }) + }) + + it('should handle zero minEvents and minTimeMs parameters', () => { + assertDivision({ + timeAgos: [3000, 2000, 1000], + minEvents: 0, + minTimeMs: 0, + expectedShortTermCount: 0, + }) + }) + + it('should handle very large time differences', () => { + assertDivision({ + timeAgos: [Number.MAX_SAFE_INTEGER, 2000, 1000], + minEvents: 1, + minTimeMs: 5000, + expectedShortTermCount: 2, + }) + }) +}) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts index 57f7771242ba..e32213afd456 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts @@ -212,19 +212,40 @@ export function getUnifiedDiffHunkFromTextDocumentChange(params: { } } -export function divideGroupedChangesIntoShortTermAndLongTerm(changes: TextDocumentChangeGroup[]): { +export function divideGroupedChangesIntoShortTermAndLongTerm(params: { + changes: TextDocumentChangeGroup[] + minEvents: number + minTimeMs: number +}): { shortTermChanges: TextDocumentChangeGroup[] longTermChanges: TextDocumentChangeGroup[] } { - if (changes.length <= 1) { - return { - shortTermChanges: changes, - longTermChanges: [], + const currentTimeStamp = Date.now() + + let longTermChangeIndex = params.changes.length - 1 + while (longTermChangeIndex >= 0) { + const group = params.changes[longTermChangeIndex] + const timestamp = Math.min(...group.changes.map(c => c.timestamp)) + const timeDiff = currentTimeStamp - timestamp + const eventCount = params.changes.length - longTermChangeIndex + if (eventCount > params.minEvents && timeDiff > params.minTimeMs) { + break } + longTermChangeIndex-- + } + const shortTermChanges = params.changes.slice(longTermChangeIndex + 1) + const longTermChanges = params.changes.slice(0, longTermChangeIndex + 1) + return { + shortTermChanges, + longTermChanges, } +} + +export function combineTextDocumentGroups(groups: TextDocumentChangeGroup[]): TextDocumentChangeGroup { return { - shortTermChanges: changes.slice(-1), - longTermChanges: changes.slice(0, -1), + changes: groups.flatMap(g => g.changes), + insertedRange: getRangeUnion(groups.map(g => g.insertedRange)), + replacementRange: getRangeUnion(groups.map(g => g.replacementRange)), } } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts index 238bff869dba..5fbf6e085a73 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.test.ts @@ -28,7 +28,12 @@ describe('RecentEditsRetriever', () => { maxAgeMs: FIVE_MINUTES, diffStrategyList: [ new UnifiedDiffStrategy({ addLineNumbers: false }), - new LineLevelDiffStrategy({ shouldGroupNonOverlappingLines: false }), + new LineLevelDiffStrategy({ + contextLines: 3, + longTermDiffCombinationStrategy: undefined, + minShortTermEvents: 1, + minShortTermTimeMs: 0, + }), ], }, { From 1a910322d3ea698d62453fdc646dc26b97813040 Mon Sep 17 00:00:00 2001 From: hitesh-1997 Date: Mon, 25 Nov 2024 07:06:47 +0530 Subject: [PATCH 18/18] metadata for logging --- lib/shared/src/completions/types.ts | 8 ++-- .../recent-edits-diff-helpers/base.ts | 3 +- .../line-level-diff.ts | 22 ++++++--- .../two-stage-unified-diff.ts | 11 ++++- .../recent-edits-diff-helpers/unified-diff.ts | 9 ++-- .../recent-edits-diff-helpers/utils.test.ts | 46 +++++++++++++++++++ .../recent-edits-diff-helpers/utils.ts | 25 +++++----- .../recent-edits-retriever.ts | 14 +++--- 8 files changed, 102 insertions(+), 36 deletions(-) diff --git a/lib/shared/src/completions/types.ts b/lib/shared/src/completions/types.ts index 91309b69fff6..c306717f9baa 100644 --- a/lib/shared/src/completions/types.ts +++ b/lib/shared/src/completions/types.ts @@ -1,6 +1,8 @@ import type * as vscode from 'vscode' import type { URI } from 'vscode-uri' +export type AutocompleteContextSnippetMetadataFields = Record + interface AutocompleteContextSnippetMetadata { /** * This field is relevant for user action context sources such as `recent-edit`, `recent-copy` and `recent-viewport`. @@ -8,11 +10,11 @@ interface AutocompleteContextSnippetMetadata { */ timeSinceActionMs?: number /** - * The diffing strategy used by the `recent-edits-retriever` to generate the diff. + * Additional metadata fields that can be used to store arbitrary key-value pairs. + * The values can be either numbers or strings. */ - recentEditsRetrieverDiffStrategy?: string + retrieverMetadata?: AutocompleteContextSnippetMetadataFields } - export interface AutocompleteFileContextSnippet { identifier: string uri: URI diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts index e69ad9a165f5..aaad20eccc43 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/base.ts @@ -1,9 +1,10 @@ import type { PromptString } from '@sourcegraph/cody-shared' import type * as vscode from 'vscode' +import type { AutocompleteContextSnippetMetadataFields } from '../../../../../../../lib/shared/src/completions/types' export interface RecentEditsRetrieverDiffStrategy { getDiffHunks(input: DiffCalculationInput): DiffHunk[] - getDiffStrategyName(): string + getDiffStrategyMetadata(): AutocompleteContextSnippetMetadataFields } export interface TextDocumentChange { diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts index 0c2d740d6812..463849c6a6c6 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/line-level-diff.ts @@ -1,4 +1,5 @@ import type * as vscode from 'vscode' +import type { AutocompleteContextSnippetMetadataFields } from '../../../../../../../lib/shared/src/completions/types' import type { DiffCalculationInput, DiffHunk, RecentEditsRetrieverDiffStrategy } from './base' import { groupNonOverlappingChangeGroups, groupOverlappingDocumentChanges } from './utils' import { @@ -66,15 +67,24 @@ export class LineLevelDiffStrategy implements RecentEditsRetrieverDiffStrategy { minEvents: this.options.minShortTermEvents, minTimeMs: this.options.minShortTermTimeMs, }) - if (this.options.longTermDiffCombinationStrategy === 'lines-based') { - longTermChanges = groupNonOverlappingChangeGroups(longTermChanges) - } else if (this.options.longTermDiffCombinationStrategy === 'unified-diff') { - longTermChanges = [combineTextDocumentGroups(longTermChanges)] + switch (this.options.longTermDiffCombinationStrategy) { + case 'lines-based': + longTermChanges = groupNonOverlappingChangeGroups(longTermChanges) + break + case 'unified-diff': + longTermChanges = [combineTextDocumentGroups(longTermChanges)] + break } return [...longTermChanges, ...shortTermChanges] } - public getDiffStrategyName(): string { - return 'line-level-diff' + public getDiffStrategyMetadata(): AutocompleteContextSnippetMetadataFields { + return { + strategy: 'line-level', + contextLines: this.options.contextLines, + longTermDiffCombinationStrategy: this.options.longTermDiffCombinationStrategy as string, + minShortTermEvents: this.options.minShortTermEvents, + minShortTermTimeMs: this.options.minShortTermTimeMs, + } } } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.ts index 271973359fbe..04d3ef577735 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/two-stage-unified-diff.ts @@ -1,3 +1,4 @@ +import type { AutocompleteContextSnippetMetadataFields } from '../../../../../../../lib/shared/src/completions/types' import type { DiffCalculationInput, DiffHunk, RecentEditsRetrieverDiffStrategy } from './base' import { groupOverlappingDocumentChanges } from './utils' import { @@ -49,7 +50,13 @@ export class TwoStageUnifiedDiffStrategy implements RecentEditsRetrieverDiffStra return diffs } - public getDiffStrategyName(): string { - return 'two-stage-unified-diff-strategy' + public getDiffStrategyMetadata(): AutocompleteContextSnippetMetadataFields { + return { + strategy: 'two-stage-unified-diff', + longTermContextLines: this.options.longTermContextLines, + shortTermContextLines: this.options.shortTermContextLines, + minShortTermEvents: this.options.minShortTermEvents, + minShortTermTimeMs: this.options.minShortTermTimeMs, + } } } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts index 1f5ab77caea7..8472d14d6d8f 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/unified-diff.ts @@ -1,3 +1,4 @@ +import type { AutocompleteContextSnippetMetadataFields } from '@sourcegraph/cody-shared' import type { DiffCalculationInput, DiffHunk, RecentEditsRetrieverDiffStrategy } from './base' import { getUnifiedDiffHunkFromTextDocumentChange } from './utils' @@ -28,9 +29,9 @@ export class UnifiedDiffStrategy implements RecentEditsRetrieverDiffStrategy { return diffHunk ? [diffHunk] : [] } - public getDiffStrategyName(): string { - return `unified-diff-strategy-${ - this.addLineNumbers ? 'with-line-numbers' : 'without-line-numbers' - }` + public getDiffStrategyMetadata(): AutocompleteContextSnippetMetadataFields { + return { + strategy: 'unified-diff', + } } } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts index f921c55717f7..95e366da087b 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.test.ts @@ -8,6 +8,7 @@ import { applyTextDocumentChanges, computeDiffWithLineNumbers, divideGroupedChangesIntoShortTermAndLongTerm, + getRangeUnion, groupConsecutiveItemsByPredicate, groupNonOverlappingChangeGroups, groupOverlappingDocumentChanges, @@ -575,3 +576,48 @@ describe('divideGroupedChangesIntoShortTermAndLongTerm', () => { }) }) }) + +describe('getRangeUnion', () => { + it('returns the union of multiple ranges', () => { + const ranges = [ + new vscode.Range(1, 0, 2, 10), + new vscode.Range(0, 5, 3, 0), + new vscode.Range(2, 0, 4, 15), + ] + + const result = getRangeUnion(ranges) + expect(result).toBeDefined() + expect(result?.start.line).toBe(0) + expect(result?.start.character).toBe(5) + expect(result?.end.line).toBe(4) + expect(result?.end.character).toBe(15) + }) + + it('handles single range correctly', () => { + const ranges = [new vscode.Range(1, 5, 2, 10)] + + const result = getRangeUnion(ranges) + expect(result).toBeDefined() + expect(result?.start.line).toBe(1) + expect(result?.start.character).toBe(5) + expect(result?.end.line).toBe(2) + expect(result?.end.character).toBe(10) + }) + + it('handles empty array of ranges', () => { + const ranges: vscode.Range[] = [] + const result = getRangeUnion(ranges) + expect(result).toBeUndefined() + }) + + it('handles overlapping ranges with same start and end', () => { + const ranges = [new vscode.Range(1, 1, 1, 1), new vscode.Range(1, 1, 1, 1)] + + const result = getRangeUnion(ranges) + expect(result).toBeDefined() + expect(result?.start.line).toBe(1) + expect(result?.start.character).toBe(1) + expect(result?.end.line).toBe(1) + expect(result?.end.character).toBe(1) + }) +}) diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts index e32213afd456..bc1da0e93097 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-diff-helpers/utils.ts @@ -19,12 +19,12 @@ export interface TextDocumentChangeGroup { /** * The union of the inserted ranges of all changes in this group */ - insertedRange: vscode.Range + insertedRange?: vscode.Range /** * The union of the replace ranges of all changes in this group */ - replacementRange: vscode.Range + replacementRange?: vscode.Range } /** @@ -46,7 +46,7 @@ export function groupOverlappingDocumentChanges( replacementRange: change.change.range, originalChange: change, })), - mergePredicate: (a, b) => doLineOverlapForRanges(a, b), + mergePredicate: (a, b) => Boolean(a && b && doLineOverlapForRanges(a, b)), getChanges: item => [item.originalChange], }) } @@ -68,7 +68,7 @@ export function groupNonOverlappingChangeGroups( ): TextDocumentChangeGroup[] { return mergeDocumentChanges({ items: groupedChanges, - mergePredicate: (a, b) => !doLineOverlapForRanges(a, b), + mergePredicate: (a, b) => Boolean(a && b && !doLineOverlapForRanges(a, b)), getChanges: group => group.changes, }) } @@ -82,10 +82,10 @@ export function groupNonOverlappingChangeGroups( * @returns Array of TextDocumentChangeGroup objects containing merged changes and their ranges */ function mergeDocumentChanges< - T extends { insertedRange: vscode.Range; replacementRange: vscode.Range }, + T extends { insertedRange?: vscode.Range; replacementRange?: vscode.Range }, >(args: { items: T[] - mergePredicate: (a: vscode.Range, b: vscode.Range) => boolean + mergePredicate: (a?: vscode.Range, b?: vscode.Range) => boolean getChanges: (item: T) => TextDocumentChange[] }): TextDocumentChangeGroup[] { if (args.items.length === 0) { @@ -105,13 +105,14 @@ function mergeDocumentChanges< })) } -function getRangeUnion(ranges: vscode.Range[]): vscode.Range { - if (ranges.length === 0) { - throw new Error('Cannot get union of empty ranges') +export function getRangeUnion(ranges: (vscode.Range | undefined)[]): vscode.Range | undefined { + const validRanges = ranges.filter((range): range is vscode.Range => range !== undefined) + if (validRanges.length === 0) { + return undefined } - let start = ranges[0].start - let end = ranges[0].end - for (const range of ranges) { + let start = validRanges[0].start + let end = validRanges[0].end + for (const range of validRanges) { start = start.isBefore(range.start) ? start : range.start end = end.isAfter(range.end) ? end : range.end } diff --git a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts index 88fc78ff1c3f..d8a74538b444 100644 --- a/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts +++ b/vscode/src/completions/context/retrievers/recent-user-actions/recent-edits-retriever.ts @@ -1,6 +1,7 @@ import { type PromptString, contextFiltersProvider } from '@sourcegraph/cody-shared' import type { AutocompleteContextSnippet } from '@sourcegraph/cody-shared' import * as vscode from 'vscode' +import type { AutocompleteContextSnippetMetadataFields } from '../../../../../../lib/shared/src/completions/types' import { getPositionAfterTextInsertion } from '../../../text-processing/utils' import type { ContextRetriever, ContextRetrieverOptions } from '../../../types' import { RetrieverIdentifier, type ShouldUseContextParams, shouldBeUsedAsContext } from '../../utils' @@ -19,7 +20,7 @@ interface TrackedDocument { } interface DiffHunkWithStrategy extends DiffHunk { - diffStrategyName: string + diffStrategyMetadata: AutocompleteContextSnippetMetadataFields } export interface RecentEditsRetrieverOptions { @@ -32,7 +33,7 @@ interface DiffAcrossDocuments { uri: vscode.Uri languageId: string latestChangeTimestamp: number - diffStrategyName: string + diffStrategyMetadata: AutocompleteContextSnippetMetadataFields } export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever { @@ -71,9 +72,6 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever const diffs = this.filterCandidateDiffs(rawDiffs, options.document) // Sort first by strategy name and then by timestamp diffs.sort((a, b) => { - if (a.diffStrategyName !== b.diffStrategyName) { - return a.diffStrategyName.localeCompare(b.diffStrategyName) - } return b.latestChangeTimestamp - a.latestChangeTimestamp }) @@ -87,7 +85,7 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever content, metadata: { timeSinceActionMs: retrievalTriggerTime - diff.latestChangeTimestamp, - recentEditsRetrieverDiffStrategy: diff.diffStrategyName, + retrieverMetadata: diff.diffStrategyMetadata, }, } satisfies Omit autocompleteContextSnippets.push(autocompleteSnippet) @@ -112,7 +110,7 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever uri: trackedDocument.uri, languageId: trackedDocument.languageId, latestChangeTimestamp: diffHunk.latestEditTimestamp, - diffStrategyName: diffHunk.diffStrategyName, + diffStrategyMetadata: diffHunk.diffStrategyMetadata, })) } return null @@ -162,7 +160,7 @@ export class RecentEditsRetriever implements vscode.Disposable, ContextRetriever for (const hunk of hunks) { diffHunks.push({ ...hunk, - diffStrategyName: diffStrategy.getDiffStrategyName(), + diffStrategyMetadata: diffStrategy.getDiffStrategyMetadata(), }) } }