From c0dd49b554314244f7fdf95cbf5966cd76e2d0eb Mon Sep 17 00:00:00 2001 From: Chris AtLee Date: Wed, 16 Oct 2024 10:46:08 -0400 Subject: [PATCH] Port fetchThemeAssets / fetchChecksums to graphQL --- .../admin/generated/get_theme_file_bodies.ts | 193 ++++++++++++++++++ .../generated/get_theme_file_checksums.ts | 119 +++++++++++ .../api/graphql/admin/generated/types.d.ts | 17 ++ .../queries/get_theme_file_bodies.graphql | 25 +++ .../queries/get_theme_file_checksums.graphql | 19 ++ packages/cli-kit/src/public/node/api/admin.ts | 10 +- .../src/public/node/themes/api.test.ts | 49 ++--- .../cli-kit/src/public/node/themes/api.ts | 111 ++++++++-- packages/theme/src/cli/constants.ts | 2 + packages/theme/src/cli/utilities/batching.ts | 32 +++ .../cli/utilities/theme-downloader.test.ts | 4 +- .../src/cli/utilities/theme-downloader.ts | 71 +++---- .../theme-environment/theme-polling.test.ts | 44 ++-- .../theme-environment/theme-polling.ts | 45 ++-- .../theme-reconciliation.test.ts | 43 ++-- .../theme-environment/theme-reconciliation.ts | 30 ++- .../theme/src/cli/utilities/theme-fs.test.ts | 48 +++-- 17 files changed, 698 insertions(+), 164 deletions(-) create mode 100644 packages/cli-kit/src/cli/api/graphql/admin/generated/get_theme_file_bodies.ts create mode 100644 packages/cli-kit/src/cli/api/graphql/admin/generated/get_theme_file_checksums.ts create mode 100644 packages/cli-kit/src/cli/api/graphql/admin/queries/get_theme_file_bodies.graphql create mode 100644 packages/cli-kit/src/cli/api/graphql/admin/queries/get_theme_file_checksums.graphql create mode 100644 packages/theme/src/cli/utilities/batching.ts diff --git a/packages/cli-kit/src/cli/api/graphql/admin/generated/get_theme_file_bodies.ts b/packages/cli-kit/src/cli/api/graphql/admin/generated/get_theme_file_bodies.ts new file mode 100644 index 0000000000..d5f382c56e --- /dev/null +++ b/packages/cli-kit/src/cli/api/graphql/admin/generated/get_theme_file_bodies.ts @@ -0,0 +1,193 @@ +/* eslint-disable @typescript-eslint/consistent-type-definitions */ +import * as Types from './types.js' + +import {TypedDocumentNode as DocumentNode} from '@graphql-typed-document-node/core' + +export type GetThemeFileBodiesQueryVariables = Types.Exact<{ + id: Types.Scalars['ID']['input'] + after?: Types.InputMaybe + filenames?: Types.InputMaybe +}> + +export type GetThemeFileBodiesQuery = { + theme?: { + files?: { + nodes: { + filename: string + size: unknown + checksumMd5?: string | null + body: + | {__typename: 'OnlineStoreThemeFileBodyBase64'; contentBase64: string} + | {__typename: 'OnlineStoreThemeFileBodyText'; content: string} + | {__typename: 'OnlineStoreThemeFileBodyUrl'; url: string} + }[] + userErrors: {filename: string; code: Types.OnlineStoreThemeFileResultType}[] + pageInfo: {hasNextPage: boolean; endCursor?: string | null} + } | null + } | null +} + +export const GetThemeFileBodies = { + kind: 'Document', + definitions: [ + { + kind: 'OperationDefinition', + operation: 'query', + name: {kind: 'Name', value: 'getThemeFileBodies'}, + variableDefinitions: [ + { + kind: 'VariableDefinition', + variable: {kind: 'Variable', name: {kind: 'Name', value: 'id'}}, + type: {kind: 'NonNullType', type: {kind: 'NamedType', name: {kind: 'Name', value: 'ID'}}}, + }, + { + kind: 'VariableDefinition', + variable: {kind: 'Variable', name: {kind: 'Name', value: 'after'}}, + type: {kind: 'NamedType', name: {kind: 'Name', value: 'String'}}, + }, + { + kind: 'VariableDefinition', + variable: {kind: 'Variable', name: {kind: 'Name', value: 'filenames'}}, + type: { + kind: 'ListType', + type: {kind: 'NonNullType', type: {kind: 'NamedType', name: {kind: 'Name', value: 'String'}}}, + }, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: {kind: 'Name', value: 'theme'}, + arguments: [ + { + kind: 'Argument', + name: {kind: 'Name', value: 'id'}, + value: {kind: 'Variable', name: {kind: 'Name', value: 'id'}}, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: {kind: 'Name', value: 'files'}, + arguments: [ + {kind: 'Argument', name: {kind: 'Name', value: 'first'}, value: {kind: 'IntValue', value: '250'}}, + { + kind: 'Argument', + name: {kind: 'Name', value: 'after'}, + value: {kind: 'Variable', name: {kind: 'Name', value: 'after'}}, + }, + { + kind: 'Argument', + name: {kind: 'Name', value: 'filenames'}, + value: {kind: 'Variable', name: {kind: 'Name', value: 'filenames'}}, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: {kind: 'Name', value: 'nodes'}, + selectionSet: { + kind: 'SelectionSet', + selections: [ + {kind: 'Field', name: {kind: 'Name', value: 'filename'}}, + {kind: 'Field', name: {kind: 'Name', value: 'size'}}, + {kind: 'Field', name: {kind: 'Name', value: 'checksumMd5'}}, + { + kind: 'Field', + name: {kind: 'Name', value: 'body'}, + selectionSet: { + kind: 'SelectionSet', + selections: [ + {kind: 'Field', name: {kind: 'Name', value: '__typename'}}, + { + kind: 'InlineFragment', + typeCondition: { + kind: 'NamedType', + name: {kind: 'Name', value: 'OnlineStoreThemeFileBodyText'}, + }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + {kind: 'Field', name: {kind: 'Name', value: 'content'}}, + {kind: 'Field', name: {kind: 'Name', value: '__typename'}}, + ], + }, + }, + { + kind: 'InlineFragment', + typeCondition: { + kind: 'NamedType', + name: {kind: 'Name', value: 'OnlineStoreThemeFileBodyBase64'}, + }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + {kind: 'Field', name: {kind: 'Name', value: 'contentBase64'}}, + {kind: 'Field', name: {kind: 'Name', value: '__typename'}}, + ], + }, + }, + { + kind: 'InlineFragment', + typeCondition: { + kind: 'NamedType', + name: {kind: 'Name', value: 'OnlineStoreThemeFileBodyUrl'}, + }, + selectionSet: { + kind: 'SelectionSet', + selections: [ + {kind: 'Field', name: {kind: 'Name', value: 'url'}}, + {kind: 'Field', name: {kind: 'Name', value: '__typename'}}, + ], + }, + }, + ], + }, + }, + {kind: 'Field', name: {kind: 'Name', value: '__typename'}}, + ], + }, + }, + { + kind: 'Field', + name: {kind: 'Name', value: 'userErrors'}, + selectionSet: { + kind: 'SelectionSet', + selections: [ + {kind: 'Field', name: {kind: 'Name', value: 'filename'}}, + {kind: 'Field', name: {kind: 'Name', value: 'code'}}, + {kind: 'Field', name: {kind: 'Name', value: '__typename'}}, + ], + }, + }, + { + kind: 'Field', + name: {kind: 'Name', value: 'pageInfo'}, + selectionSet: { + kind: 'SelectionSet', + selections: [ + {kind: 'Field', name: {kind: 'Name', value: 'hasNextPage'}}, + {kind: 'Field', name: {kind: 'Name', value: 'endCursor'}}, + {kind: 'Field', name: {kind: 'Name', value: '__typename'}}, + ], + }, + }, + {kind: 'Field', name: {kind: 'Name', value: '__typename'}}, + ], + }, + }, + {kind: 'Field', name: {kind: 'Name', value: '__typename'}}, + ], + }, + }, + ], + }, + }, + ], +} as unknown as DocumentNode diff --git a/packages/cli-kit/src/cli/api/graphql/admin/generated/get_theme_file_checksums.ts b/packages/cli-kit/src/cli/api/graphql/admin/generated/get_theme_file_checksums.ts new file mode 100644 index 0000000000..712229fa2e --- /dev/null +++ b/packages/cli-kit/src/cli/api/graphql/admin/generated/get_theme_file_checksums.ts @@ -0,0 +1,119 @@ +/* eslint-disable @typescript-eslint/consistent-type-definitions */ +import * as Types from './types.js' + +import {TypedDocumentNode as DocumentNode} from '@graphql-typed-document-node/core' + +export type GetThemeFileChecksumsQueryVariables = Types.Exact<{ + id: Types.Scalars['ID']['input'] + after?: Types.InputMaybe +}> + +export type GetThemeFileChecksumsQuery = { + theme?: { + files?: { + nodes: {filename: string; size: unknown; checksumMd5?: string | null}[] + userErrors: {filename: string; code: Types.OnlineStoreThemeFileResultType}[] + pageInfo: {hasNextPage: boolean; endCursor?: string | null} + } | null + } | null +} + +export const GetThemeFileChecksums = { + kind: 'Document', + definitions: [ + { + kind: 'OperationDefinition', + operation: 'query', + name: {kind: 'Name', value: 'getThemeFileChecksums'}, + variableDefinitions: [ + { + kind: 'VariableDefinition', + variable: {kind: 'Variable', name: {kind: 'Name', value: 'id'}}, + type: {kind: 'NonNullType', type: {kind: 'NamedType', name: {kind: 'Name', value: 'ID'}}}, + }, + { + kind: 'VariableDefinition', + variable: {kind: 'Variable', name: {kind: 'Name', value: 'after'}}, + type: {kind: 'NamedType', name: {kind: 'Name', value: 'String'}}, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: {kind: 'Name', value: 'theme'}, + arguments: [ + { + kind: 'Argument', + name: {kind: 'Name', value: 'id'}, + value: {kind: 'Variable', name: {kind: 'Name', value: 'id'}}, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: {kind: 'Name', value: 'files'}, + arguments: [ + {kind: 'Argument', name: {kind: 'Name', value: 'first'}, value: {kind: 'IntValue', value: '250'}}, + { + kind: 'Argument', + name: {kind: 'Name', value: 'after'}, + value: {kind: 'Variable', name: {kind: 'Name', value: 'after'}}, + }, + ], + selectionSet: { + kind: 'SelectionSet', + selections: [ + { + kind: 'Field', + name: {kind: 'Name', value: 'nodes'}, + selectionSet: { + kind: 'SelectionSet', + selections: [ + {kind: 'Field', name: {kind: 'Name', value: 'filename'}}, + {kind: 'Field', name: {kind: 'Name', value: 'size'}}, + {kind: 'Field', name: {kind: 'Name', value: 'checksumMd5'}}, + {kind: 'Field', name: {kind: 'Name', value: '__typename'}}, + ], + }, + }, + { + kind: 'Field', + name: {kind: 'Name', value: 'userErrors'}, + selectionSet: { + kind: 'SelectionSet', + selections: [ + {kind: 'Field', name: {kind: 'Name', value: 'filename'}}, + {kind: 'Field', name: {kind: 'Name', value: 'code'}}, + {kind: 'Field', name: {kind: 'Name', value: '__typename'}}, + ], + }, + }, + { + kind: 'Field', + name: {kind: 'Name', value: 'pageInfo'}, + selectionSet: { + kind: 'SelectionSet', + selections: [ + {kind: 'Field', name: {kind: 'Name', value: 'hasNextPage'}}, + {kind: 'Field', name: {kind: 'Name', value: 'endCursor'}}, + {kind: 'Field', name: {kind: 'Name', value: '__typename'}}, + ], + }, + }, + {kind: 'Field', name: {kind: 'Name', value: '__typename'}}, + ], + }, + }, + {kind: 'Field', name: {kind: 'Name', value: '__typename'}}, + ], + }, + }, + ], + }, + }, + ], +} as unknown as DocumentNode diff --git a/packages/cli-kit/src/cli/api/graphql/admin/generated/types.d.ts b/packages/cli-kit/src/cli/api/graphql/admin/generated/types.d.ts index 8fecf2958a..910af7f783 100644 --- a/packages/cli-kit/src/cli/api/graphql/admin/generated/types.d.ts +++ b/packages/cli-kit/src/cli/api/graphql/admin/generated/types.d.ts @@ -120,6 +120,23 @@ export type Scalars = { UtcOffset: {input: any; output: any} } +/** Type of a theme file operation result. */ +export type OnlineStoreThemeFileResultType = + /** Operation was malformed or invalid. */ + | 'BAD_REQUEST' + /** Operation faced a conflict with the current state of the file. */ + | 'CONFLICT' + /** Operation encountered an error. */ + | 'ERROR' + /** Operation file could not be found. */ + | 'NOT_FOUND' + /** Operation was successful. */ + | 'SUCCESS' + /** Operation timed out. */ + | 'TIMEOUT' + /** Operation could not be processed due to issues with input data. */ + | 'UNPROCESSABLE_ENTITY' + /** The input fields for Theme attributes to update. */ export type OnlineStoreThemeInput = { /** The new name of the theme. */ diff --git a/packages/cli-kit/src/cli/api/graphql/admin/queries/get_theme_file_bodies.graphql b/packages/cli-kit/src/cli/api/graphql/admin/queries/get_theme_file_bodies.graphql new file mode 100644 index 0000000000..5e27bc0538 --- /dev/null +++ b/packages/cli-kit/src/cli/api/graphql/admin/queries/get_theme_file_bodies.graphql @@ -0,0 +1,25 @@ +query getThemeFileBodies($id: ID!, $after: String, $filenames: [String!]) { + theme(id: $id) { + files(first: 250, after: $after, filenames: $filenames) { + nodes { + filename + size + checksumMd5 + body { + __typename + ... on OnlineStoreThemeFileBodyText { content } + ... on OnlineStoreThemeFileBodyBase64 { contentBase64 } + ... on OnlineStoreThemeFileBodyUrl { url } + } + } + userErrors { + filename + code + } + pageInfo { + hasNextPage + endCursor + } + } + } +} diff --git a/packages/cli-kit/src/cli/api/graphql/admin/queries/get_theme_file_checksums.graphql b/packages/cli-kit/src/cli/api/graphql/admin/queries/get_theme_file_checksums.graphql new file mode 100644 index 0000000000..52be7a3997 --- /dev/null +++ b/packages/cli-kit/src/cli/api/graphql/admin/queries/get_theme_file_checksums.graphql @@ -0,0 +1,19 @@ +query getThemeFileChecksums($id: ID!, $after: String) { + theme(id: $id) { + files(first: 250, after: $after) { + nodes { + filename + size + checksumMd5 + } + userErrors { + filename + code + } + pageInfo { + hasNextPage + endCursor + } + } + } +} diff --git a/packages/cli-kit/src/public/node/api/admin.ts b/packages/cli-kit/src/public/node/api/admin.ts index 1312fe5e6d..58d3ede497 100644 --- a/packages/cli-kit/src/public/node/api/admin.ts +++ b/packages/cli-kit/src/public/node/api/admin.ts @@ -15,6 +15,8 @@ import {themeKitAccessDomain} from '../../../private/node/constants.js' import {ClientError, Variables} from 'graphql-request' import {TypedDocumentNode} from '@graphql-typed-document-node/core' +const LatestApiVersionByFQDN = new Map() + /** * Executes a GraphQL query against the Admin API. * @@ -49,8 +51,8 @@ export async function adminRequestDoc( version?: string, responseOptions?: GraphQLResponseOptions, ): Promise { - let apiVersion = version - if (!version) { + let apiVersion = version ?? LatestApiVersionByFQDN.get(session.storeFqdn) + if (!apiVersion) { apiVersion = await fetchLatestSupportedApiVersion(session) } const store = await normalizeStoreFqdn(session.storeFqdn) @@ -80,7 +82,9 @@ function themeAccessHeaders(session: AdminSession): {[header: string]: string} { async function fetchLatestSupportedApiVersion(session: AdminSession): Promise { const apiVersions = await supportedApiVersions(session) // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - return apiVersions.reverse()[0]! + const latest = apiVersions.reverse()[0]! + LatestApiVersionByFQDN.set(session.storeFqdn, latest) + return latest } /** diff --git a/packages/cli-kit/src/public/node/themes/api.test.ts b/packages/cli-kit/src/public/node/themes/api.test.ts index 31a21dcfca..79dac76f5f 100644 --- a/packages/cli-kit/src/public/node/themes/api.test.ts +++ b/packages/cli-kit/src/public/node/themes/api.test.ts @@ -14,8 +14,9 @@ import {RemoteBulkUploadResponse} from './factories.js' import {ThemeDelete} from '../../../cli/api/graphql/admin/generated/theme_delete.js' import {ThemeUpdate} from '../../../cli/api/graphql/admin/generated/theme_update.js' import {ThemePublish} from '../../../cli/api/graphql/admin/generated/theme_publish.js' +import {GetThemeFileChecksums} from '../../../cli/api/graphql/admin/generated/get_theme_file_checksums.js' import {test, vi, expect, describe} from 'vitest' -import {adminRequestDoc, restRequest} from '@shopify/cli-kit/node/api/admin' +import {adminRequestDoc, restRequest, supportedApiVersions} from '@shopify/cli-kit/node/api/admin' import {AbortError} from '@shopify/cli-kit/node/error' vi.mock('@shopify/cli-kit/node/api/admin') @@ -57,29 +58,30 @@ describe('fetchThemes', () => { }) }) -describe('fetwchChecksums', () => { +describe('fetchChecksums', () => { test('returns theme checksums', async () => { // Given - vi.mocked(restRequest).mockResolvedValue({ - json: { - assets: [ - { - key: 'snippets/product-variant-picker.liquid', - checksum: '29e2e56057c3b58c02bc7946d7600481', - }, - { - key: 'templates/404.json', - checksum: 'f14a0bd594f4fee47b13fc09543098ff', - }, - { - key: 'templates/article.json', - // May be null if an asset has not been updated recently. - checksum: null, - }, - ], + vi.mocked(supportedApiVersions).mockResolvedValue(['2024-10']) + vi.mocked(adminRequestDoc).mockResolvedValue({ + theme: { + files: { + nodes: [ + { + filename: 'snippets/product-variant-picker.liquid', + checksumMd5: '29e2e56057c3b58c02bc7946d7600481', + }, + { + filename: 'templates/404.json', + checksumMd5: 'f14a0bd594f4fee47b13fc09543098ff', + }, + { + filename: 'templates/article.json', + checksumMd5: null, + }, + ], + pageInfo: {hasNextPage: false, endCursor: null}, + }, }, - status: 200, - headers: {}, }) // When @@ -87,8 +89,9 @@ describe('fetwchChecksums', () => { const checksum = await fetchChecksums(id, session) // Then - expect(restRequest).toHaveBeenCalledWith('GET', `/themes/${id}/assets`, session, undefined, { - fields: 'key,checksum', + expect(adminRequestDoc).toHaveBeenCalledWith(GetThemeFileChecksums, session, { + id: `gid://shopify/OnlineStoreTheme/${id}`, + after: null, }) expect(checksum).toHaveLength(3) expect(checksum[0]!.key).toEqual('snippets/product-variant-picker.liquid') diff --git a/packages/cli-kit/src/public/node/themes/api.ts b/packages/cli-kit/src/public/node/themes/api.ts index e3a346387e..eb445567be 100644 --- a/packages/cli-kit/src/public/node/themes/api.ts +++ b/packages/cli-kit/src/public/node/themes/api.ts @@ -4,15 +4,12 @@ import * as throttler from '../api/rest-api-throttler.js' import {ThemeUpdate} from '../../../cli/api/graphql/admin/generated/theme_update.js' import {ThemeDelete} from '../../../cli/api/graphql/admin/generated/theme_delete.js' import {ThemePublish} from '../../../cli/api/graphql/admin/generated/theme_publish.js' +import {GetThemeFileBodies} from '../../../cli/api/graphql/admin/generated/get_theme_file_bodies.js' +import {GetThemeFileChecksums} from '../../../cli/api/graphql/admin/generated/get_theme_file_checksums.js' import {restRequest, RestResponse, adminRequestDoc} from '@shopify/cli-kit/node/api/admin' import {AdminSession} from '@shopify/cli-kit/node/session' import {AbortError} from '@shopify/cli-kit/node/error' -import { - buildBulkUploadResults, - buildChecksum, - buildTheme, - buildThemeAsset, -} from '@shopify/cli-kit/node/themes/factories' +import {buildBulkUploadResults, buildTheme} from '@shopify/cli-kit/node/themes/factories' import {Result, Checksum, Key, Theme, ThemeAsset} from '@shopify/cli-kit/node/themes/types' import {outputDebug} from '@shopify/cli-kit/node/output' import {sleep} from '@shopify/cli-kit/node/system' @@ -45,12 +42,45 @@ export async function createTheme(params: ThemeParams, session: AdminSession): P return buildTheme({...response.json.theme, createdAtRuntime: true}) } -export async function fetchThemeAsset(id: number, key: Key, session: AdminSession): Promise { - const response = await request('GET', `/themes/${id}/assets`, session, undefined, { - 'asset[key]': key, - }) - - return buildThemeAsset(response.json.asset) +export async function fetchThemeAssets(id: number, filenames: Key[], session: AdminSession): Promise { + const assets: ThemeAsset[] = [] + let after: string | null = null + + while (true) { + // eslint-disable-next-line no-await-in-loop + const response = await adminRequestDoc(GetThemeFileBodies, session, { + id: themeGid(id), + filenames, + after, + }) + + if (!response.theme?.files?.nodes || !response.theme?.files?.pageInfo) { + const userErrors = response.theme?.files?.userErrors.map((error) => error.filename).join(', ') + throw new AbortError(`Error fetching assets: ${userErrors}`) + } + + const {nodes, pageInfo} = response.theme.files + + assets.push( + // eslint-disable-next-line no-await-in-loop + ...(await Promise.all( + nodes.map(async (file) => { + const content = await parseThemeFileContent(file.body) + return { + key: file.filename, + checksum: file.checksumMd5 as string, + value: content, + } + }), + )), + ) + + if (!pageInfo.hasNextPage) { + return assets + } + + after = pageInfo.endCursor as string + } } export async function deleteThemeAsset(id: number, key: Key, session: AdminSession): Promise { @@ -73,12 +103,33 @@ export async function bulkUploadThemeAssets( } export async function fetchChecksums(id: number, session: AdminSession): Promise { - const response = await request('GET', `/themes/${id}/assets`, session, undefined, {fields: 'key,checksum'}) - const assets = response.json.assets + const checksums: Checksum[] = [] + let after: string | null = null - if (assets?.length > 0) return assets.map(buildChecksum) + while (true) { + // eslint-disable-next-line no-await-in-loop + const response = await adminRequestDoc(GetThemeFileChecksums, session, {id: themeGid(id), after}) - return [] + if (!response?.theme?.files?.nodes || !response?.theme?.files?.pageInfo) { + const userErrors = response.theme?.files?.userErrors.map((error) => error.filename).join(', ') + throw new AbortError(`Failed to fetch checksums for: ${userErrors}`) + } + + const {nodes, pageInfo} = response.theme.files + + checksums.push( + ...nodes.map((file) => ({ + key: file.filename, + checksum: file.checksumMd5 as string, + })), + ) + + if (!pageInfo.hasNextPage) { + return checksums + } + + after = pageInfo.endCursor as string + } } export async function themeUpdate(id: number, params: ThemeParams, session: AdminSession): Promise { @@ -274,3 +325,31 @@ async function handleRetriableError({path, retries, retry, fail}: RetriableError await sleep(0.2) return retry() } + +function themeGid(id: number): string { + return `gid://shopify/OnlineStoreTheme/${id}` +} + +type OnlineStoreThemeFileBody = + | {__typename: 'OnlineStoreThemeFileBodyBase64'; contentBase64: string} + | {__typename: 'OnlineStoreThemeFileBodyText'; content: string} + | {__typename: 'OnlineStoreThemeFileBodyUrl'; url: string} + +async function parseThemeFileContent(body: OnlineStoreThemeFileBody): Promise { + switch (body.__typename) { + case 'OnlineStoreThemeFileBodyText': + return body.content + break + case 'OnlineStoreThemeFileBodyBase64': + return Buffer.from(body.contentBase64, 'base64').toString() + break + case 'OnlineStoreThemeFileBodyUrl': + try { + const response = await fetch(body.url) + return await response.text() + } catch (error) { + // Raise error if we can't download the file + throw new AbortError(`Error downloading content from URL: ${body.url}`) + } + } +} diff --git a/packages/theme/src/cli/constants.ts b/packages/theme/src/cli/constants.ts index fe25ef8666..b8a86bd926 100644 --- a/packages/theme/src/cli/constants.ts +++ b/packages/theme/src/cli/constants.ts @@ -32,3 +32,5 @@ export const DEFAULT_IGNORE_PATTERNS = [ '**/node_modules/', '.prettierrc.json', ] + +export const MAX_GRAPHQL_THEME_FILES = 50 diff --git a/packages/theme/src/cli/utilities/batching.ts b/packages/theme/src/cli/utilities/batching.ts new file mode 100644 index 0000000000..f142a8e18d --- /dev/null +++ b/packages/theme/src/cli/utilities/batching.ts @@ -0,0 +1,32 @@ +export function batchedRequests( + items: TItem[], + batchSize: number, + fn: (batch: TItem[]) => Promise, +): Promise[] { + const requests = [] + for (let i = 0; i < items.length; i += batchSize) { + const batch = items.slice(i, i + batchSize) + requests.push(fn(batch)) + } + return requests +} + +export interface Task { + title: string + task: () => Promise +} + +export function batchedTasks( + items: TItem[], + batchSize: number, + fn: (batch: TItem[], start: number) => Task, +): Task[] { + const tasks: Task[] = [] + + for (let i = 0; i < items.length; i += batchSize) { + const batch = items.slice(i, i + batchSize) + tasks.push(fn(batch, i)) + } + + return tasks +} diff --git a/packages/theme/src/cli/utilities/theme-downloader.test.ts b/packages/theme/src/cli/utilities/theme-downloader.test.ts index 055c3785e1..bde58d971d 100644 --- a/packages/theme/src/cli/utilities/theme-downloader.test.ts +++ b/packages/theme/src/cli/utilities/theme-downloader.test.ts @@ -1,6 +1,6 @@ import {downloadTheme} from './theme-downloader.js' import {fakeThemeFileSystem} from './theme-fs/theme-fs-mock-factory.js' -import {fetchThemeAsset} from '@shopify/cli-kit/node/themes/api' +import {fetchThemeAssets} from '@shopify/cli-kit/node/themes/api' import {Checksum, ThemeAsset} from '@shopify/cli-kit/node/themes/types' import {test, describe, expect, vi} from 'vitest' @@ -85,7 +85,7 @@ describe('theme-downloader', () => { ] const spy = vi.spyOn(fileSystem, 'write') - vi.mocked(fetchThemeAsset).mockResolvedValue(fileToDownload) + vi.mocked(fetchThemeAssets).mockResolvedValue([fileToDownload]) // When await downloadTheme(remoteTheme, adminSession, remote, fileSystem, downloadOptions) diff --git a/packages/theme/src/cli/utilities/theme-downloader.ts b/packages/theme/src/cli/utilities/theme-downloader.ts index 85f082fc8a..3e29ceefe9 100644 --- a/packages/theme/src/cli/utilities/theme-downloader.ts +++ b/packages/theme/src/cli/utilities/theme-downloader.ts @@ -1,6 +1,8 @@ +import {batchedTasks, Task} from './batching.js' +import {MAX_GRAPHQL_THEME_FILES} from '../constants.js' import {AdminSession} from '@shopify/cli-kit/node/session' -import {fetchThemeAsset} from '@shopify/cli-kit/node/themes/api' -import {ThemeFileSystem, Theme, Checksum} from '@shopify/cli-kit/node/themes/types' +import {fetchThemeAssets} from '@shopify/cli-kit/node/themes/api' +import {ThemeFileSystem, Theme, Checksum, ThemeAsset} from '@shopify/cli-kit/node/themes/types' import {renderTasks} from '@shopify/cli-kit/node/ui' interface DownloadOptions { @@ -45,44 +47,39 @@ function buildDownloadTasks( theme: Theme, themeFileSystem: ThemeFileSystem, session: AdminSession, -) { - const checksums = themeFileSystem.applyIgnoreFilters(remoteChecksums) - - return checksums - .map((checksum) => { - const remoteChecksumValue = checksum.checksum - const localAsset = themeFileSystem.files.get(checksum.key) - - if (localAsset?.checksum === remoteChecksumValue) { - return - } - - const progress = progressPct(remoteChecksums, checksum) - const title = `Pulling theme "${theme.name}" (#${theme.id}) from ${session.storeFqdn} [${progress}%]` - - return { - title, - task: async () => downloadFile(theme, themeFileSystem, checksum, session), - } - }) - .filter(notNull) -} - -async function downloadFile(theme: Theme, fileSystem: ThemeFileSystem, checksum: Checksum, session: AdminSession) { - const themeAsset = await fetchThemeAsset(theme.id, checksum.key, session) +): Task[] { + let checksums = themeFileSystem.applyIgnoreFilters(remoteChecksums) + + // Filter out files we already have + checksums = checksums.filter((checksum) => { + const remoteChecksumValue = checksum.checksum + const localAsset = themeFileSystem.files.get(checksum.key) + + if (localAsset?.checksum === remoteChecksumValue) { + return false + } else { + return true + } + }) - if (!themeAsset) return + const filenames = checksums.map((checksum) => checksum.key) - await fileSystem.write(themeAsset) + const batches = batchedTasks(filenames, MAX_GRAPHQL_THEME_FILES, (batchedFilenames, i) => { + const title = `Downloading files ${i}..${i + batchedFilenames.length} / ${filenames.length} files` + return { + title, + task: async () => downloadFiles(theme, themeFileSystem, batchedFilenames, session), + } + }) + return batches } -function progressPct(themeChecksums: Checksum[], checksum: Checksum): number { - const current = themeChecksums.indexOf(checksum) + 1 - const total = themeChecksums.length +async function downloadFiles(theme: Theme, fileSystem: ThemeFileSystem, filenames: string[], session: AdminSession) { + const assets = await fetchThemeAssets(theme.id, filenames, session) + if (!assets) return - return Math.round((current / total) * 100) -} - -function notNull(value: T | null | undefined): value is T { - return value !== null && value !== undefined + // eslint-disable-next-line @typescript-eslint/no-misused-promises + assets.forEach(async (asset: ThemeAsset) => { + await fileSystem.write(asset) + }) } diff --git a/packages/theme/src/cli/utilities/theme-environment/theme-polling.test.ts b/packages/theme/src/cli/utilities/theme-environment/theme-polling.test.ts index 45dfbbca45..4e53b3f6cf 100644 --- a/packages/theme/src/cli/utilities/theme-environment/theme-polling.test.ts +++ b/packages/theme/src/cli/utilities/theme-environment/theme-polling.test.ts @@ -1,7 +1,7 @@ import {PollingOptions, pollRemoteJsonChanges, deleteRemovedAssets} from './theme-polling.js' import {fakeThemeFileSystem} from '../theme-fs/theme-fs-mock-factory.js' import {AbortError} from '@shopify/cli-kit/node/error' -import {fetchChecksums, fetchThemeAsset} from '@shopify/cli-kit/node/themes/api' +import {fetchChecksums, fetchThemeAssets} from '@shopify/cli-kit/node/themes/api' import {Checksum, ThemeAsset} from '@shopify/cli-kit/node/themes/types' import {describe, expect, test, vi} from 'vitest' import {buildTheme} from '@shopify/cli-kit/node/themes/factories' @@ -21,7 +21,7 @@ describe('pollRemoteJsonChanges', async () => { const remoteChecksums = [{checksum: '1', key: 'templates/asset.json'}] const updatedRemoteChecksums = [{checksum: '2', key: 'templates/asset.json'}] vi.mocked(fetchChecksums).mockResolvedValue(updatedRemoteChecksums) - vi.mocked(fetchThemeAsset).mockResolvedValue({checksum: '2', key: 'templates/asset.json', value: 'content'}) + vi.mocked(fetchThemeAssets).mockResolvedValue([{checksum: '2', key: 'templates/asset.json', value: 'content'}]) // When await pollRemoteJsonChanges(developmentTheme, adminSession, remoteChecksums, themeFileSystem, defaultOptions) @@ -40,7 +40,7 @@ describe('pollRemoteJsonChanges', async () => { const remoteChecksums: Checksum[] = [] const updatedRemoteChecksums = [{checksum: '1', key: 'templates/asset.json'}] vi.mocked(fetchChecksums).mockResolvedValue(updatedRemoteChecksums) - vi.mocked(fetchThemeAsset).mockResolvedValue({checksum: '1', key: 'templates/asset.json', value: 'content'}) + vi.mocked(fetchThemeAssets).mockResolvedValue([{checksum: '1', key: 'templates/asset.json', value: 'content'}]) // When await pollRemoteJsonChanges(developmentTheme, adminSession, remoteChecksums, themeFileSystem, defaultOptions) @@ -88,7 +88,7 @@ describe('pollRemoteJsonChanges', async () => { }) // Then - expect(fetchThemeAsset).not.toHaveBeenCalled() + expect(fetchThemeAssets).not.toHaveBeenCalled() expect(themeFileSystem.files.get('templates/asset.json')).toBeUndefined() }) @@ -106,7 +106,7 @@ describe('pollRemoteJsonChanges', async () => { }) // Then - expect(fetchThemeAsset).not.toHaveBeenCalled() + expect(fetchThemeAssets).not.toHaveBeenCalled() expect(themeFileSystem.files.get('templates/asset.json')).toEqual({checksum: '1', key: 'templates/asset.json'}) }) @@ -120,7 +120,7 @@ describe('pollRemoteJsonChanges', async () => { const themeFileSystem = fakeThemeFileSystem('tmp', new Map()) themeFileSystem.read = async (fileKey: string) => { themeFileSystem.files.set(fileKey, {checksum: '3', key: fileKey}) - return themeFileSystem.files.get(fileKey)?.value || themeFileSystem.files.get(fileKey)?.attachment + return themeFileSystem.files.get(fileKey)?.value ?? themeFileSystem.files.get(fileKey)?.attachment } // When @@ -147,7 +147,7 @@ describe('pollRemoteJsonChanges', async () => { await pollRemoteJsonChanges(developmentTheme, adminSession, remoteChecksums, themeFileSystem, defaultOptions) // Then - expect(fetchThemeAsset).not.toHaveBeenCalled() + expect(fetchThemeAssets).not.toHaveBeenCalled() expect(spy).not.toHaveBeenCalled() }) @@ -177,7 +177,7 @@ describe('pollRemoteJsonChanges', async () => { {checksum: '2', key: 'templates/asset.json'}, {checksum: '2', key: 'templates/asset2.json'}, ] - vi.mocked(fetchThemeAsset).mockResolvedValue({checksum: '2', key: 'templates/asset.json', value: 'content'}) + vi.mocked(fetchThemeAssets).mockResolvedValue([{checksum: '2', key: 'templates/asset.json', value: 'content'}]) vi.mocked(fetchChecksums).mockResolvedValue(updatedRemoteChecksums) // When @@ -186,8 +186,8 @@ describe('pollRemoteJsonChanges', async () => { }) // Then - expect(fetchThemeAsset).toHaveBeenCalledOnce() - expect(fetchThemeAsset).toHaveBeenCalledWith(1, 'templates/asset.json', adminSession) + expect(fetchThemeAssets).toHaveBeenCalledOnce() + expect(fetchThemeAssets).toHaveBeenCalledWith(1, ['templates/asset.json'], adminSession) expect(themeFileSystem.files.get('templates/asset.json')).toEqual({ checksum: '2', key: 'templates/asset.json', @@ -207,7 +207,7 @@ describe('pollRemoteJsonChanges', async () => { {checksum: '2', key: 'templates/asset.json'}, {checksum: '2', key: 'templates/asset2.json'}, ] - vi.mocked(fetchThemeAsset).mockResolvedValue({checksum: '2', key: 'templates/asset2.json', value: 'content'}) + vi.mocked(fetchThemeAssets).mockResolvedValue([{checksum: '2', key: 'templates/asset2.json', value: 'content'}]) vi.mocked(fetchChecksums).mockResolvedValue(updatedRemoteChecksums) // When @@ -216,8 +216,8 @@ describe('pollRemoteJsonChanges', async () => { }) // Then - expect(fetchThemeAsset).toHaveBeenCalledOnce() - expect(fetchThemeAsset).toHaveBeenCalledWith(1, 'templates/asset2.json', adminSession) + expect(fetchThemeAssets).toHaveBeenCalledOnce() + expect(fetchThemeAssets).toHaveBeenCalledWith(1, ['templates/asset2.json'], adminSession) expect(themeFileSystem.files.get('templates/asset2.json')).toEqual({ checksum: '2', key: 'templates/asset2.json', @@ -238,11 +238,13 @@ describe('pollRemoteJsonChanges', async () => { {checksum: '6', key: 'templates/asset3.json'}, ] vi.mocked(fetchChecksums).mockResolvedValue(updatedRemoteChecksums) - vi.mocked(fetchThemeAsset).mockImplementation(async (_, key) => ({ - checksum: '2', - key, - value: 'content', - })) + vi.mocked(fetchThemeAssets).mockResolvedValue([ + { + checksum: '2', + key: 'templates/asset2.json', + value: 'content', + }, + ]) const themeFileSystem = { ...fakeThemeFileSystem('tmp', new Map()), @@ -253,10 +255,8 @@ describe('pollRemoteJsonChanges', async () => { await pollRemoteJsonChanges(developmentTheme, adminSession, remoteChecksums, themeFileSystem, defaultOptions) // Then - expect(fetchThemeAsset).toHaveBeenCalledTimes(2) - expect(fetchThemeAsset).toHaveBeenCalledWith(1, 'templates/asset1.json', adminSession) - expect(fetchThemeAsset).toHaveBeenCalledWith(1, 'templates/asset3.json', adminSession) - expect(fetchThemeAsset).not.toHaveBeenCalledWith(1, 'templates/asset2.json', adminSession) + expect(fetchThemeAssets).toHaveBeenCalledWith(1, ['templates/asset1.json', 'templates/asset3.json'], adminSession) + expect(fetchThemeAssets).not.toHaveBeenCalledWith(1, ['templates/asset2.json'], adminSession) }) }) diff --git a/packages/theme/src/cli/utilities/theme-environment/theme-polling.ts b/packages/theme/src/cli/utilities/theme-environment/theme-polling.ts index 67bfbaf861..4ecf2f79f4 100644 --- a/packages/theme/src/cli/utilities/theme-environment/theme-polling.ts +++ b/packages/theme/src/cli/utilities/theme-environment/theme-polling.ts @@ -1,7 +1,8 @@ -import {timestampDateFormat} from '../../constants.js' +import {MAX_GRAPHQL_THEME_FILES, timestampDateFormat} from '../../constants.js' +import {batchedRequests} from '../batching.js' import {renderThrownError} from '../errors.js' import {Checksum, Theme, ThemeFileSystem} from '@shopify/cli-kit/node/themes/types' -import {fetchChecksums, fetchThemeAsset} from '@shopify/cli-kit/node/themes/api' +import {fetchChecksums, fetchThemeAssets} from '@shopify/cli-kit/node/themes/api' import {outputDebug, outputInfo, outputContent, outputToken} from '@shopify/cli-kit/node/output' import {AdminSession} from '@shopify/cli-kit/node/session' import {renderFatalError} from '@shopify/cli-kit/node/ui' @@ -121,22 +122,32 @@ async function syncChangedAssets( localFileSystem: ThemeFileSystem, assetsChangedOnRemote: Checksum[], ) { - await Promise.all( - assetsChangedOnRemote.map(async (file) => { - if (localFileSystem.files.get(file.key)?.checksum === file.checksum) { - return - } - const asset = await fetchThemeAsset(targetTheme.id, file.key, currentSession) - if (asset) { - await localFileSystem.write(asset) - outputInfo( - outputContent`• ${timestampDateFormat.format(new Date())} Synced ${outputToken.raw('»')} ${outputToken.gray( - `download ${asset.key} from remote theme`, - )}`, - ) - } - }), + const filesToGet = assetsChangedOnRemote.filter( + (file) => localFileSystem.files.get(file.key)?.checksum !== file.checksum, ) + + const chunks = batchedRequests(filesToGet, MAX_GRAPHQL_THEME_FILES, async (chunk) => { + return fetchThemeAssets( + targetTheme.id, + chunk.map((file) => file.key), + currentSession, + ).then((assets) => { + return Promise.all( + assets.map(async (asset) => { + if (asset) { + await localFileSystem.write(asset) + outputInfo( + outputContent`• ${timestampDateFormat.format(new Date())} Synced ${outputToken.raw( + '»', + )} ${outputToken.gray(`download ${asset.key} from remote theme`)}`, + ) + } + }), + ) + }) + }) + + await Promise.all(chunks) } export async function deleteRemovedAssets( diff --git a/packages/theme/src/cli/utilities/theme-environment/theme-reconciliation.test.ts b/packages/theme/src/cli/utilities/theme-environment/theme-reconciliation.test.ts index c6ddcf0704..e20e8b8897 100644 --- a/packages/theme/src/cli/utilities/theme-environment/theme-reconciliation.test.ts +++ b/packages/theme/src/cli/utilities/theme-environment/theme-reconciliation.test.ts @@ -1,12 +1,12 @@ import {reconcileJsonFiles} from './theme-reconciliation.js' import {REMOTE_STRATEGY, LOCAL_STRATEGY} from './remote-theme-watcher.js' import {fakeThemeFileSystem} from '../theme-fs/theme-fs-mock-factory.js' -import {deleteThemeAsset, fetchThemeAsset} from '@shopify/cli-kit/node/themes/api' +import {deleteThemeAsset, fetchThemeAssets} from '@shopify/cli-kit/node/themes/api' import {buildTheme} from '@shopify/cli-kit/node/themes/factories' -import {Checksum, ThemeAsset} from '@shopify/cli-kit/node/themes/types' +import {Checksum, ThemeAsset, ThemeFileSystem} from '@shopify/cli-kit/node/themes/types' import {DEVELOPMENT_THEME_ROLE} from '@shopify/cli-kit/node/themes/utils' import {renderSelectPrompt} from '@shopify/cli-kit/node/ui' -import {describe, expect, test, vi} from 'vitest' +import {beforeEach, describe, expect, test, vi} from 'vitest' vi.mock('@shopify/cli-kit/node/ui') vi.mock('@shopify/cli-kit/node/themes/api') @@ -19,13 +19,20 @@ describe('reconcileJsonFiles', () => { const adminSession = {token: '', storeFqdn: ''} const remoteChecksums: Checksum[] = [{checksum: '1', key: 'config/settings_schema.json'}] const files = new Map([]) - const defaultThemeFileSystem = fakeThemeFileSystem('tmp', files) const defaultOptions = {noDelete: false, only: [], ignore: []} + let defaultThemeFileSystem: ThemeFileSystem + + beforeEach(() => { + defaultThemeFileSystem = fakeThemeFileSystem('tmp', new Map([])) + }) + describe('file filters', () => { test('should only reconcile JSON files', async () => { // Given vi.mocked(renderSelectPrompt).mockResolvedValue(REMOTE_STRATEGY) + vi.mocked(fetchThemeAssets).mockResolvedValue([{checksum: '1', key: 'templates/template.json', value: 'content'}]) + const remoteChecksums = [ {checksum: '1', key: 'templates/template.json', value: 'content'}, {checksum: '2', key: 'sections/section.liquid', value: 'content'}, @@ -42,13 +49,15 @@ describe('reconcileJsonFiles', () => { ) // Then - expect(fetchThemeAsset).toHaveBeenCalledTimes(1) - expect(fetchThemeAsset).toHaveBeenCalledWith(developmentTheme.id, 'templates/template.json', adminSession) + expect(fetchThemeAssets).toHaveBeenCalledTimes(1) + expect(fetchThemeAssets).toHaveBeenCalledWith(developmentTheme.id, ['templates/template.json'], adminSession) }) test('should only reconcile files that match the `only` option', async () => { // Given vi.mocked(renderSelectPrompt).mockResolvedValue(REMOTE_STRATEGY) + vi.mocked(fetchThemeAssets).mockResolvedValue([{checksum: '1', key: 'templates/template.json', value: 'content'}]) + const remoteChecksums = [ {checksum: '1', key: 'templates/template.json', value: 'content'}, {checksum: '2', key: 'sections/section.liquid', value: 'content'}, @@ -69,8 +78,8 @@ describe('reconcileJsonFiles', () => { ) // Then - expect(fetchThemeAsset).toHaveBeenCalledTimes(1) - expect(fetchThemeAsset).toHaveBeenCalledWith(developmentTheme.id, 'templates/template.json', adminSession) + expect(fetchThemeAssets).toHaveBeenCalledTimes(1) + expect(fetchThemeAssets).toHaveBeenCalledWith(developmentTheme.id, ['templates/template.json'], adminSession) }) test('should not reconcile files that match the `ignore` option', async () => { @@ -90,7 +99,7 @@ describe('reconcileJsonFiles', () => { }) // Then - expect(fetchThemeAsset).toHaveBeenCalledTimes(0) + expect(fetchThemeAssets).toHaveBeenCalledTimes(0) }) }) @@ -99,22 +108,22 @@ describe('reconcileJsonFiles', () => { // Given vi.mocked(renderSelectPrompt).mockResolvedValue(REMOTE_STRATEGY) const assetToBeDownloaded = {checksum: '2', key: 'templates/second_asset.json', value: 'content'} - const remoteChecksums = [assetToBeDownloaded] + const remoteChecksums = assetToBeDownloaded - vi.mocked(fetchThemeAsset).mockResolvedValue(assetToBeDownloaded) + vi.mocked(fetchThemeAssets).mockResolvedValue([assetToBeDownloaded]) // When expect(defaultThemeFileSystem.files.get('templates/asset.json')).toBeUndefined() await reconcileAndWaitForReconciliationFinish( developmentTheme, adminSession, - remoteChecksums, + [remoteChecksums], defaultThemeFileSystem, defaultOptions, ) // Then - expect(fetchThemeAsset).toHaveBeenCalledWith(developmentTheme.id, assetToBeDownloaded.key, adminSession) + expect(fetchThemeAssets).toHaveBeenCalledWith(developmentTheme.id, [assetToBeDownloaded.key], adminSession) expect(defaultThemeFileSystem.files.get('templates/second_asset.json')).toEqual(assetToBeDownloaded) }) @@ -143,6 +152,8 @@ describe('reconcileJsonFiles', () => { test('should delete files from local disk when `remote` source is selected', async () => { // Given vi.mocked(renderSelectPrompt).mockResolvedValue(REMOTE_STRATEGY) + vi.mocked(fetchThemeAssets).mockResolvedValue([]) + const files = new Map([['templates/asset.json', {checksum: '1', key: 'templates/asset.json'}]]) const localThemeFileSystem = fakeThemeFileSystem('tmp', files) const spy = vi.spyOn(localThemeFileSystem, 'delete') @@ -212,6 +223,8 @@ describe('reconcileJsonFiles', () => { test('should download files from remote theme when `remote` source is selected', async () => { // Given vi.mocked(renderSelectPrompt).mockResolvedValue(REMOTE_STRATEGY) + vi.mocked(fetchThemeAssets).mockResolvedValue([{checksum: '1', key: 'templates/asset.json', value: 'content'}]) + const files = new Map([['templates/asset.json', {checksum: '1', key: 'templates/asset.json'}]]) const localThemeFileSystem = fakeThemeFileSystem('tmp', files) const remoteChecksums = [{checksum: '2', key: 'templates/asset.json'}] @@ -226,7 +239,7 @@ describe('reconcileJsonFiles', () => { ) // Then - expect(fetchThemeAsset).toHaveBeenCalled() + expect(fetchThemeAssets).toHaveBeenCalled() }) test('should not download files from remote when `local` source is selected', async () => { @@ -246,7 +259,7 @@ describe('reconcileJsonFiles', () => { ) // Then - expect(fetchThemeAsset).not.toHaveBeenCalled() + expect(fetchThemeAssets).not.toHaveBeenCalled() }) }) diff --git a/packages/theme/src/cli/utilities/theme-environment/theme-reconciliation.ts b/packages/theme/src/cli/utilities/theme-environment/theme-reconciliation.ts index 63ebfa715f..7d973f4a08 100644 --- a/packages/theme/src/cli/utilities/theme-environment/theme-reconciliation.ts +++ b/packages/theme/src/cli/utilities/theme-environment/theme-reconciliation.ts @@ -1,7 +1,9 @@ import {REMOTE_STRATEGY, LOCAL_STRATEGY} from './remote-theme-watcher.js' +import {batchedRequests} from '../batching.js' +import {MAX_GRAPHQL_THEME_FILES} from '../../constants.js' import {outputDebug} from '@shopify/cli-kit/node/output' import {AdminSession} from '@shopify/cli-kit/node/session' -import {fetchThemeAsset, deleteThemeAsset} from '@shopify/cli-kit/node/themes/api' +import {deleteThemeAsset, fetchThemeAssets} from '@shopify/cli-kit/node/themes/api' import {Checksum, ThemeFileSystem, ThemeAsset, Theme} from '@shopify/cli-kit/node/themes/types' import {renderInfo, renderSelectPrompt} from '@shopify/cli-kit/node/ui' @@ -38,8 +40,10 @@ export async function reconcileJsonFiles( outputDebug('Initiating theme asset reconciliation process') - const {filesOnlyPresentLocally, filesOnlyPresentOnRemote, filesWithConflictingChecksums} = - await identifyFilesToReconcile(remoteChecksums, localThemeFileSystem) + const {filesOnlyPresentLocally, filesOnlyPresentOnRemote, filesWithConflictingChecksums} = identifyFilesToReconcile( + remoteChecksums, + localThemeFileSystem, + ) if ( filesOnlyPresentLocally.length === 0 && @@ -159,12 +163,22 @@ async function performFileReconciliation( const {localFilesToDelete, filesToDownload, remoteFilesToDelete} = partitionedFiles const deleteLocalFiles = localFilesToDelete.map((file) => localThemeFileSystem.delete(file.key)) - const downloadRemoteFiles = filesToDownload.map(async (file) => { - const asset = await fetchThemeAsset(targetTheme.id, file.key, session) - if (asset) { - return localThemeFileSystem.write(asset) - } + + const downloadRemoteFiles = batchedRequests(filesToDownload, MAX_GRAPHQL_THEME_FILES, async (chunk) => { + const assets = await fetchThemeAssets( + targetTheme.id, + chunk.map((file) => file.key), + session, + ) + return Promise.all( + assets.map((asset) => { + if (asset) { + return localThemeFileSystem.write(asset) + } + }), + ) }) + const deleteRemoteFiles = remoteFilesToDelete.map((file) => deleteThemeAsset(targetTheme.id, file.key, session)) await Promise.all([...deleteLocalFiles, ...downloadRemoteFiles, ...deleteRemoteFiles]) diff --git a/packages/theme/src/cli/utilities/theme-fs.test.ts b/packages/theme/src/cli/utilities/theme-fs.test.ts index a2c1298568..09b2b386ff 100644 --- a/packages/theme/src/cli/utilities/theme-fs.test.ts +++ b/packages/theme/src/cli/utilities/theme-fs.test.ts @@ -11,7 +11,7 @@ import {getPatternsFromShopifyIgnore, applyIgnoreFilters} from './asset-ignore.j import {removeFile, writeFile} from '@shopify/cli-kit/node/fs' import {test, describe, expect, vi, beforeEach} from 'vitest' import chokidar from 'chokidar' -import {deleteThemeAsset, fetchThemeAsset} from '@shopify/cli-kit/node/themes/api' +import {deleteThemeAsset, fetchThemeAssets} from '@shopify/cli-kit/node/themes/api' import {renderError} from '@shopify/cli-kit/node/ui' import EventEmitter from 'events' import type {Checksum, ThemeAsset} from '@shopify/cli-kit/node/themes/types' @@ -499,13 +499,15 @@ describe('theme-fs', () => { test('deletes file from remote theme', async () => { // Given - vi.mocked(fetchThemeAsset).mockResolvedValue({ - key: 'assets/base.css', - checksum: '1', - value: 'content', - attachment: '', - stats: {size: 100, mtime: 100}, - }) + vi.mocked(fetchThemeAssets).mockResolvedValue([ + { + key: 'assets/base.css', + checksum: '1', + value: 'content', + attachment: '', + stats: {size: 100, mtime: 100}, + }, + ]) vi.mocked(deleteThemeAsset).mockResolvedValue(true) // When @@ -532,13 +534,15 @@ describe('theme-fs', () => { test('does not delete file from remote when options.noDelete is true', async () => { // Given - vi.mocked(fetchThemeAsset).mockResolvedValue({ - key: 'assets/base.css', - checksum: '1', - value: 'content', - attachment: '', - stats: {size: 100, mtime: 100}, - }) + vi.mocked(fetchThemeAssets).mockResolvedValue([ + { + key: 'assets/base.css', + checksum: '1', + value: 'content', + attachment: '', + stats: {size: 100, mtime: 100}, + }, + ]) vi.mocked(deleteThemeAsset).mockResolvedValue(true) // When @@ -565,12 +569,14 @@ describe('theme-fs', () => { test('renders a warning to debug if the file deletion fails', async () => { // Given - vi.mocked(fetchThemeAsset).mockResolvedValue({ - key: 'assets/base.css', - value: 'file content', - checksum: '1', - stats: {size: 100, mtime: 100}, - }) + vi.mocked(fetchThemeAssets).mockResolvedValue([ + { + key: 'assets/base.css', + value: 'file content', + checksum: '1', + stats: {size: 100, mtime: 100}, + }, + ]) vi.mocked(deleteThemeAsset).mockResolvedValue(false) // When