feat(cli): [WIP] Add support for OpenAPI input w/out fs

generators/go-v2/dynamic-snippets/package.json

@@ -36,7 +36,6 @@
     "@fern-api/go-formatter": "workspace:*",
     "@fern-fern/ir-sdk": "^53.21.0",
     "@types/jest": "^29.5.12",
-    "@types/node": "^18.7.18",
     "depcheck": "^1.4.6",
     "eslint": "^8.56.0",
     "organize-imports-cli": "^0.10.0",

generators/go-v2/dynamic-snippets/src/context/DynamicSnippetsGeneratorContext.ts

@@ -8,7 +8,6 @@ import { DiscriminatedUnionTypeInstance } from "../DiscriminatedUnionTypeInstanc
 import { DynamicTypeMapper } from "./DynamicTypeMapper";
 import { DynamicTypeInstantiationMapper } from "./DynamicTypeInstantiationMapper";
 import { go } from "@fern-api/go-ast";
-import path from "path";
 import { ErrorReporter, Severity } from "./ErrorReporter";
 import { FilePropertyMapper } from "./FilePropertyMapper";
 import { AbstractDynamicSnippetsGeneratorContext } from "@fern-api/generator-commons";
@@ -329,11 +328,11 @@ export class DynamicSnippetsGeneratorContext extends AbstractDynamicSnippetsGene
     }
 
     public getClientImportPath(): string {
-        return path.join(this.rootImportPath, "client");
+        return `${this.rootImportPath}/client`;
     }
 
     public getOptionImportPath(): string {
-        return path.join(this.rootImportPath, "option");
+        return `${this.rootImportPath}/option`;
     }
 
     public resolveEndpointOrThrow(rawEndpoint: string): DynamicSnippets.Endpoint[] {

packages/cli/api-importers/openapi/openapi-ir-parser/src/index.ts

@@ -1,4 +1,4 @@
 export { type ParseOpenAPIOptions } from "./options";
-export { parse, type Spec } from "./parse";
+export { parse, parseOpenAPISpecs, type Spec, type ParsedOpenAPISpec } from "./parse";
 export { generateEnumNameFromValue, VALID_ENUM_NAME_REGEX } from "./schema/convertEnum";
 export { isSchemaEqual } from "./schema/utils/isSchemaEqual";

packages/cli/api-importers/openapi/openapi-ir-parser/src/loadOpenAPI.ts

@@ -7,6 +7,7 @@ import { NodeType } from "@redocly/openapi-core/lib/types";
 import { OpenAPI } from "openapi-types";
 import { mergeWithOverrides } from "./mergeWithOverrides";
 import { FernOpenAPIExtension } from "./openapi/v3/extensions/fernExtensions";
+import { mergeWithOverrides as coreMergeWithOverrides } from "@fern-api/core-utils";
 
 const XFernStreaming: NodeType = {
     properties: {
@@ -78,6 +79,28 @@ export async function loadOpenAPI({
     return parsed;
 }
 
+export async function loadParsedOpenAPI({
+    openapi,
+    overrides
+}: {
+    openapi: OpenAPI.Document;
+    overrides: OpenAPI.Document | undefined;
+}): Promise<OpenAPI.Document> {
+    const memoryFilepath = AbsoluteFilePath.of("<memory>");
+    const parsed = await parseOpenAPI({
+        absolutePathToOpenAPI: memoryFilepath,
+        parsed: openapi
+    });
+    if (overrides != null) {
+        const merged = await coreMergeWithOverrides({ data: parsed, overrides });
+        return await parseOpenAPI({
+            absolutePathToOpenAPI: memoryFilepath,
+            parsed: merged
+        });
+    }
+    return parsed;
+}
+
 async function parseOpenAPI({
     absolutePathToOpenAPI,
     parsed

packages/cli/api-importers/openapi/openapi-ir-parser/src/parse.ts

@@ -7,7 +7,7 @@ import { OpenAPI, OpenAPIV2, OpenAPIV3 } from "openapi-types";
 import { DEFAULT_PARSE_ASYNCAPI_SETTINGS, ParseAsyncAPIOptions } from "./asyncapi/options";
 import { parseAsyncAPI } from "./asyncapi/parse";
 import { AsyncAPIV2 } from "./asyncapi/v2";
-import { loadOpenAPI } from "./loadOpenAPI";
+import { loadOpenAPI, loadParsedOpenAPI } from "./loadOpenAPI";
 import { mergeWithOverrides } from "./mergeWithOverrides";
 import { generateIr as generateIrFromV2 } from "./openapi/v2/generateIr";
 import { generateIr as generateIrFromV3 } from "./openapi/v3/generateIr";
@@ -21,6 +21,13 @@ export interface Spec {
     settings?: SpecImportSettings;
 }
 
+export interface ParsedOpenAPISpec {
+    parsed: OpenAPI.Document;
+    overrides?: OpenAPI.Document;
+    namespace?: string;
+    settings?: SpecImportSettings;
+}
+
 export interface SpecImportSettings {
     audiences: string[];
     shouldUseTitleAsName: boolean;
@@ -169,6 +176,69 @@ export async function parse({
     return ir;
 }
 
+export async function parseOpenAPISpecs({
+    specs,
+    taskContext,
+    optionOverrides
+}: {
+    specs: ParsedOpenAPISpec[];
+    taskContext: TaskContext;
+    optionOverrides?: Partial<ParseOpenAPIOptions>;
+}): Promise<OpenApiIntermediateRepresentation> {
+    let ir: OpenApiIntermediateRepresentation = {
+        apiVersion: undefined,
+        title: undefined,
+        description: undefined,
+        basePath: undefined,
+        servers: [],
+        tags: {
+            tagsById: {},
+            orderedTagIds: undefined
+        },
+        hasEndpointsMarkedInternal: false,
+        endpoints: [],
+        webhooks: [],
+        channel: [],
+        groupedSchemas: {
+            rootSchemas: {},
+            namespacedSchemas: {}
+        },
+        variables: {},
+        nonRequestReferencedSchemas: new Set(),
+        securitySchemes: {},
+        globalHeaders: [],
+        idempotencyHeaders: [],
+        groups: {}
+    };
+    const source = OpenApiIrSource.openapi({ file: "<memory>" });
+    for (const spec of specs) {
+        const parsed = await loadParsedOpenAPI({
+            openapi: spec.parsed,
+            overrides: spec.overrides
+        });
+        if (isOpenApiV3(parsed)) {
+            const openapiIr = generateIrFromV3({
+                openApi: parsed,
+                taskContext,
+                options: getParseOptions({ specSettings: spec.settings, overrides: optionOverrides }),
+                source,
+                namespace: spec.namespace
+            });
+            ir = merge(ir, openapiIr);
+        } else if (isOpenApiV2(parsed)) {
+            const openapiIr = await generateIrFromV2({
+                openApi: parsed,
+                taskContext,
+                options: getParseOptions({ specSettings: spec.settings }),
+                source,
+                namespace: spec.namespace
+            });
+            ir = merge(ir, openapiIr);
+        }
+    }
+    return ir;
+}
+
 function getParseOptions({
     specSettings,
     overrides

packages/cli/lazy-fern-workspace/package.json

@@ -49,6 +49,7 @@
     "js-yaml": "^4.1.0",
     "lodash-es": "^4.17.21",
     "object-hash": "^3.0.0",
+    "openapi-types": "^12.1.3",
     "tar": "^6.2.1",
     "tmp-promise": "^3.0.3",
     "uuid": "^9.0.1",

packages/cli/lazy-fern-workspace/src/ParsedOpenAPIWorkspace.ts

@@ -0,0 +1,199 @@
+import { AbstractAPIWorkspace, FernDefinition, FernWorkspace } from "@fern-api/api-workspace-commons";
+import { OSSWorkspace, SpecImportSettings } from "./OSSWorkspace";
+import { OpenAPI } from "openapi-types";
+import { AbsoluteFilePath, RelativeFilePath } from "@fern-api/fs-utils";
+import { TaskContext } from "@fern-api/task-context";
+import { OpenApiIntermediateRepresentation } from "@fern-api/openapi-ir";
+import { ParseOpenAPIOptions, parseOpenAPISpecs } from "@fern-api/openapi-ir-parser";
+import { mapValues } from "./utils/mapValues";
+import { convert } from "@fern-api/openapi-ir-to-fern";
+import { FERN_PACKAGE_MARKER_FILENAME } from "@fern-api/configuration";
+import yaml from "js-yaml";
+
+export interface ParsedOpenAPISpec {
+    type: "openapi";
+    parsed: OpenAPI.Document;
+    overrides?: OpenAPI.Document;
+    namespace?: string;
+    settings?: SpecImportSettings;
+}
+
+export declare namespace ParsedOpenAPIWorkspace {
+    export interface Args extends AbstractAPIWorkspace.Args {
+        specs: ParsedOpenAPISpec[];
+    }
+
+    // TODO: Move this to a shared location.
+    export interface Settings {
+        /*
+         * Whether or not to parse unique errors for OpenAPI operation. This is
+         * an option that is typically enabled for docs generation.
+         */
+        enableUniqueErrorsPerEndpoint?: boolean;
+        /*
+         * Whether or not to parse discriminated unions as undiscriminated unions with literals.
+         * Typically enabled for duck typed languages like Python / TypeScript.
+         */
+        enableDiscriminatedUnionV2?: boolean;
+        /*
+         * Whether or not to extract frequently used headers out of the endpoints into a
+         * global header. This is primarily used for generating SDKs, but disabled for docs
+         * as it allows the documentation to more closely mirror the OpenAPI spec.
+         */
+        detectGlobalHeaders?: boolean;
+        /*
+         * Whether or not to let additional property values in OpenAPI come through as
+         * optional.
+         */
+        optionalAdditionalProperties?: boolean;
+        /*
+         * Whether or not to cooerce enums to undiscriminated union literals.
+         */
+        cooerceEnumsToLiterals?: boolean;
+        /*
+         * Whether or not to parse object query parameters.
+         */
+        objectQueryParameters?: boolean;
+        /*
+         * Whether or not to preserve original schema ids.
+         */
+        preserveSchemaIds?: boolean;
+    }
+}
+
+export class ParsedOpenAPIWorkspace extends AbstractAPIWorkspace<ParsedOpenAPIWorkspace.Settings> {
+    public specs: ParsedOpenAPISpec[];
+
+    private respectReadonlySchemas: boolean;
+    private onlyIncludeReferencedSchemas: boolean;
+    private inlinePathParameters: boolean;
+
+    constructor({ specs, ...superArgs }: ParsedOpenAPIWorkspace.Args) {
+        super(superArgs);
+        this.specs = specs;
+        this.respectReadonlySchemas = this.specs.every((spec) => spec.settings?.respectReadonlySchemas ?? false);
+        this.onlyIncludeReferencedSchemas = this.specs.every(
+            (spec) => spec.settings?.onlyIncludeReferencedSchemas ?? false
+        );
+        this.inlinePathParameters = this.specs.every((spec) => spec.settings?.inlinePathParameters ?? false);
+    }
+
+    public async getDefinition({
+        context,
+        settings
+    }: {
+        context: TaskContext;
+        settings?: OSSWorkspace.Settings;
+    }): Promise<FernDefinition> {
+        const openApiIr = await this.getOpenAPIIr({ context, settings });
+
+        const objectQueryParameters = this.specs.every((spec) => spec.settings?.objectQueryParameters);
+        const definition = convert({
+            authOverrides:
+                this.generatorsConfiguration?.api?.auth != null ? { ...this.generatorsConfiguration?.api } : undefined,
+            environmentOverrides:
+                this.generatorsConfiguration?.api?.environments != null
+                    ? { ...this.generatorsConfiguration?.api }
+                    : undefined,
+            globalHeaderOverrides:
+                this.generatorsConfiguration?.api?.headers != null
+                    ? { ...this.generatorsConfiguration?.api }
+                    : undefined,
+            taskContext: context,
+            ir: openApiIr,
+            enableUniqueErrorsPerEndpoint: settings?.enableUniqueErrorsPerEndpoint ?? false,
+            detectGlobalHeaders: settings?.detectGlobalHeaders ?? true,
+            objectQueryParameters,
+            respectReadonlySchemas: this.respectReadonlySchemas,
+            onlyIncludeReferencedSchemas: this.onlyIncludeReferencedSchemas,
+            inlinePathParameters: this.inlinePathParameters
+        });
+
+        return {
+            // These files are held in-memory, so there's no absolute filepath.
+            absoluteFilePath: AbsoluteFilePath.of("/DUMMY_PATH"),
+            rootApiFile: {
+                defaultUrl: definition.rootApiFile["default-url"],
+                contents: definition.rootApiFile,
+                rawContents: yaml.dump(definition.rootApiFile)
+            },
+            namedDefinitionFiles: {
+                ...mapValues(definition.definitionFiles, (definitionFile) => ({
+                    absoluteFilepath: AbsoluteFilePath.of("/DUMMY_PATH"),
+                    rawContents: yaml.dump(definitionFile),
+                    contents: definitionFile
+                })),
+                [RelativeFilePath.of(FERN_PACKAGE_MARKER_FILENAME)]: {
+                    absoluteFilepath: AbsoluteFilePath.of("/DUMMY_PATH"),
+                    rawContents: yaml.dump(definition.packageMarkerFile),
+                    contents: definition.packageMarkerFile
+                }
+            },
+            packageMarkers: {},
+            importedDefinitions: {}
+        };
+    }
+
+    public async getOpenAPIIr({
+        context,
+        settings
+    }: {
+        context: TaskContext;
+        settings?: ParsedOpenAPIWorkspace.Settings;
+    }): Promise<OpenApiIntermediateRepresentation> {
+        const optionOverrides = getOptionsOverridesFromSettings(settings);
+        return await parseOpenAPISpecs({
+            specs: this.specs,
+            taskContext: context,
+            optionOverrides: {
+                ...optionOverrides,
+                respectReadonlySchemas: this.respectReadonlySchemas,
+                onlyIncludeReferencedSchemas: this.onlyIncludeReferencedSchemas
+            }
+        });
+    }
+
+    public async toFernWorkspace(
+        { context }: { context: TaskContext },
+        settings?: OSSWorkspace.Settings
+    ): Promise<FernWorkspace> {
+        const definition = await this.getDefinition({ context, settings });
+        return new FernWorkspace({
+            absoluteFilePath: this.absoluteFilePath,
+            workspaceName: this.workspaceName,
+            generatorsConfiguration: this.generatorsConfiguration,
+            dependenciesConfiguration: {
+                dependencies: {}
+            },
+            definition,
+            cliVersion: this.cliVersion
+        });
+    }
+
+    public getAbsoluteFilePaths(): AbsoluteFilePath[] {
+        return [];
+    }
+}
+
+// TODO: Move this to a shared location.
+function getOptionsOverridesFromSettings(
+    settings?: ParsedOpenAPIWorkspace.Settings
+): Partial<ParseOpenAPIOptions> | undefined {
+    if (settings == null) {
+        return undefined;
+    }
+    const result: Partial<ParseOpenAPIOptions> = {};
+    if (settings.enableDiscriminatedUnionV2) {
+        result.discriminatedUnionV2 = true;
+    }
+    if (settings.optionalAdditionalProperties) {
+        result.optionalAdditionalProperties = true;
+    }
+    if (settings.cooerceEnumsToLiterals) {
+        result.cooerceEnumsToLiterals = true;
+    }
+    if (settings.preserveSchemaIds) {
+        result.preserveSchemaIds = true;
+    }
+    return result;
+}

packages/cli/lazy-fern-workspace/src/index.ts

@@ -1,5 +1,6 @@
 export { type AsyncAPISource, type OpenAPISource, type ProtobufSource, type Source } from "./OSSWorkspace";
 export * from "./utils";
+export * from "./ParsedOpenAPIWorkspace";
 export * from "./LazyFernWorkspace";
 export * from "./OSSWorkspace";
 export * from "./ConjureWorkspace";

packages/cli/lazy-fern-workspace/src/utils/Result.ts

@@ -4,13 +4,14 @@ import { OSSWorkspace } from "../OSSWorkspace";
 import { LazyFernWorkspace } from "../LazyFernWorkspace";
 import { ConjureWorkspace } from "../ConjureWorkspace";
 import { validateAgainstJsonSchema } from "@fern-api/core-utils";
+import { ParsedOpenAPIWorkspace } from "../ParsedOpenAPIWorkspace";
 
 export declare namespace WorkspaceLoader {
     export type Result = SuccessfulResult | FailedResult;
 
     export interface SuccessfulResult {
         didSucceed: true;
-        workspace: LazyFernWorkspace | OSSWorkspace | ConjureWorkspace;
+        workspace: LazyFernWorkspace | OSSWorkspace | ParsedOpenAPIWorkspace | ConjureWorkspace;
     }
 
     export interface FailedResult {