Use new doc urls and prepare for spec version 3.6 (#204)

* Match new operation ids * Operation mapping with new id pattern * Use the new doc URL paths * Download newest docs * Correct symbolKind * Better package resolution * Hierarchy fixes * Refetch docs
MortenGregersen · Oct 2, 2024 · 78ae9f0 · 78ae9f0
1 parent 1777711
commit 78ae9f0
Show file tree

Hide file tree

Showing 24 changed files with 94,029 additions and 98,473 deletions.
diff --git a/Documentation/OperationDocumentation.json b/Documentation/OperationDocumentation.json
diff --git a/Documentation/SchemaDocumentation.json b/Documentation/SchemaDocumentation.json
diff --git a/Documentation/SchemaIndex.json b/Documentation/SchemaIndex.json
diff --git a/Sources/BagbutikCLI/BagbutikCLI.swift b/Sources/BagbutikCLI/BagbutikCLI.swift
@@ -71,6 +71,8 @@ struct BagbutikCLI: AsyncParsableCommand {
 
         @Option(name: .shortAndLong, help: "The output folder for the generated files. Should contain the current Endpoints and Models.")
         var outputPath = "./Documentation"
+
+        @Flag var dryRun: Bool = false
 
         func run() async throws {
             let specFileURL: URL
@@ -81,7 +83,7 @@ struct BagbutikCLI: AsyncParsableCommand {
                 specFileURL = try await downloadNewestSpec()
             }
             let outputDirURL = URL(fileURLWithPath: outputPath)
-            try await DocsFetcher().fetchAllDocs(specFileURL: specFileURL, outputDirURL: outputDirURL)
+            try await DocsFetcher().fetchAllDocs(specFileURL: specFileURL, outputDirURL: outputDirURL, dryRun: dryRun)
         }
     }
 

diff --git a/Sources/BagbutikDocsCollector/DocsFetcher.swift b/Sources/BagbutikDocsCollector/DocsFetcher.swift
@@ -84,27 +84,28 @@ public class DocsFetcher {
         self.print = print
     }
 
-    public func fetchAllDocs(specFileURL: URL, outputDirURL: URL) async throws {
+    public func fetchAllDocs(specFileURL: URL, outputDirURL: URL, dryRun: Bool) async throws {
         guard specFileURL.isFileURL else { throw DocsFetcherError.notFileUrl(.specFileURL) }
         print("🔍 Loading spec \(specFileURL.path)...")
         let spec = try loadSpec(specFileURL)
 
         var operationDocumentationById = [String: Documentation]()
-        let operationIds = spec.paths.values.map(\.operations).flatMap { $0 }.map(\.id).sorted()
-        for operationId in operationIds {
-            if let operationUrl = createJsonDocumentationUrl(fromOperationId: operationId) {
-                print("Fetching documentation for operation '\(operationId)' (\(operationUrl))")
-                let documentation = try await fetchDocumentation(for: operationUrl)
-                operationDocumentationById[operationId] = documentation
+        let operationIdsAndDocUrls: [(operationId: String, docUrl: URL)] = spec.paths.values
+            .flatMap { path in
+                path.operations.map {
+                    (operationId: $0.id, docUrl: createJsonDocumentationUrl(fromOperation: $0, in: path))
+                }
+            }
+            .sorted(by: { $0.operationId < $1.operationId })
+        for (operationId, docUrl) in operationIdsAndDocUrls {
+            if dryRun {
+                print("Would fetch documentation for operation '\(operationId)' (\(docUrl))")
             } else {
-                print("⚠️ Documentation URL missing for operation: '\(operationId)'")
+                print("Fetching documentation for operation '\(operationId)' (\(docUrl))")
+                let documentation = try await fetchDocumentation(for: docUrl)
+                operationDocumentationById[operationId] = documentation
             }
         }
-        // Comment in to get a list of obsolete URLs
-//        let operationIdsWithUrlButNotInSpec = OperationMapping.allMappings.keys.filter { !operationIds.contains($0) }
-//        operationIdsWithUrlButNotInSpec.forEach { operationId in
-//            print("⚠️ Documentation URL exists for removed operation: '\(operationId)'")
-//        }
 
         var identifierBySchemaName = [String: String]()
         var schemaDocumentationById = [String: Documentation]()
@@ -195,13 +196,13 @@ public class DocsFetcher {
     private func createJsonDocumentationUrl(fromDocId id: String) -> URL {
         URL(string: id
             .replacingOccurrences(
-                of: "doc://com.apple.documentation/documentation",
+                of: "doc://com.apple.appstoreconnectapi/documentation",
                 with: "https://developer.apple.com/tutorials/data/documentation")
             .appending(".json"))!
     }
 
-    private func createJsonDocumentationUrl(fromOperationId operationId: String) -> URL? {
-        guard let urlPath = OperationMapping.allMappings[operationId]?.appending(".json") else { return nil }
-        return URL(string: "https://developer.apple.com/tutorials/data/documentation/appstoreconnectapi/" + urlPath)!
+    private func createJsonDocumentationUrl(fromOperation operation: BagbutikSpecDecoder.Operation, in path: Path) -> URL {
+        let urlPath = operation.getDocumentationId(path: path).appending(".json")
+        return URL(string: "https://developer.apple.com/tutorials/data/documentation/AppStoreConnectAPI/" + urlPath)!
     }
 }
diff --git a/Sources/BagbutikDocsCollector/DocsLoader.swift b/Sources/BagbutikDocsCollector/DocsLoader.swift
@@ -71,7 +71,7 @@ public actor DocsLoader {
         guard let packageName = packageNames.first else {
             let paths = documentation.hierarchy.paths.flatMap { $0 }
             guard let longestPath = paths.sorted(by: { $0.lengthOfBytes(using: .utf8) > $1.lengthOfBytes(using: .utf8) }).first,
-                  longestPath == "doc://com.apple.documentation/documentation/appstoreconnectapi" else {
+                  longestPath == "doc://com.apple.appstoreconnectapi/documentation/AppStoreConnectAPI" else {
                 throw DocsLoaderError.couldNotResolvePackageName(id: documentation.id, paths: documentation.hierarchy.paths)
             }
             if documentation.id.hasSuffix("ageratingdeclarationwithoutincludesresponse")
@@ -115,11 +115,7 @@ public actor DocsLoader {
         return documentation
     }
 
-    public func createUrlForOperation(withId operationId: String) -> String {
-        "https://developer.apple.com/documentation/appstoreconnectapi/" + OperationMapping.allMappings[operationId]!
-    }
-
     private func createDocumentationId(fromUrl url: String) -> String {
-        url.replacingOccurrences(of: "https://developer.apple.com", with: "doc://com.apple.documentation")
+        url.replacingOccurrences(of: "https://developer.apple.com", with: "doc://com.apple.appstoreconnectapi")
     }
 }
diff --git a/Sources/BagbutikDocsCollector/Models/Documentation.swift b/Sources/BagbutikDocsCollector/Models/Documentation.swift
@@ -3,13 +3,16 @@ import Foundation
 
 public enum Documentation: Codable, Equatable, Sendable {
     case `enum`(EnumDocumentation)
+    case `typealias`(TypealiasDocumentation)
     case object(ObjectDocumentation)
     case operation(OperationDocumentation)
 
     public var id: String {
         switch self {
         case .enum(let documentation):
             documentation.id
+        case .typealias(let documentation):
+            documentation.id
         case .object(let documentation):
             documentation.id
         case .operation(let documentation):
@@ -21,6 +24,8 @@ public enum Documentation: Codable, Equatable, Sendable {
         switch self {
         case .enum(let documentation):
             documentation.hierarchy
+        case .typealias(let documentation):
+            documentation.hierarchy
         case .object(let documentation):
             documentation.hierarchy
         case .operation(let documentation):
@@ -32,6 +37,8 @@ public enum Documentation: Codable, Equatable, Sendable {
         switch self {
         case .enum(let documentation):
             documentation.title
+        case .typealias(let documentation):
+            documentation.title
         case .object(let documentation):
             documentation.title
         case .operation(let documentation):
@@ -43,6 +50,8 @@ public enum Documentation: Codable, Equatable, Sendable {
         switch self {
         case .enum(let documentation):
             documentation.abstract
+        case .typealias(let documentation):
+            documentation.abstract
         case .object(let documentation):
             documentation.abstract
         case .operation(let documentation):
@@ -54,6 +63,8 @@ public enum Documentation: Codable, Equatable, Sendable {
         switch self {
         case .enum(let documentation):
             documentation.discussion
+        case .typealias(let documentation):
+            documentation.discussion
         case .object(let documentation):
             documentation.discussion
         case .operation(let documentation):
@@ -63,11 +74,9 @@ public enum Documentation: Codable, Equatable, Sendable {
 
     var subDocumentationIds: [String] {
         switch self {
-        case .enum:
-            []
         case .object(let documentation):
             documentation.subDocumentationIds
-        case .operation:
+        case .enum, .typealias, .operation:
             []
         }
     }
@@ -132,7 +141,18 @@ public enum Documentation: Codable, Equatable, Sendable {
                                abstract: abstract,
                                discussion: discussion,
                                cases: values))
-        } else if metadata.symbolKind == "dict" /* Object */ {
+        } else if metadata.symbolKind == "typealias" /* Enum like */ {
+            let values: [String: String] = contentSections.compactMap { contentSection -> [String: String]? in
+                guard case .possibleValues(let values) = contentSection else { return nil }
+                return values.compactMapValues { formatContent($0) }
+            }.first ?? [:]
+            self = .typealias(.init(id: id,
+                                    hierarchy: hierarchy,
+                                    title: metadata.title,
+                                    abstract: abstract,
+                                    discussion: discussion,
+                                    values: values))
+        } else if metadata.symbolKind == "dictionary" /* Object */ {
             let properties: [Property] = contentSections.compactMap { (contentSection: ContentSection) -> [Property]? in
                 guard case .properties(let properties) = contentSection else { return nil }
                 return properties
@@ -150,7 +170,7 @@ public enum Documentation: Codable, Equatable, Sendable {
                                  discussion: discussion,
                                  properties: propertyDocumentations,
                                  subDocumentationIds: subDocumentationIds))
-        } else if metadata.symbolKind == "operation" || metadata.symbolKind == "httpGet" || metadata.symbolKind == "httpPatch" || metadata.symbolKind == "httpPost" || metadata.symbolKind == "httpDelete" /* Operation */ {
+        } else if metadata.symbolKind == "operation" || metadata.symbolKind == "httpRequest" /* Operation */ {
             let pathParameters: [String: String] = (contentSections.compactMap { contentSection -> [Parameter]? in
                 guard case .pathParameters(let parameters) = contentSection else { return nil }
                 return parameters
@@ -203,8 +223,11 @@ public enum Documentation: Codable, Equatable, Sendable {
         case .enum(let documentation):
             try container.encode(Metadata(title: documentation.title, symbolKind: "tdef"), forKey: .metadata)
             contentSections.append(ContentSection.possibleValues(documentation.cases.mapValues { Content(text: $0) }))
+        case .typealias(let documentation):
+            try container.encode(Metadata(title: documentation.title, symbolKind: "typealias"), forKey: .metadata)
+            contentSections.append(ContentSection.possibleValues(documentation.values.mapValues { Content(text: $0) }))
         case .object(let documentation):
-            try container.encode(Metadata(title: documentation.title, symbolKind: "dict"), forKey: .metadata)
+            try container.encode(Metadata(title: documentation.title, symbolKind: "dictionary"), forKey: .metadata)
             let properties = documentation.properties.reduce(into: [Property]()) { partialResult, keyValue in
                 let propertyName = keyValue.key
                 let propertyDocumentation = keyValue.value
@@ -275,10 +298,10 @@ public enum Documentation: Codable, Equatable, Sendable {
 
         init(from decoder: Decoder) throws {
             let container = try decoder.container(keyedBy: CodingKeys.self)
-            title = try container.decode(String.self, forKey: .title)
+            title = try container.decodeIfPresent(String.self, forKey: .title) ?? ""
             let roleString = try container.decodeIfPresent(String.self, forKey: .role)
             role = Role(rawValue: roleString ?? "")
-            let url = try container.decode(String.self, forKey: .url)
+            let url = try container.decodeIfPresent(String.self, forKey: .url) ?? ""
             if role == .symbol || role == .article || role == nil {
                 self.url = "https://developer.apple.com" + url
             } else {
@@ -319,7 +342,7 @@ public enum Documentation: Codable, Equatable, Sendable {
                 let values = try container
                     .decode([Value].self, forKey: .values)
                     .reduce(into: [String: Content]()) { partialResult, value in
-                        partialResult[value.name] = value.content.first
+                        partialResult[value.name] = value.content.first ?? .init(text: "")
                     }
                 self = .possibleValues(values)
             } else if kind == "properties" {
@@ -349,10 +372,10 @@ public enum Documentation: Codable, Equatable, Sendable {
             } else if kind == "restResponses" {
                 let responses = try container.decode([Response].self, forKey: .items)
                 self = .restResponses(responses)
-            } else if kind == "declarations" || kind == "restEndpoint" {
+            } else if kind == "declarations" || kind == "restEndpoint" || kind == "mentions" {
                 self = .unused
             } else {
-                throw DecodingError.dataCorruptedError(forKey: .kind, in: container, debugDescription: "Unkown kind '\(kind)'")
+                throw DecodingError.dataCorruptedError(forKey: .kind, in: container, debugDescription: "Unknown kind '\(kind)'")
             }
         }
 

diff --git a/Sources/BagbutikDocsCollector/Models/EnumDocumentation.swift b/Sources/BagbutikDocsCollector/Models/EnumDocumentation.swift
@@ -6,7 +6,7 @@ public struct EnumDocumentation: Equatable, Sendable {
     public let discussion: String?
     public var cases: [String: String]
 
-    public init(id: String, hierarchy: Documentation.Hierarchy = .init(paths: []), title: String, abstract: String? = nil, discussion: String? = nil, cases: [String: String]) {
+    public init(id: String, hierarchy: Documentation.Hierarchy, title: String, abstract: String? = nil, discussion: String? = nil, cases: [String: String]) {
         self.id = id
         self.hierarchy = hierarchy
         self.title = title