added support for format v2; new file Merkle.swift and some additiona…

Package.resolved

@@ -9,6 +9,24 @@
           "revision": "864d6a014a4aa343dd47f3804f12b8a42a7b21eb",
           "version": "2.2.4"
         }
+      },
+      {
+        "package": "swift-algorithms",
+        "repositoryURL": "https://github.com/apple/swift-algorithms",
+        "state": {
+          "branch": null,
+          "revision": "b14b7f4c528c942f121c8b860b9410b2bf57825e",
+          "version": "1.0.0"
+        }
+      },
+      {
+        "package": "swift-numerics",
+        "repositoryURL": "https://github.com/apple/swift-numerics",
+        "state": {
+          "branch": null,
+          "revision": "0a23770641f65a4de61daf5425a37ae32a3fd00d",
+          "version": "1.0.1"
+        }
       }
     ]
   },

Package.swift

@@ -16,11 +16,12 @@ let package = Package(
     ],
     dependencies: [
         .package(url: "https://github.com/lukereichold/JOSESwift.git", .upToNextMajor(from: "2.2.4")),
+        .package(url: "https://github.com/apple/swift-algorithms", from: "1.0.0"),
     ],
     targets: [
         .target(
             name: "Arweave",
-            dependencies: ["JOSESwift"],
+            dependencies: ["JOSESwift", .product(name: "Algorithms", package: "swift-algorithms")],
             path: "Sources"),
         .testTarget(
             name: "ArweaveTests",

Sources/API/HttpClient.swift

@@ -2,7 +2,7 @@ import Foundation
 
 extension String: Error { }
 
-struct HttpResponse {
+public struct HttpResponse {
     let data: Data
     let statusCode: Int
 }
@@ -14,7 +14,9 @@ struct HttpClient {
         var request = URLRequest(url: target.url)
         request.httpMethod = target.method
         request.httpBody = target.body
-        request.allHTTPHeaderFields = target.headers
+        if request.httpMethod?.uppercased() == "POST" {
+            request.allHTTPHeaderFields = target.headers
+        }
 
         let (data, response) = try await URLSession.shared.data(for: request)
 

Sources/KeyUtilities.swift

@@ -6,14 +6,69 @@ extension Digest {
     var data: Data { Data(bytes) }
 }
 
-extension String {
+public extension String {
     var base64URLEncoded: String {
         Data(utf8).base64URLEncodedString()
     }
+
+    var base64URLDecoded: String {
+        guard let data = Data(base64URLEncoded: self) else { return "" }
+        return String(data: data, encoding: .utf8) ?? ""
+    }
 }
 
 extension Array where Element == Data {
     var combined: Data {
        reduce(.init(), +)
     }
 }
+
+func concatData(data: [Data]) -> Data {
+    var length = 0
+    for d in data {
+        length += d.count
+    }
+
+    var temp = Data(capacity: length)
+    for d in data {
+        temp.append(d)
+    }
+
+    return temp
+}
+
+func deepHash(data: [Data]) -> Data {
+    if data.count > 1 {
+        let tag = concatData(data: [
+            "list".data(using: .utf8)!,
+            String(data.count).data(using: .utf8)!
+        ])
+
+        return deepHashChunks(chunks: data, acc: SHA384.hash(data: tag).data)
+    }
+
+    let tag = concatData(data: [
+        "blob".data(using: .utf8)!,
+        data.first!.count.description.data(using: .utf8)!
+    ])
+
+    let taggedHash = concatData(data: [
+        SHA384.hash(data: tag).data,
+        SHA384.hash(data: data.first!).data
+    ])
+
+    return SHA384.hash(data: taggedHash).data
+}
+
+func deepHashChunks(chunks: [Data], acc: Data) -> Data {
+    if chunks.count < 1 {
+        return acc
+    }
+
+    let hashPair = concatData(data: [
+        acc,
+        deepHash(data: [chunks.first!])
+    ])
+    let newAcc = SHA384.hash(data: hashPair).data
+    return deepHashChunks(chunks: Array(chunks.prefix(through: 1)), acc: newAcc)
+}

Sources/Merkle.swift

@@ -0,0 +1,232 @@
+//
+//  File.swift
+//  
+//
+//  Created by David Choi on 11/6/21.
+//
+
+import Foundation
+import CryptoKit
+
+struct Chunk {
+    let dataHash: Data
+    let minByteRange: Int
+    let maxByteRange: Int
+}
+
+class BranchNode: MerkelNode {
+    var id: Data
+    var type: BranchOrLeaf = BranchOrLeaf.branch
+    let byteRange: Int
+    var maxByteRange: Int
+    let leftChild: MerkelNode?
+    let rightChild: MerkelNode?
+
+    init(id: Data, byteRange: Int, maxByteRange: Int, leftChild: MerkelNode? = nil, rightChild: MerkelNode? = nil) {
+        self.id = id
+        self.byteRange = byteRange
+        self.maxByteRange = maxByteRange
+        self.leftChild = leftChild
+        self.rightChild = rightChild
+    }
+}
+
+class LeafNode: MerkelNode {
+    var id: Data
+    let dataHash: Data
+    var type: BranchOrLeaf = BranchOrLeaf.leaf
+    let minByteRange: Int
+    var maxByteRange: Int
+
+    init(id: Data, dataHash: Data, minByteRange: Int, maxByteRange: Int) {
+        self.id = id
+        self.dataHash = dataHash
+        self.minByteRange = minByteRange
+        self.maxByteRange = maxByteRange
+    }
+}
+
+protocol MerkelNode {
+    var id: Data { get set }
+    var maxByteRange: Int { get set }
+    var type: BranchOrLeaf { get set }
+}
+
+enum BranchOrLeaf {
+    case branch
+    case leaf
+}
+enum BranchOrLeafError: Error {
+    case UnknownNodeType
+}
+
+struct Proof {
+    let offset: Int
+    let proof: Data
+}
+
+let MAX_CHUNK_SIZE = 256 * 1024
+let MIN_CHUNK_SIZE = 32 * 1024
+let NOTE_SIZE = 32
+let HASH_SIZE = 32
+
+func chunkData(data: Data) -> [Chunk] {
+    var chunks = [Chunk]()
+
+    var rest = data
+    var cursor = 0
+
+    while(rest.count >= MAX_CHUNK_SIZE) {
+        var chunkSize = MAX_CHUNK_SIZE
+
+        let nextChunkSize = rest.count - MAX_CHUNK_SIZE
+        if (nextChunkSize > 0 && nextChunkSize < MIN_CHUNK_SIZE) {
+            chunkSize = Int(Double(rest.count / 2).rounded())
+        }
+
+        let chunk = rest.subdata(in: 0..<chunkSize)
+        let dataHash = SHA256.hash(data: chunk)
+        cursor += chunk.count
+        chunks.append(Chunk(dataHash: dataHash.data, minByteRange: cursor - chunk.count, maxByteRange: cursor))
+        rest = rest.subdata(in: (chunkSize <= 0 ? 0 : chunkSize - 1)..<rest.count)
+    }
+
+    chunks.append(Chunk(dataHash: SHA256.hash(data: rest).data, minByteRange: cursor, maxByteRange: cursor + rest.count))
+    return chunks
+}
+
+func generateLeaves(chunks: [Chunk]) -> [LeafNode] {
+    return chunks.map { chunk in
+        var idData = [Data]()
+        idData.append(chunk.dataHash)
+        idData.append(intToBuffer(note: chunk.maxByteRange))
+
+        return LeafNode(
+            id: hashId(data: idData),
+            dataHash: chunk.dataHash,
+            minByteRange: chunk.minByteRange,
+            maxByteRange: chunk.maxByteRange
+        )
+    }
+}
+
+func hashId(data: [Data]) -> Data {
+    let data = concatBuffers(buffers: data)
+    return Data(SHA256.hash(data: data))
+}
+
+func intToBuffer(note: Int) -> Data {
+    var note = note
+    var buffer = Data(capacity: NOTE_SIZE)
+
+    for i in stride(from: buffer.count - 1, through: 0, by: -1) {
+        let byte = note % 256
+        buffer[i] = UInt8(byte)
+        note = (note - byte) / 256
+    }
+
+    return buffer
+}
+
+// of leafs or branches
+func buildLayers(nodes: [MerkelNode], level: Int = 0) -> MerkelNode {
+    if nodes.count < 2 {
+        return hashBranch(left: nodes[0])
+    }
+
+    var nextLayer = [MerkelNode]()
+    for i in stride(from: 0, to: nodes.count, by: 2) {
+        nextLayer.append(hashBranch(left: nodes[i], right: nodes[i + 1]))
+    }
+
+    return buildLayers(nodes: nextLayer, level: level + 1)
+}
+
+func generateTransactionChunks(data: Data) -> Chunks {
+    var chunks = chunkData(data: data)
+    let leaves = generateLeaves(chunks: chunks)
+    let root = buildLayers(nodes: leaves)
+    var proofs = generateProofs(root: root)
+
+    if chunks.count > 0 {
+        let lastChunk = chunks.last
+        if ((lastChunk!.maxByteRange - lastChunk!.minByteRange) == 0) {
+            chunks.remove(at: chunks.count - 1)
+            proofs.remove(at: proofs.count - 1)
+        }
+    }
+
+    return Chunks(data_root: root.id, chunks: chunks, proofs: proofs)
+}
+
+func generateProofs(root: MerkelNode) -> [Proof] {
+    var proofs: [Proof] = [Proof]()
+    do {
+        proofs = try resolveBranchProofs(node: root)
+    } catch {
+        print("failed to resolve branch proofs \(error)")
+    }
+    return proofs
+}
+
+func resolveBranchProofs(node: MerkelNode, proof: Data = Data(), depth: Int = 0) throws -> [Proof] {
+    if node.type == BranchOrLeaf.leaf {
+        let dataHash = (node as! LeafNode).dataHash
+        return [
+            Proof(offset: node.maxByteRange - 1, proof: concatBuffers(buffers: [proof, dataHash, intToBuffer(note: node.maxByteRange)]))
+        ]
+    }
+
+    if node.type == BranchOrLeaf.branch {
+        let branch = (node as! BranchNode)
+
+        var buffers = [
+            proof,
+            intToBuffer(note: branch.byteRange)
+        ]
+        if let leftChild = branch.leftChild {
+            buffers.append(leftChild.id)
+        }
+        if let rightChild = branch.rightChild {
+            buffers.append(rightChild.id)
+        }
+        let partialProof = concatBuffers(buffers: buffers)
+
+        var resolvedProofs = [[Proof]]()
+        if let leftChild = branch.leftChild {
+            resolvedProofs.append(try resolveBranchProofs(node: leftChild, proof: partialProof, depth: depth + 1))
+        }
+        if let rightChild = branch.rightChild {
+            resolvedProofs.append(try resolveBranchProofs(node: rightChild, proof: partialProof, depth: depth + 1))
+        }
+        return Array(resolvedProofs.joined())
+    }
+
+    throw BranchOrLeafError.UnknownNodeType
+}
+
+func hashBranch(left: MerkelNode, right: MerkelNode? = nil) -> MerkelNode {
+    if right == nil {
+        return BranchNode(
+                id: hashId(data: [
+                hashId(data: [left.id]),
+                hashId(data: [intToBuffer(note: left.maxByteRange)]),
+            ]),
+            byteRange: left.maxByteRange,
+            maxByteRange: left.maxByteRange,
+            leftChild: left)
+    }
+
+    let branch = BranchNode(
+        id: hashId(data: [
+            hashId(data: [left.id]),
+            hashId(data: [right!.id]),
+            hashId(data: [intToBuffer(note: left.maxByteRange)]),
+        ]),
+        byteRange: left.maxByteRange,
+        maxByteRange: right!.maxByteRange,
+        leftChild: left,
+        rightChild: right
+    )
+    return branch
+}