Crypto3 and transpiler updated, JSON proof generation is compatible w…

…ith proof-producer #581
NilFoundation · Mar 29, 2024 · 43430b4 · 43430b4
1 parent 0af37d7
commit 43430b4
Show file tree

Hide file tree

Showing 10 changed files with 478 additions and 3,541 deletions.
diff --git a/.github/workflows/build_linux.yml b/.github/workflows/build_linux.yml
@@ -249,15 +249,9 @@ jobs:
         run: |
           cmake --build build -t compile_and_run_transpiler_tests
 
-      - name: Run recursive verifier single prover version
+      - name: Run recursive verifier tests
         run: |
-          chmod +x ./gen_and_check.sh
-          bash ./gen_and_check.sh
-
-      - name: Run recursive verifier multi prover version
-        run: |
-          chmod +x ./multi_gen_and_check.sh
-          bash ./multi_gen_and_check.sh
+          cmake --build build -t recursion
 
       - name: Build rslang
         run: |
@@ -323,11 +317,11 @@ jobs:
       always() && !cancelled() &&
       (needs.handle-syncwith.result == 'success' || needs.handle-syncwith.result == 'skipped') &&
       needs.build-and-test-linux.result == 'success'
-    uses: NilFoundation/proof-producer/.github/workflows/reusable-generate-proofs-linux.yml@4ddd85878d66f9b916e9fbdafe51180058e5bdaa
+    uses: NilFoundation/proof-producer/.github/workflows/reusable-generate-proofs-linux.yml@b1e380040b4714b6ad5fe9223555d854c49cf065
     with:
       artifact-name: ${{ needs.build-and-test-linux.outputs.examples-artifact-name }}
       # Update next line if you need new version of proof producer
-      proof-producer-ref: 4ddd85878d66f9b916e9fbdafe51180058e5bdaa
+      proof-producer-ref: 938426d309e586d87cd2b7895a0dea05ff88c0c5
       refs: ${{ needs.handle-syncwith.outputs.prs-refs }}
       targets: ${{ needs.build-and-test-linux.outputs.prover-targets }}
 

diff --git a/bin/recursive_gen/src/main.cpp b/bin/recursive_gen/src/main.cpp
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
@@ -94,7 +94,6 @@ function(assign_ir)
 endfunction()
 
 function(gen_proof target curve_type provers_amount)
-
     if(provers_amount EQUAL 0)
         gen_single_proof(${target} ${curve_type} 0)
     else()
@@ -116,8 +115,8 @@ function(gen_proof target curve_type provers_amount)
     endif()
 endfunction()
 
-function(gen_single_proof target curve_type provers_amount)
 
+function(gen_single_proof target curve_type provers_amount)
     if(NOT provers_amount EQUAL 0)
         set(multi_prover_flag --multi-prover)
         math(EXPR prover_num "${provers_amount} - 1")
@@ -137,6 +136,262 @@ function(gen_single_proof target curve_type provers_amount)
         VERBATIM)
 endfunction()
 
+function(gen_recursive_proof target curve_type provers_amount)
+    if(NOT provers_amount EQUAL 0)
+        math(EXPR prover_num "${provers_amount} - 1")
+    else()
+        set(prover_num "")
+    endif()
+
+    add_custom_target(${target}_gen_json_proof${prover_num}
+        COMMAND ${CMAKE_COMMAND} -E make_directory recursive_${target}/${prover_num}
+        COMMAND $<TARGET_FILE:recursive_gen> -m gen-input
+            -c circuit_${target}.crct${prover_num}
+            -t assignment_${target}.tbl${prover_num}
+            -o recursive_${target}/${prover_num}
+            -e ${curve_type}
+            ${multi_prover_flag}
+            DEPENDS ${target}_generate_crct ${target}_generate_tbl $<TARGET_FILE:transpiler>
+        COMMAND_EXPAND_LISTS
+        VERBATIM
+    )
+    add_dependencies(${target}_gen_json_proofs ${target}_gen_json_proof${prover_num})
+endfunction()
+
+function(gen_recursive_proofs target curve_type provers_amount)
+    if(provers_amount EQUAL 0)
+        add_custom_target(${target}_gen_json_proofs)
+        gen_recursive_proof(${target} ${curve_type} 0)
+        add_dependencies(${target}_recursion ${target}_gen_json_proofs)
+    else()
+        add_custom_target(${target}_gen_json_proofs
+            COMMAND ${CMAKE_COMMAND} -E make_directory recursive_${target}
+        )
+        foreach(prover_num RANGE 1 ${provers_amount})
+            gen_recursive_proof(${target} ${curve_type} ${prover_num})
+        endforeach()
+        add_dependencies(${target}_recursion ${target}_gen_json_proofs)
+    endif()
+endfunction()
+
+function(gen_recursive_verifier target curve_type provers_amount)
+    if(NOT provers_amount EQUAL 0)
+        math(EXPR prover_num "${provers_amount} - 1")
+    else()
+        set(prover_num "")
+    endif()
+
+    add_custom_target(${target}_gen_recursive_verifier${prover_num}
+        COMMAND ${CMAKE_COMMAND} -E make_directory recursive_${target}/${prover_num}
+        COMMAND $<TARGET_FILE:recursive_gen> -m gen-verifier
+            -c circuit_${target}.crct${prover_num}
+            -t assignment_${target}.tbl${prover_num}
+            -o recursive_${target}/${prover_num}
+            -e ${curve_type}
+            ${multi_prover_flag}
+            DEPENDS ${target}_generate_crct $<TARGET_FILE:transpiler>
+        COMMAND_EXPAND_LISTS
+        VERBATIM
+    )
+    add_dependencies(${target}_gen_recursive_verifiers ${target}_gen_recursive_verifier${prover_num})
+endfunction()
+
+function(gen_recursive_verifiers target curve_type provers_amount)
+    if(provers_amount EQUAL 0)
+        add_custom_target(${target}_gen_recursive_verifiers)
+        gen_recursive_verifier(${target} ${curve_type} 0)
+        add_dependencies(${target}_recursion ${target}_gen_recursive_verifiers)
+    else()
+        add_custom_target(${target}_gen_recursive_verifiers
+            COMMAND ${CMAKE_COMMAND} -E make_directory recursive_${target}
+        )
+        foreach(prover_num RANGE 1 ${provers_amount})
+            gen_recursive_verifier(${target} ${curve_type} ${prover_num})
+        endforeach()
+        add_dependencies(${target}_recursion ${target}_gen_recursive_verifiers)
+    endif()
+endfunction()
+
+function( add_circuit_with_recursion target )
+    set(prefix ARG)
+    set(name ${target}_build_placeholder_verifier${prover_num})
+    set(noValues "")
+    set(singleValues)
+    set(multiValues SOURCES INCLUDE_DIRECTORIES LINK_LIBRARIES COMPILER_OPTIONS)
+    cmake_parse_arguments(${prefix}
+                          "${noValues}"
+                          "${singleValues}"
+                          "${multiValues}"
+                          ${ARGN})
+
+    if(NOT ARG_SOURCES)
+        message(FATAL_ERROR "SOURCES for ${name} circuit was not defined")
+    endif()
+
+    foreach(source ${ARG_SOURCES})
+        if(NOT IS_ABSOLUTE ${source})
+            list(APPEND CIRCUIT_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/${source}")
+        else()
+            list(APPEND CIRCUIT_SOURCES "${source}")
+        endif()
+    endforeach()
+    list(REMOVE_DUPLICATES CIRCUIT_SOURCES)
+
+    foreach(ITR ${CIRCUIT_SOURCES})
+        if(NOT EXISTS ${ITR})
+            message(SEND_ERROR "Cannot find circuit source file: ${source}")
+        endif()
+    endforeach()
+
+    set(INCLUDE_DIRS_LIST "")
+    # Collect include directories from dependencies first
+    foreach(lib ${ARG_LINK_LIBRARIES})
+        get_target_property(lib_include_dirs ${lib} INTERFACE_INCLUDE_DIRECTORIES)
+        foreach(dir ${lib_include_dirs})
+            list(APPEND INCLUDE_DIRS_LIST "-I${dir}")
+        endforeach()
+    endforeach()
+    # Add passed include directories
+    foreach(include_dir ${ARG_INCLUDE_DIRECTORIES})
+        if(NOT IS_ABSOLUTE ${include_dir})
+            set(include_dir "${CMAKE_CURRENT_SOURCE_DIR}/${include_dir}")
+        endif()
+        list(APPEND INCLUDE_DIRS_LIST "-I${include_dir}")
+    endforeach()
+    if (ZKLLVM_DEV_ENVIRONMENT)
+        list(APPEND INCLUDE_DIRS_LIST -I${CMAKE_SOURCE_DIR}/libs/stdlib/libcpp -I${CMAKE_SOURCE_DIR}/libs/circifier/clang/lib/Headers -I${CMAKE_SOURCE_DIR}/libs/stdlib/libc/include)
+    endif()
+    list(REMOVE_DUPLICATES INCLUDE_DIRS_LIST)
+
+    if (NOT ${CIRCUIT_BINARY_OUTPUT})
+        set(link_options "-S")
+    endif()
+
+    if (ZKLLVM_DEV_ENVIRONMENT)
+        set(CLANG $<TARGET_FILE:clang>)
+        set(LINKER $<TARGET_FILE:llvm-link>)
+    else()
+        set(CLANG clang-zkllvm)
+        set(LINKER llvm-link-zkllvm)
+    endif()
+
+    # Compile sources
+    set(compiler_outputs "")
+    add_custom_target(${name}_compile_sources)
+    foreach(source ${CIRCUIT_SOURCES})
+        get_filename_component(source_base_name ${source} NAME)
+        add_custom_target(${name}_${source_base_name}_ll
+            COMMAND $<TARGET_FILE:recursive_gen> -m gen-input
+            -c circuit_${target}.crct${prover_num}
+            -t assignment_${target}.tbl${prover_num}
+            -o recursive_${target}/${prover_num}
+            -e ${curve_type}
+            ${multi_prover_flag}
+            DEPENDS ${target}_generate_crct ${target}_generate_tbl $<TARGET_FILE:recursive_gen>
+
+            COMMAND $<TARGET_FILE:recursive_gen> -m gen-verifier
+            -c circuit_${target}.crct${prover_num}
+            -t assignment_${target}.tbl${prover_num}
+            -o recursive_${target}/${prover_num}
+            -e ${curve_type}
+            ${multi_prover_flag}
+            DEPENDS ${target}_generate_crct ${target}_generate_tbl $<TARGET_FILE:recursive_gen>
+
+            COMMAND ${CLANG} -target assigner -Xclang -fpreserve-vec3-type -Werror=unknown-attributes -D_LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION
+            -D__ZKLLVM__ ${INCLUDE_DIRS_LIST} -emit-llvm -O1 -S ${ARG_COMPILER_OPTIONS}  -o ${name}_${source_base_name}.ll ${source}
+
+            VERBATIM COMMAND_EXPAND_LISTS
+            SOURCES ${source}
+        )
+        add_dependencies(${name}_compile_sources ${name}_${source_base_name}_ll)
+        list(APPEND compiler_outputs "${name}_${source_base_name}.ll")
+    endforeach()
+
+    if(NOT ${prover_num} EQUAL "")
+        set(sources_path ${CMAKE_CURRENT_BINARY_DIR}/recursive_${target}/${prover_num}/)
+    else()
+        set(sources_path ${CMAKE_CURRENT_BINARY_DIR}/recursive_${target}/)
+    endif()
+
+    # Link sources
+    add_custom_target(${name}
+                    COMMAND ${LINKER} ${link_options} -o ${name}.ll ${compiler_outputs}
+                    COMMAND $<TARGET_FILE:assigner>
+                      -b ${CMAKE_CURRENT_BINARY_DIR}/${target}_build_placeholder_verifier${prover_num}.ll
+                      -i ${sources_path}placeholder_verifier.inp
+                      -c ${sources_path}/circuit_${target}.crct
+                      -t ${sources_path}/assignment_${target}.tbl -e ${curve_type} --check
+                      --generate-type circuit-assignment
+                    DEPENDS ${name}_compile_sources
+                      VERBATIM COMMAND_EXPAND_LISTS)
+    set_target_properties(${name} PROPERTIES OUTPUT_NAME ${name}.ll)
+endfunction()
+
+function(assign_recursive_verifier target curve_type provers_amount)
+    if(NOT provers_amount EQUAL 0)
+        math(EXPR prover_num "${provers_amount} - 1")
+        set(sources_path ${CMAKE_CURRENT_BINARY_DIR}/recursive_${target}/${prover_num}/)
+    else()
+        set(prover_num "")
+        set(sources_path ${CMAKE_CURRENT_BINARY_DIR}/recursive_${target}/)
+    endif()
+
+    file(MAKE_DIRECTORY ${sources_path})
+    if( NOT EXISTS ${sources_path}/placeholder_verifier.cpp )
+        FILE(WRITE ${sources_path}/placeholder_verifier.cpp "Hello, world")
+    endif()
+
+    add_circuit_with_recursion(${target}
+        PROVER_NUM ${prover_num}
+        SOURCES ${sources_path}placeholder_verifier.cpp
+        COMPILER_OPTIONS ${ARG_COMPILER_OPTIONS}
+
+        LINK_LIBRARIES
+        crypto3::algebra
+        crypto3::block
+        crypto3::blueprint
+        crypto3::codec
+        crypto3::containers
+        crypto3::hash
+        crypto3::kdf
+        crypto3::mac
+        marshalling::core
+        marshalling::crypto3_algebra
+        marshalling::crypto3_multiprecision
+        marshalling::crypto3_zk
+        crypto3::math
+        crypto3::modes
+        crypto3::multiprecision
+        crypto3::passhash
+        crypto3::pbkdf
+        crypto3::threshold
+        crypto3::pkpad
+        crypto3::pubkey
+        crypto3::random
+        crypto3::stream
+        crypto3::vdf
+        crypto3::zk
+
+        ${Boost_LIBRARIES}
+    )
+    add_dependencies(${target}_assign_placeholder_verifiers ${target}_build_placeholder_verifier${prover_num})
+endfunction()
+
+function(assign_recursive_verifiers target curve_type provers_amount)
+    add_custom_target( ${target}_assign_placeholder_verifiers
+        COMMAND ${CMAKE_COMMAND} -E make_directory recursive_${target}
+    )
+    gen_recursive_proofs(${target} ${ARG_CURVE_TYPE} ${ARG_MAX_NUM_PROVERS})
+    gen_recursive_verifiers(${target} ${ARG_CURVE_TYPE} ${ARG_MAX_NUM_PROVERS})
+    if(provers_amount EQUAL 0)
+        assign_recursive_verifier(${target} ${curve_type} 0)
+    else()
+        foreach(prover_num RANGE 1 ${provers_amount})
+            assign_recursive_verifier(${target} ${curve_type} ${prover_num})
+        endforeach()
+    endif()
+endfunction()
+
 function(gen_evm_verifier target curve_type provers_amount)
 
     if(provers_amount EQUAL 0)
@@ -160,7 +415,6 @@ function(gen_evm_verifier target curve_type provers_amount)
 endfunction()
 
 function(gen_single_evm_verifier target curve_type provers_amount)
-
     if(NOT provers_amount EQUAL 0)
         set(multi_prover_flag --multi-prover)
         math(EXPR prover_num "${provers_amount} - 1")

diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt
@@ -16,7 +16,8 @@ function(add_example_without_proving example_target)
                         "${multiValues}"
                         ${ARGN})
 
-    add_circuit(${example_target}
+    add_circuit(
+        ${example_target}
                 SOURCES ${ARG_SOURCES}
                 COMPILER_OPTIONS ${ARG_COMPILER_OPTIONS}
 
@@ -99,15 +100,38 @@ function(add_example_with_proving example_target)
 
 endfunction()
 
-add_example_without_proving(placeholder_verifier_cpp_example SOURCES placeholder_verifier.cpp INPUT placeholder_verifier.inp CURVE_TYPE pallas)
+function(add_example_with_proving_and_recursion example_target)
+    set(prefix ARG)
+    set(noValues "")
+    set(singleValues INPUT PRIVATE_INPUT CURVE_TYPE MAX_NUM_PROVERS)
+    set(multiValues SOURCES)
+    cmake_parse_arguments(${prefix}
+                        "${noValues}"
+                        "${singleValues}"
+                        "${multiValues}"
+                        ${ARGN})
+
+    add_example_with_proving(${example_target}  ${ARG_CURVE_TYPE} ${ARGN})
+
+    if(NOT DEFINED ARG_MAX_NUM_PROVERS)
+        set(ARG_MAX_NUM_PROVERS 0)
+    endif()
+
+    add_custom_target(${example_target}_recursion)
+    assign_recursive_verifiers(${example_target} ${ARG_CURVE_TYPE} ${ARG_MAX_NUM_PROVERS})
+    add_dependencies(${example_target}_recursion ${example_target}_assign_placeholder_verifiers)
+    add_dependencies(recursion ${example_target}_recursion)
+endfunction()
+
+add_custom_target("recursion")
+
 add_example_with_proving(arithmetics_cpp_example SOURCES arithmetics.cpp INPUT arithmetics.inp CURVE_TYPE pallas)
 add_example_with_proving(integer_arithmetics_cpp_example SOURCES integer_arithmetics.cpp INPUT integer_arithmetics.inp CURVE_TYPE pallas)
-
 add_example_with_proving(poseidon_cpp_example SOURCES poseidon.cpp INPUT poseidon.inp CURVE_TYPE pallas)
-add_example_with_proving(merkle_tree_poseidon_cpp_example SOURCES merkle_tree_poseidon.cpp INPUT merkle_tree_poseidon.inp CURVE_TYPE pallas)
+add_example_with_proving_and_recursion(merkle_tree_poseidon_cpp_example SOURCES merkle_tree_poseidon.cpp INPUT merkle_tree_poseidon.inp CURVE_TYPE pallas)
 add_example_with_proving(merkle_poseidon_1prover_cpp_example SOURCES merkle_poseidon_1prover.cpp INPUT merkle_poseidon_1prover_public.inp PRIVATE_INPUT merkle_poseidon_1prover_private.inp CURVE_TYPE pallas)
 add_example_with_proving(merkle_poseidon_2provers_cpp_example SOURCES merkle_poseidon_2provers.cpp INPUT merkle_poseidon_2provers_public.inp PRIVATE_INPUT merkle_poseidon_2provers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 2)
-add_example_with_proving(merkle_poseidon_3provers_cpp_example SOURCES merkle_poseidon_3provers.cpp INPUT merkle_poseidon_3provers_public.inp PRIVATE_INPUT merkle_poseidon_3provers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 3)
+add_example_with_proving_and_recursion(merkle_poseidon_3provers_cpp_example SOURCES merkle_poseidon_3provers.cpp INPUT merkle_poseidon_3provers_public.inp PRIVATE_INPUT merkle_poseidon_3provers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 3)
 add_example_with_proving(merkle_poseidon_5provers_cpp_example SOURCES merkle_poseidon_5provers.cpp INPUT merkle_poseidon_5provers_public.inp PRIVATE_INPUT merkle_poseidon_5provers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 5)
 add_example_with_proving(merkle_poseidon_21prover_3layers_cpp_example SOURCES merkle_poseidon_21prover_3layers.cpp INPUT merkle_poseidon_21prover_3layers_public.inp PRIVATE_INPUT merkle_poseidon_21prover_3layers_private.inp CURVE_TYPE pallas MAX_NUM_PROVERS 21)