benchmark cpp implementation of f6 mul synth loop

benchmark_results_data/evm_benchmarks.csv

@@ -1,108 +1,109 @@
 engine, test_name, total_time, gas_used
 engine,test_name,total_time,gas_used
-evmone,sha1-10808-bits,0.002056286,790156
-evmone,sha1-21896-bits,0.00393736,1543605
-evmone,sha1-42488-bits,0.007656268,3014404
-evmone,sha1-10808-bits-shiftopt,0.001038472,759785
-evmone,sha1-21896-bits-shiftopt,0.002031567,1484254
-evmone,sha1-42488-bits-shiftopt,0.004000971,2898473
-evmone,bn256g2mul-0xAshish,0.001509638,455757
-evmone,blake2b-8415-bytes-shiftopt,0.018414062,12043105
-evmone,blake2b-5610-bytes-shiftopt,0.01218996,7434820
-evmone,blake2b-2805-bytes-shiftopt,0.006019918,3423526
-evmone,mul256-gcolvin-drag-race,0.009680082,5610277
-evmone,bn128_mul_weierstrudel-cdetrio2,0.000397231,47165
-evmone,bn128_mul_weierstrudel-chfast1,0.000210124,23449
-evmone,bn128_mul_weierstrudel-chfast2,0.000339741,39337
-evmone,bn128_mul_weierstrudel-chfast3,0.000399416,47291
-evmone,bn128_mul_weierstrudel-cdetrio6,0.00039595,47561
-evmone,bn128_mul_weierstrudel-cdetrio7,0.000398454,47165
-evmone,bn128_mul_weierstrudel-cdetrio11,0.000404129,47561
-evmone,bn128_mul_weierstrudel-cdetrio12,0.000396089,47165
-evmone,blake2b-8415-bytes,0.022527799,12351815
-evmone,blake2b-5610-bytes,0.015275491,7640669
-evmone,blake2b-2805-bytes,0.007459261,3526514
-evmone,blake2b_huff-8415-bytes,0.001514828,879920
-evmone,blake2b_huff-5610-bytes,0.001142694,586839
-evmone,blake2b_huff-2805-bytes,0.000524197,293755
+evmone,sha1-10808-bits-shiftopt,0.000825821,759785
+evmone,sha1-21896-bits-shiftopt,0.001562969,1484254
+evmone,sha1-42488-bits-shiftopt,0.003186443,2898473
+evmone,bn256g2mul-0xAshish,0.001271556,455757
+evmone,blake2b-8415-bytes-shiftopt,0.013668602,12043105
+evmone,blake2b-5610-bytes-shiftopt,0.009378376,7434820
+evmone,blake2b-2805-bytes-shiftopt,0.004491046,3423526
+evmone,mul256-gcolvin-drag-race,0.007599084,5610277
+evmone,bn128_mul_weierstrudel-cdetrio2,0.000360764,47165
+evmone,bn128_mul_weierstrudel-chfast1,0.000182614,23449
+evmone,bn128_mul_weierstrudel-chfast2,0.000298584,39337
+evmone,bn128_mul_weierstrudel-chfast3,0.000373645,47291
+evmone,bn128_mul_weierstrudel-cdetrio6,0.000360047,47561
+evmone,bn128_mul_weierstrudel-cdetrio7,0.000348965,47165
+evmone,bn128_mul_weierstrudel-cdetrio11,0.000364878,47561
+evmone,bn128_mul_weierstrudel-cdetrio12,0.000337997,47165
+evmone,blake2b_huff-8415-bytes,0.00112764,879920
+evmone,blake2b_huff-5610-bytes,0.000734551,586839
+evmone,blake2b_huff-2805-bytes,0.000397603,293755
+evmone,blake2b-8415-bytes,0.017831598,12351815
+evmone,blake2b-5610-bytes,0.012580117,7640669
+evmone,blake2b-2805-bytes,0.005750261,3526514
+evmone,sha1-10808-bits,0.001631885,790156
+evmone,sha1-21896-bits,0.003034628,1543605
+evmone,sha1-42488-bits,0.006046502,3014404
 engine,test_name,total_time,gas_used
-parity-evm,sha1-10808-bits,0.069457059,790156
-parity-evm,sha1-21896-bits,0.128567939,1543605
-parity-evm,sha1-42488-bits,0.250837896,3014404
-parity-evm,sha1-10808-bits-shiftopt,0.026990627,759785
-parity-evm,sha1-21896-bits-shiftopt,0.047456338,1484254
-parity-evm,sha1-42488-bits-shiftopt,0.092930734,2898473
-parity-evm,bn256g2mul-0xAshish,0.024067669,455757
-parity-evm,blake2b-8415-bytes-shiftopt,0.302345632,12043105
-parity-evm,blake2b-5610-bytes-shiftopt,0.216411367,7434820
-parity-evm,blake2b-2805-bytes-shiftopt,0.116367526,3423526
-parity-evm,mul256-gcolvin-drag-race,0.143134311,5610277
-parity-evm,bn128_mul_weierstrudel-cdetrio2,0.004681313,47165
-parity-evm,bn128_mul_weierstrudel-chfast1,0.002312094,23449
-parity-evm,bn128_mul_weierstrudel-chfast2,0.003910487,39337
-parity-evm,bn128_mul_weierstrudel-chfast3,0.004720527,47291
-parity-evm,bn128_mul_weierstrudel-cdetrio6,0.004867541,47561
-parity-evm,bn128_mul_weierstrudel-cdetrio7,0.004679791,47165
-parity-evm,bn128_mul_weierstrudel-cdetrio11,0.004646788,47561
-parity-evm,bn128_mul_weierstrudel-cdetrio12,0.004630612,47165
-parity-evm,blake2b-8415-bytes,0.340011586,12351815
-parity-evm,blake2b-5610-bytes,0.205717038,7640669
-parity-evm,blake2b-2805-bytes,0.096635901,3526514
-parity-evm,blake2b_huff-8415-bytes,0.026595507,879920
-parity-evm,blake2b_huff-5610-bytes,0.01661306,586839
-parity-evm,blake2b_huff-2805-bytes,0.009291144,293755
+parity-evm,sha1-10808-bits-shiftopt,0.022681687,759785
+parity-evm,sha1-21896-bits-shiftopt,0.037793747,1484254
+parity-evm,sha1-42488-bits-shiftopt,0.080097641,2898473
+parity-evm,bn256g2mul-0xAshish,0.019093158,455757
+parity-evm,blake2b-8415-bytes-shiftopt,0.24098813,12043105
+parity-evm,blake2b-5610-bytes-shiftopt,0.165490126,7434820
+parity-evm,blake2b-2805-bytes-shiftopt,0.079788988,3423526
+parity-evm,mul256-gcolvin-drag-race,0.102421079,5610277
+parity-evm,bn128_mul_weierstrudel-cdetrio2,0.003706916,47165
+parity-evm,bn128_mul_weierstrudel-chfast1,0.001689577,23449
+parity-evm,bn128_mul_weierstrudel-chfast2,0.003045638,39337
+parity-evm,bn128_mul_weierstrudel-chfast3,0.003566339,47291
+parity-evm,bn128_mul_weierstrudel-cdetrio6,0.003535184,47561
+parity-evm,bn128_mul_weierstrudel-cdetrio7,0.003536592,47165
+parity-evm,bn128_mul_weierstrudel-cdetrio11,0.003550023,47561
+parity-evm,bn128_mul_weierstrudel-cdetrio12,0.003981209,47165
+parity-evm,blake2b_huff-8415-bytes,0.023224086,879920
+parity-evm,blake2b_huff-5610-bytes,0.018505375,586839
+parity-evm,blake2b_huff-2805-bytes,0.008630519,293755
+parity-evm,blake2b-8415-bytes,0.273193152,12351815
+parity-evm,blake2b-5610-bytes,0.177323457,7640669
+parity-evm,blake2b-2805-bytes,0.087049753,3526514
+parity-evm,sha1-10808-bits,0.050909587,790156
+parity-evm,sha1-21896-bits,0.107236213,1543605
+parity-evm,sha1-42488-bits,0.206253037,3014404
 engine,test_name,total_time,gas_used
-geth-evm,sha1-10808-bits,0.015162857,790156
-geth-evm,sha1-21896-bits,0.029013649,1543605
-geth-evm,sha1-42488-bits,0.056405393,3014404
-geth-evm,sha1-10808-bits-shiftopt,0.012383896,759785
-geth-evm,sha1-21896-bits-shiftopt,0.023512095,1484254
-geth-evm,sha1-42488-bits-shiftopt,0.046008429,2898473
-geth-evm,bn256g2mul-0xAshish,0.00882746,455757
-geth-evm,blake2b-8415-bytes-shiftopt,0.143346789,12043105
-geth-evm,blake2b-5610-bytes-shiftopt,0.097426734,7434820
-geth-evm,blake2b-2805-bytes-shiftopt,0.048999189,3423526
-geth-evm,mul256-gcolvin-drag-race,0.14476789,5610277
-geth-evm,bn128_mul_weierstrudel-cdetrio2,0.001808681,47165
-geth-evm,bn128_mul_weierstrudel-chfast1,0.000838142,23449
-geth-evm,bn128_mul_weierstrudel-chfast2,0.001490949,39337
-geth-evm,bn128_mul_weierstrudel-chfast3,0.001791839,47291
-geth-evm,bn128_mul_weierstrudel-cdetrio6,0.001907578,47561
-geth-evm,bn128_mul_weierstrudel-cdetrio7,0.001796564,47165
-geth-evm,bn128_mul_weierstrudel-cdetrio11,0.001807007,47561
-geth-evm,bn128_mul_weierstrudel-cdetrio12,0.001773678,47165
-geth-evm,blake2b-8415-bytes,0.150170679,12351815
-geth-evm,blake2b-5610-bytes,0.107144698,7640669
-geth-evm,blake2b-2805-bytes,0.051748147,3526514
-geth-evm,blake2b_huff-8415-bytes,0.018262182,879920
-geth-evm,blake2b_huff-5610-bytes,0.012279144,586839
-geth-evm,blake2b_huff-2805-bytes,0.00614165,293755
+geth-evm,sha1-10808-bits-shiftopt,0.010794754,759785
+geth-evm,sha1-21896-bits-shiftopt,0.022239662,1484254
+geth-evm,sha1-42488-bits-shiftopt,0.040845536,2898473
+geth-evm,bn256g2mul-0xAshish,0.007856129,455757
+geth-evm,blake2b-8415-bytes-shiftopt,0.132213726,12043105
+geth-evm,blake2b-5610-bytes-shiftopt,0.088160969,7434820
+geth-evm,blake2b-2805-bytes-shiftopt,0.044373907,3423526
+geth-evm,mul256-gcolvin-drag-race,0.129451529,5610277
+geth-evm,bn128_mul_weierstrudel-cdetrio2,0.001591463,47165
+geth-evm,bn128_mul_weierstrudel-chfast1,0.000792984,23449
+geth-evm,bn128_mul_weierstrudel-chfast2,0.001313077,39337
+geth-evm,bn128_mul_weierstrudel-chfast3,0.001583505,47291
+geth-evm,bn128_mul_weierstrudel-cdetrio6,0.001660588,47561
+geth-evm,bn128_mul_weierstrudel-cdetrio7,0.001589395,47165
+geth-evm,bn128_mul_weierstrudel-cdetrio11,0.001573866,47561
+geth-evm,bn128_mul_weierstrudel-cdetrio12,0.001556755,47165
+geth-evm,blake2b_huff-8415-bytes,0.016672025,879920
+geth-evm,blake2b_huff-5610-bytes,0.01070878,586839
+geth-evm,blake2b_huff-2805-bytes,0.005402838,293755
+geth-evm,blake2b-8415-bytes,0.13058734,12351815
+geth-evm,blake2b-5610-bytes,0.09154925,7640669
+geth-evm,blake2b-2805-bytes,0.045537792,3526514
+geth-evm,sha1-10808-bits,0.012907775,790156
+geth-evm,sha1-21896-bits,0.026743191,1543605
+geth-evm,sha1-42488-bits,0.050452738,3014404
 engine,test_name,total_time,gas_used
-cita-evm,sha1-10808-bits,0.050446102,903488
-cita-evm,sha1-21896-bits,0.097787582,1750413
-cita-evm,sha1-42488-bits,0.190869116,3396188
-cita-evm,sha1-10808-bits-shiftopt,0.011862067,873117
-cita-evm,sha1-21896-bits-shiftopt,0.022392031,1691062
-cita-evm,sha1-42488-bits-shiftopt,0.045725851,3280257
-cita-evm,bn256g2mul-0xAshish,0.041878381,486053
-cita-evm,blake2b-8415-bytes-shiftopt,0.215833646,12635513
-cita-evm,blake2b-5610-bytes-shiftopt,0.147089137,7837276
-cita-evm,blake2b-2805-bytes-shiftopt,0.070500257,3635902
-cita-evm,mul256-gcolvin-drag-race,0.042455215,5635901
-cita-evm,bn128_mul_weierstrudel-cdetrio2,0.016598673,74437
-cita-evm,bn128_mul_weierstrudel-chfast1,0.007348978,49441
-cita-evm,bn128_mul_weierstrudel-chfast2,0.013854024,66865
-cita-evm,bn128_mul_weierstrudel-chfast3,0.01711688,74819
-cita-evm,bn128_mul_weierstrudel-cdetrio6,0.017692487,75089
-cita-evm,bn128_mul_weierstrudel-cdetrio7,0.017047063,74501
-cita-evm,bn128_mul_weierstrudel-cdetrio11,0.022045002,75089
-cita-evm,bn128_mul_weierstrudel-cdetrio12,0.016597804,74501
-cita-evm,blake2b-8415-bytes,0.271612442,12944223
-cita-evm,blake2b-5610-bytes,0.178762917,8043125
-cita-evm,blake2b-2805-bytes,0.09112152,3738890
-cita-evm,blake2b_huff-8415-bytes,0.01433503,1471220
-cita-evm,blake2b_huff-5610-bytes,0.009230364,988103
-cita-evm,blake2b_huff-2805-bytes,0.004808496,504983
+cita-evm,sha1-10808-bits-shiftopt,0.010432911,873117
+cita-evm,sha1-21896-bits-shiftopt,0.019753476,1691062
+cita-evm,sha1-42488-bits-shiftopt,0.039671776,3280257
+cita-evm,bn256g2mul-0xAshish,0.036949621,486053
+cita-evm,blake2b-8415-bytes-shiftopt,0.191493324,12635513
+cita-evm,blake2b-5610-bytes-shiftopt,0.127574863,7837276
+cita-evm,blake2b-2805-bytes-shiftopt,0.06295393,3635902
+cita-evm,mul256-gcolvin-drag-race,0.037527379,5635901
+cita-evm,bn128_mul_weierstrudel-cdetrio2,0.014468663,74437
+cita-evm,bn128_mul_weierstrudel-chfast1,0.00632152,49441
+cita-evm,bn128_mul_weierstrudel-chfast2,0.012445637,66865
+cita-evm,bn128_mul_weierstrudel-chfast3,0.015182058,74819
+cita-evm,bn128_mul_weierstrudel-cdetrio6,0.015238954,75089
+cita-evm,bn128_mul_weierstrudel-cdetrio7,0.015010512,74501
+cita-evm,bn128_mul_weierstrudel-cdetrio11,0.015671057,75089
+cita-evm,bn128_mul_weierstrudel-cdetrio12,0.014657662,74501
+cita-evm,blake2b_huff-8415-bytes,0.012236252,1471220
+cita-evm,blake2b_huff-5610-bytes,0.008748997,988103
+cita-evm,blake2b_huff-2805-bytes,0.0043396,504983
+cita-evm,blake2b-8415-bytes,0.244230501,12944223
+cita-evm,blake2b-5610-bytes,0.16335111,8043125
+cita-evm,blake2b-2805-bytes,0.084903451,3738890
+cita-evm,sha1-10808-bits,0.050418756,903488
+cita-evm,sha1-21896-bits,0.090731809,1750413
+cita-evm,sha1-42488-bits,0.172284761,3396188
 engine,test_name,total_time,gas_used
-evmone384,evm384-synth-loop-v1,0.023534375,12569235
-evmone384,evm384-synth-loop-v2,0.015339457,5851179
+evmone384,evm384-synth-loop-v1,0.021697,12569235
+evmone384,evm384-synth-loop-v2,0.012063,5534811
+cpp-native,f6m-mul-synth-loop,0.0048518499999999996,0

evm/evmone-evm384/Dockerfile

@@ -17,9 +17,16 @@ RUN git clone --recursive --single-branch --branch v0.5.0-evm384-v2 https://gith
 RUN cd evmone-evm384-v2 && mkdir build && \
   cd build && CXXFLAGS="-w" cmake .. -DEVMONE_TESTING=ON && make -j4
 
-
 # fetch evm384 f6m_mul synthetic loop repo
-RUN git clone --single-branch --branch v0.0.1 https://github.com/ewasm/evm384_f6m_mul.git
+RUN git clone --single-branch --branch f6m_mul_native https://github.com/ewasm/evm384_f6m_mul.git
+
+RUN cd evm384_f6m_mul && \
+	git submodule update --init && \
+	cd src/native && \
+	mkdir build && \
+	cd build && \
+	cmake -DCMAKE_BUILD_TYPE=RELEASE .. && \
+	make -j4
 
 WORKDIR /
 CMD /bin/bash

evm/scripts/bench_evm384.py

@@ -10,6 +10,7 @@
 import shutil
 import shlex
 import json
+from statistics import mean
 
 # output paths should be mounted docker volumes
 RESULT_CSV_OUTPUT_PATH = "/evmraceresults"
@@ -66,6 +67,23 @@ def do_evmone_bench(evmone_bench_cmd):
     time = durationpy.from_str(total_time)
     return {'gas_used': gasused, 'time': time.total_seconds()}
 
+def do_cpp_native_f6m_mul_bench():
+    print("running f6m_mul cpp native benchmark")
+    f6m_mul_cpp_native_cmd = ['/root/evm384_f6m_mul/src/native/build/synth_loop_benchmark']
+
+    stdoutlines = []
+    with subprocess.Popen(f6m_mul_cpp_native_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True) as p:
+        for line in p.stdout: # b'\n'-separated lines
+            print(line, end='')
+            stdoutlines.append(line)  # pass bytes as is
+        p.wait()
+        output = ''.join(stdoutlines)[:-1] + 's'
+
+        return {
+            'time': durationpy.from_str(output).total_seconds(),
+            'bench_name': 'f6m-mul-synth-loop',
+            'engine': 'cpp-native'
+        }
 
 def saveResults(precompile_benchmarks):
     # move existing csv file to backup-datetime-folder
@@ -99,6 +117,21 @@ def main():
       bench_result['engine'] = "evmone384"
       all_bench_resuls.append(bench_result)
 
+    # do 20 iterations and average results for cpp-native f6m-mul synth loop
+    cpp_native_results = []
+    for i in range(20):
+        result = do_cpp_native_f6m_mul_bench()
+        cpp_native_results.append(result)
+
+    cpp_native_result = {
+        'time': mean(map(lambda x: x['time'], cpp_native_results)),
+        'bench_name': 'f6m-mul-synth-loop',
+        'engine': 'cpp-native',
+        'gas_used': 0
+    }
+
+    all_bench_resuls.append(cpp_native_result)
+
     saveResults(all_bench_resuls)