From e00dc1f8a58986b90d3d7db89cfb096a79c9c2bd Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Tue, 11 Aug 2020 22:28:46 +0000 Subject: [PATCH 1/2] benchmark cpp implementation of f6 mul synth loop --- evm/evmone-evm384/Dockerfile | 11 +++++++++-- evm/scripts/bench_evm384.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/evm/evmone-evm384/Dockerfile b/evm/evmone-evm384/Dockerfile index c0e81e2..cc7172f 100644 --- a/evm/evmone-evm384/Dockerfile +++ b/evm/evmone-evm384/Dockerfile @@ -17,9 +17,16 @@ RUN git clone --recursive --single-branch --branch v0.5.0-evm384-v2 https://gith RUN cd evmone-evm384-v2 && mkdir build && \ cd build && CXXFLAGS="-w" cmake .. -DEVMONE_TESTING=ON && make -j4 - # fetch evm384 f6m_mul synthetic loop repo -RUN git clone --single-branch --branch v0.0.1 https://github.com/ewasm/evm384_f6m_mul.git +RUN git clone --single-branch --branch f6m_mul_native https://github.com/ewasm/evm384_f6m_mul.git + +RUN cd evm384_f6m_mul && \ + git submodule update --init && \ + cd src/native && \ + mkdir build && \ + cd build && \ + cmake -DCMAKE_BUILD_TYPE=RELEASE .. && \ + make -j4 WORKDIR / CMD /bin/bash diff --git a/evm/scripts/bench_evm384.py b/evm/scripts/bench_evm384.py index 30558bd..5a1e8e0 100644 --- a/evm/scripts/bench_evm384.py +++ b/evm/scripts/bench_evm384.py @@ -10,6 +10,7 @@ import shutil import shlex import json +from statistics import mean # output paths should be mounted docker volumes RESULT_CSV_OUTPUT_PATH = "/evmraceresults" @@ -66,6 +67,23 @@ def do_evmone_bench(evmone_bench_cmd): time = durationpy.from_str(total_time) return {'gas_used': gasused, 'time': time.total_seconds()} +def do_cpp_native_f6m_mul_bench(): + print("running f6m_mul cpp native benchmark") + f6m_mul_cpp_native_cmd = ['/root/evm384_f6m_mul/src/native/build/synth_loop_benchmark'] + + stdoutlines = [] + with subprocess.Popen(f6m_mul_cpp_native_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True) as p: + for line in p.stdout: # b'\n'-separated lines + print(line, end='') + stdoutlines.append(line) # pass bytes as is + p.wait() + output = ''.join(stdoutlines)[:-1] + 's' + + return { + 'time': durationpy.from_str(output).total_seconds(), + 'bench_name': 'f6m-mul-synth-loop', + 'engine': 'cpp-native' + } def saveResults(precompile_benchmarks): # move existing csv file to backup-datetime-folder @@ -99,6 +117,21 @@ def main(): bench_result['engine'] = "evmone384" all_bench_resuls.append(bench_result) + # do 20 iterations and average results for cpp-native f6m-mul synth loop + cpp_native_results = [] + for i in range(20): + result = do_cpp_native_f6m_mul_bench() + cpp_native_results.append(result) + + cpp_native_result = { + 'time': mean(map(lambda x: x['time'], cpp_native_results)), + 'bench_name': 'f6m-mul-synth-loop', + 'engine': 'cpp-native', + 'gas_used': 0 + } + + all_bench_resuls.append(cpp_native_result) + saveResults(all_bench_resuls) From e70fb63b077b54a707af6ddce1c30fb9d6f80d33 Mon Sep 17 00:00:00 2001 From: Jared Wasinger Date: Tue, 11 Aug 2020 23:59:09 +0000 Subject: [PATCH 2/2] add benchmarks --- benchmark_results_data/evm_benchmarks.csv | 205 +++++++++++----------- 1 file changed, 103 insertions(+), 102 deletions(-) diff --git a/benchmark_results_data/evm_benchmarks.csv b/benchmark_results_data/evm_benchmarks.csv index 2e8d972..032078d 100644 --- a/benchmark_results_data/evm_benchmarks.csv +++ b/benchmark_results_data/evm_benchmarks.csv @@ -1,108 +1,109 @@ engine, test_name, total_time, gas_used engine,test_name,total_time,gas_used -evmone,sha1-10808-bits,0.002056286,790156 -evmone,sha1-21896-bits,0.00393736,1543605 -evmone,sha1-42488-bits,0.007656268,3014404 -evmone,sha1-10808-bits-shiftopt,0.001038472,759785 -evmone,sha1-21896-bits-shiftopt,0.002031567,1484254 -evmone,sha1-42488-bits-shiftopt,0.004000971,2898473 -evmone,bn256g2mul-0xAshish,0.001509638,455757 -evmone,blake2b-8415-bytes-shiftopt,0.018414062,12043105 -evmone,blake2b-5610-bytes-shiftopt,0.01218996,7434820 -evmone,blake2b-2805-bytes-shiftopt,0.006019918,3423526 -evmone,mul256-gcolvin-drag-race,0.009680082,5610277 -evmone,bn128_mul_weierstrudel-cdetrio2,0.000397231,47165 -evmone,bn128_mul_weierstrudel-chfast1,0.000210124,23449 -evmone,bn128_mul_weierstrudel-chfast2,0.000339741,39337 -evmone,bn128_mul_weierstrudel-chfast3,0.000399416,47291 -evmone,bn128_mul_weierstrudel-cdetrio6,0.00039595,47561 -evmone,bn128_mul_weierstrudel-cdetrio7,0.000398454,47165 -evmone,bn128_mul_weierstrudel-cdetrio11,0.000404129,47561 -evmone,bn128_mul_weierstrudel-cdetrio12,0.000396089,47165 -evmone,blake2b-8415-bytes,0.022527799,12351815 -evmone,blake2b-5610-bytes,0.015275491,7640669 -evmone,blake2b-2805-bytes,0.007459261,3526514 -evmone,blake2b_huff-8415-bytes,0.001514828,879920 -evmone,blake2b_huff-5610-bytes,0.001142694,586839 -evmone,blake2b_huff-2805-bytes,0.000524197,293755 +evmone,sha1-10808-bits-shiftopt,0.000825821,759785 +evmone,sha1-21896-bits-shiftopt,0.001562969,1484254 +evmone,sha1-42488-bits-shiftopt,0.003186443,2898473 +evmone,bn256g2mul-0xAshish,0.001271556,455757 +evmone,blake2b-8415-bytes-shiftopt,0.013668602,12043105 +evmone,blake2b-5610-bytes-shiftopt,0.009378376,7434820 +evmone,blake2b-2805-bytes-shiftopt,0.004491046,3423526 +evmone,mul256-gcolvin-drag-race,0.007599084,5610277 +evmone,bn128_mul_weierstrudel-cdetrio2,0.000360764,47165 +evmone,bn128_mul_weierstrudel-chfast1,0.000182614,23449 +evmone,bn128_mul_weierstrudel-chfast2,0.000298584,39337 +evmone,bn128_mul_weierstrudel-chfast3,0.000373645,47291 +evmone,bn128_mul_weierstrudel-cdetrio6,0.000360047,47561 +evmone,bn128_mul_weierstrudel-cdetrio7,0.000348965,47165 +evmone,bn128_mul_weierstrudel-cdetrio11,0.000364878,47561 +evmone,bn128_mul_weierstrudel-cdetrio12,0.000337997,47165 +evmone,blake2b_huff-8415-bytes,0.00112764,879920 +evmone,blake2b_huff-5610-bytes,0.000734551,586839 +evmone,blake2b_huff-2805-bytes,0.000397603,293755 +evmone,blake2b-8415-bytes,0.017831598,12351815 +evmone,blake2b-5610-bytes,0.012580117,7640669 +evmone,blake2b-2805-bytes,0.005750261,3526514 +evmone,sha1-10808-bits,0.001631885,790156 +evmone,sha1-21896-bits,0.003034628,1543605 +evmone,sha1-42488-bits,0.006046502,3014404 engine,test_name,total_time,gas_used -parity-evm,sha1-10808-bits,0.069457059,790156 -parity-evm,sha1-21896-bits,0.128567939,1543605 -parity-evm,sha1-42488-bits,0.250837896,3014404 -parity-evm,sha1-10808-bits-shiftopt,0.026990627,759785 -parity-evm,sha1-21896-bits-shiftopt,0.047456338,1484254 -parity-evm,sha1-42488-bits-shiftopt,0.092930734,2898473 -parity-evm,bn256g2mul-0xAshish,0.024067669,455757 -parity-evm,blake2b-8415-bytes-shiftopt,0.302345632,12043105 -parity-evm,blake2b-5610-bytes-shiftopt,0.216411367,7434820 -parity-evm,blake2b-2805-bytes-shiftopt,0.116367526,3423526 -parity-evm,mul256-gcolvin-drag-race,0.143134311,5610277 -parity-evm,bn128_mul_weierstrudel-cdetrio2,0.004681313,47165 -parity-evm,bn128_mul_weierstrudel-chfast1,0.002312094,23449 -parity-evm,bn128_mul_weierstrudel-chfast2,0.003910487,39337 -parity-evm,bn128_mul_weierstrudel-chfast3,0.004720527,47291 -parity-evm,bn128_mul_weierstrudel-cdetrio6,0.004867541,47561 -parity-evm,bn128_mul_weierstrudel-cdetrio7,0.004679791,47165 -parity-evm,bn128_mul_weierstrudel-cdetrio11,0.004646788,47561 -parity-evm,bn128_mul_weierstrudel-cdetrio12,0.004630612,47165 -parity-evm,blake2b-8415-bytes,0.340011586,12351815 -parity-evm,blake2b-5610-bytes,0.205717038,7640669 -parity-evm,blake2b-2805-bytes,0.096635901,3526514 -parity-evm,blake2b_huff-8415-bytes,0.026595507,879920 -parity-evm,blake2b_huff-5610-bytes,0.01661306,586839 -parity-evm,blake2b_huff-2805-bytes,0.009291144,293755 +parity-evm,sha1-10808-bits-shiftopt,0.022681687,759785 +parity-evm,sha1-21896-bits-shiftopt,0.037793747,1484254 +parity-evm,sha1-42488-bits-shiftopt,0.080097641,2898473 +parity-evm,bn256g2mul-0xAshish,0.019093158,455757 +parity-evm,blake2b-8415-bytes-shiftopt,0.24098813,12043105 +parity-evm,blake2b-5610-bytes-shiftopt,0.165490126,7434820 +parity-evm,blake2b-2805-bytes-shiftopt,0.079788988,3423526 +parity-evm,mul256-gcolvin-drag-race,0.102421079,5610277 +parity-evm,bn128_mul_weierstrudel-cdetrio2,0.003706916,47165 +parity-evm,bn128_mul_weierstrudel-chfast1,0.001689577,23449 +parity-evm,bn128_mul_weierstrudel-chfast2,0.003045638,39337 +parity-evm,bn128_mul_weierstrudel-chfast3,0.003566339,47291 +parity-evm,bn128_mul_weierstrudel-cdetrio6,0.003535184,47561 +parity-evm,bn128_mul_weierstrudel-cdetrio7,0.003536592,47165 +parity-evm,bn128_mul_weierstrudel-cdetrio11,0.003550023,47561 +parity-evm,bn128_mul_weierstrudel-cdetrio12,0.003981209,47165 +parity-evm,blake2b_huff-8415-bytes,0.023224086,879920 +parity-evm,blake2b_huff-5610-bytes,0.018505375,586839 +parity-evm,blake2b_huff-2805-bytes,0.008630519,293755 +parity-evm,blake2b-8415-bytes,0.273193152,12351815 +parity-evm,blake2b-5610-bytes,0.177323457,7640669 +parity-evm,blake2b-2805-bytes,0.087049753,3526514 +parity-evm,sha1-10808-bits,0.050909587,790156 +parity-evm,sha1-21896-bits,0.107236213,1543605 +parity-evm,sha1-42488-bits,0.206253037,3014404 engine,test_name,total_time,gas_used -geth-evm,sha1-10808-bits,0.015162857,790156 -geth-evm,sha1-21896-bits,0.029013649,1543605 -geth-evm,sha1-42488-bits,0.056405393,3014404 -geth-evm,sha1-10808-bits-shiftopt,0.012383896,759785 -geth-evm,sha1-21896-bits-shiftopt,0.023512095,1484254 -geth-evm,sha1-42488-bits-shiftopt,0.046008429,2898473 -geth-evm,bn256g2mul-0xAshish,0.00882746,455757 -geth-evm,blake2b-8415-bytes-shiftopt,0.143346789,12043105 -geth-evm,blake2b-5610-bytes-shiftopt,0.097426734,7434820 -geth-evm,blake2b-2805-bytes-shiftopt,0.048999189,3423526 -geth-evm,mul256-gcolvin-drag-race,0.14476789,5610277 -geth-evm,bn128_mul_weierstrudel-cdetrio2,0.001808681,47165 -geth-evm,bn128_mul_weierstrudel-chfast1,0.000838142,23449 -geth-evm,bn128_mul_weierstrudel-chfast2,0.001490949,39337 -geth-evm,bn128_mul_weierstrudel-chfast3,0.001791839,47291 -geth-evm,bn128_mul_weierstrudel-cdetrio6,0.001907578,47561 -geth-evm,bn128_mul_weierstrudel-cdetrio7,0.001796564,47165 -geth-evm,bn128_mul_weierstrudel-cdetrio11,0.001807007,47561 -geth-evm,bn128_mul_weierstrudel-cdetrio12,0.001773678,47165 -geth-evm,blake2b-8415-bytes,0.150170679,12351815 -geth-evm,blake2b-5610-bytes,0.107144698,7640669 -geth-evm,blake2b-2805-bytes,0.051748147,3526514 -geth-evm,blake2b_huff-8415-bytes,0.018262182,879920 -geth-evm,blake2b_huff-5610-bytes,0.012279144,586839 -geth-evm,blake2b_huff-2805-bytes,0.00614165,293755 +geth-evm,sha1-10808-bits-shiftopt,0.010794754,759785 +geth-evm,sha1-21896-bits-shiftopt,0.022239662,1484254 +geth-evm,sha1-42488-bits-shiftopt,0.040845536,2898473 +geth-evm,bn256g2mul-0xAshish,0.007856129,455757 +geth-evm,blake2b-8415-bytes-shiftopt,0.132213726,12043105 +geth-evm,blake2b-5610-bytes-shiftopt,0.088160969,7434820 +geth-evm,blake2b-2805-bytes-shiftopt,0.044373907,3423526 +geth-evm,mul256-gcolvin-drag-race,0.129451529,5610277 +geth-evm,bn128_mul_weierstrudel-cdetrio2,0.001591463,47165 +geth-evm,bn128_mul_weierstrudel-chfast1,0.000792984,23449 +geth-evm,bn128_mul_weierstrudel-chfast2,0.001313077,39337 +geth-evm,bn128_mul_weierstrudel-chfast3,0.001583505,47291 +geth-evm,bn128_mul_weierstrudel-cdetrio6,0.001660588,47561 +geth-evm,bn128_mul_weierstrudel-cdetrio7,0.001589395,47165 +geth-evm,bn128_mul_weierstrudel-cdetrio11,0.001573866,47561 +geth-evm,bn128_mul_weierstrudel-cdetrio12,0.001556755,47165 +geth-evm,blake2b_huff-8415-bytes,0.016672025,879920 +geth-evm,blake2b_huff-5610-bytes,0.01070878,586839 +geth-evm,blake2b_huff-2805-bytes,0.005402838,293755 +geth-evm,blake2b-8415-bytes,0.13058734,12351815 +geth-evm,blake2b-5610-bytes,0.09154925,7640669 +geth-evm,blake2b-2805-bytes,0.045537792,3526514 +geth-evm,sha1-10808-bits,0.012907775,790156 +geth-evm,sha1-21896-bits,0.026743191,1543605 +geth-evm,sha1-42488-bits,0.050452738,3014404 engine,test_name,total_time,gas_used -cita-evm,sha1-10808-bits,0.050446102,903488 -cita-evm,sha1-21896-bits,0.097787582,1750413 -cita-evm,sha1-42488-bits,0.190869116,3396188 -cita-evm,sha1-10808-bits-shiftopt,0.011862067,873117 -cita-evm,sha1-21896-bits-shiftopt,0.022392031,1691062 -cita-evm,sha1-42488-bits-shiftopt,0.045725851,3280257 -cita-evm,bn256g2mul-0xAshish,0.041878381,486053 -cita-evm,blake2b-8415-bytes-shiftopt,0.215833646,12635513 -cita-evm,blake2b-5610-bytes-shiftopt,0.147089137,7837276 -cita-evm,blake2b-2805-bytes-shiftopt,0.070500257,3635902 -cita-evm,mul256-gcolvin-drag-race,0.042455215,5635901 -cita-evm,bn128_mul_weierstrudel-cdetrio2,0.016598673,74437 -cita-evm,bn128_mul_weierstrudel-chfast1,0.007348978,49441 -cita-evm,bn128_mul_weierstrudel-chfast2,0.013854024,66865 -cita-evm,bn128_mul_weierstrudel-chfast3,0.01711688,74819 -cita-evm,bn128_mul_weierstrudel-cdetrio6,0.017692487,75089 -cita-evm,bn128_mul_weierstrudel-cdetrio7,0.017047063,74501 -cita-evm,bn128_mul_weierstrudel-cdetrio11,0.022045002,75089 -cita-evm,bn128_mul_weierstrudel-cdetrio12,0.016597804,74501 -cita-evm,blake2b-8415-bytes,0.271612442,12944223 -cita-evm,blake2b-5610-bytes,0.178762917,8043125 -cita-evm,blake2b-2805-bytes,0.09112152,3738890 -cita-evm,blake2b_huff-8415-bytes,0.01433503,1471220 -cita-evm,blake2b_huff-5610-bytes,0.009230364,988103 -cita-evm,blake2b_huff-2805-bytes,0.004808496,504983 +cita-evm,sha1-10808-bits-shiftopt,0.010432911,873117 +cita-evm,sha1-21896-bits-shiftopt,0.019753476,1691062 +cita-evm,sha1-42488-bits-shiftopt,0.039671776,3280257 +cita-evm,bn256g2mul-0xAshish,0.036949621,486053 +cita-evm,blake2b-8415-bytes-shiftopt,0.191493324,12635513 +cita-evm,blake2b-5610-bytes-shiftopt,0.127574863,7837276 +cita-evm,blake2b-2805-bytes-shiftopt,0.06295393,3635902 +cita-evm,mul256-gcolvin-drag-race,0.037527379,5635901 +cita-evm,bn128_mul_weierstrudel-cdetrio2,0.014468663,74437 +cita-evm,bn128_mul_weierstrudel-chfast1,0.00632152,49441 +cita-evm,bn128_mul_weierstrudel-chfast2,0.012445637,66865 +cita-evm,bn128_mul_weierstrudel-chfast3,0.015182058,74819 +cita-evm,bn128_mul_weierstrudel-cdetrio6,0.015238954,75089 +cita-evm,bn128_mul_weierstrudel-cdetrio7,0.015010512,74501 +cita-evm,bn128_mul_weierstrudel-cdetrio11,0.015671057,75089 +cita-evm,bn128_mul_weierstrudel-cdetrio12,0.014657662,74501 +cita-evm,blake2b_huff-8415-bytes,0.012236252,1471220 +cita-evm,blake2b_huff-5610-bytes,0.008748997,988103 +cita-evm,blake2b_huff-2805-bytes,0.0043396,504983 +cita-evm,blake2b-8415-bytes,0.244230501,12944223 +cita-evm,blake2b-5610-bytes,0.16335111,8043125 +cita-evm,blake2b-2805-bytes,0.084903451,3738890 +cita-evm,sha1-10808-bits,0.050418756,903488 +cita-evm,sha1-21896-bits,0.090731809,1750413 +cita-evm,sha1-42488-bits,0.172284761,3396188 engine,test_name,total_time,gas_used -evmone384,evm384-synth-loop-v1,0.023534375,12569235 -evmone384,evm384-synth-loop-v2,0.015339457,5851179 +evmone384,evm384-synth-loop-v1,0.021697,12569235 +evmone384,evm384-synth-loop-v2,0.012063,5534811 +cpp-native,f6m-mul-synth-loop,0.0048518499999999996,0