From 6198cf7e6e97d8ab89602f9afc5c1324575e8ffd Mon Sep 17 00:00:00 2001 From: Kunal Bhargava Date: Tue, 29 Oct 2024 18:15:51 +0000 Subject: [PATCH 1/2] add exec-fixtures --- commands.md | 19 ++++++++ src/test_suite/test_suite.py | 91 +++++++++++++++++++++++++++++++++++- 2 files changed, 108 insertions(+), 2 deletions(-) diff --git a/commands.md b/commands.md index 4201f67..315b7b5 100644 --- a/commands.md +++ b/commands.md @@ -19,6 +19,7 @@ $ solana-test-suite [OPTIONS] COMMAND [ARGS]... * `create-fixtures`: Create test fixtures from a directory of... * `debug-mismatches`: Run tests on a set of targets with a list... * `decode-protobufs`: Convert Context and/or Fixture messages to... +* `exec-fixtures`: Execute fixtures and check for correct... * `execute`: Execute Context or Fixture message(s) and... * `fix-to-ctx`: Extract Context messages from Fixtures. * `list-harness-types`: List harness types available for use. @@ -95,6 +96,24 @@ $ solana-test-suite decode-protobufs [OPTIONS] * `-h, --default-harness-type TEXT`: Harness type to use for Context protobufs [default: InstrHarness] * `--help`: Show this message and exit. +## `solana-test-suite exec-fixtures` + +Execute fixtures and check for correct effects + +**Usage**: + +```console +$ solana-test-suite exec-fixtures [OPTIONS] +``` + +**Options**: + +* `-i, --input PATH`: Input protobuf file or directory of protobuf files +* `-t, --target PATH`: Shared object (.so) target file path to execute [default: impl/firedancer/build/native/clang/lib/libfd_exec_sol_compat.so] +* `-r, --randomize-output-buffer`: Randomizes bytes in output buffer before shared library execution +* `-l, --log-level INTEGER`: FD logging level [default: 2] +* `--help`: Show this message and exit. + ## `solana-test-suite execute` Execute Context or Fixture message(s) and print the Effects. diff --git a/src/test_suite/test_suite.py b/src/test_suite/test_suite.py index c13d334..557026a 100644 --- a/src/test_suite/test_suite.py +++ b/src/test_suite/test_suite.py @@ -473,8 +473,10 @@ def run_tests( print(f"Total test cases: {passed + failed + skipped}") print(f"Passed: {passed}, Failed: {failed}, Skipped: {skipped}") if verbose: - print(f"Failed tests: {failed_tests}") - print(f"Skipped tests: {skipped_tests}") + if failed != 0: + print(f"Failed tests: {failed_tests}") + if skipped != 0: + print(f"Skipped tests: {skipped_tests}") if failed != 0 and save_failures: print("Failures tests are in: ", globals.output_dir / "failed_protobufs") @@ -923,5 +925,90 @@ def get_harness_type_for_folder(src, regenerate_folder): print(f"Regenerated fixtures from {test_vectors} to {output_dir}") +@app.command( + help=f""" + Execute fixtures and check for correct effects + """ +) +def exec_fixtures( + input: Path = typer.Option( + None, + "--input", + "-i", + help=f"Input protobuf file or directory of protobuf files", + ), + shared_library: Path = typer.Option( + Path("impl/firedancer/build/native/clang/lib/libfd_exec_sol_compat.so"), + "--target", + "-t", + help="Shared object (.so) target file path to execute", + ), + randomize_output_buffer: bool = typer.Option( + False, + "--randomize-output-buffer", + "-r", + help="Randomizes bytes in output buffer before shared library execution", + ), + log_level: int = typer.Option( + 2, + "--log-level", + "-l", + help="FD logging level", + ), +): + # Initialize output buffers and shared library + initialize_process_output_buffers(randomize_output_buffer=randomize_output_buffer) + try: + lib = ctypes.CDLL(shared_library) + lib.sol_compat_init(log_level) + globals.target_libraries[shared_library] = lib + globals.reference_shared_library = shared_library + except: + set_ld_preload_asan() + + passed = 0 + failed = 0 + skipped = 0 + failed_tests = [] + skipped_tests = [] + + files_to_exec = input.iterdir() if input.is_dir() else [input] + for file in files_to_exec: + if file.suffix == ".fix": + fn_entrypoint = extract_metadata(file).fn_entrypoint + harness_ctx = ENTRYPOINT_HARNESS_MAP[fn_entrypoint] + fixture = read_fixture(file) + context = fixture.input + output = fixture.output + else: + print(f"File {file} is not a fixture") + skipped += 1 + skipped_tests.append(file.stem) + continue + + effects = process_target(harness_ctx, lib, context) + + if not effects: + print(f"No {harness_ctx.effects_type.__name__} returned for file {file}") + skipped += 1 + skipped_tests.append(file.stem) + continue + + if output == effects: + passed += 1 + else: + failed += 1 + failed_tests.append(file.stem) + + print(f"Total test cases: {passed + failed + skipped}") + print(f"Passed: {passed}, Failed: {failed}, Skipped: {skipped}") + if failed != 0: + print(f"Failed tests: {failed_tests}") + if skipped != 0: + print(f"Skipped tests: {skipped_tests}") + + lib.sol_compat_fini() + + if __name__ == "__main__": app() From 07ffa4e6a7ea486772dbe9930482047d78121ea1 Mon Sep 17 00:00:00 2001 From: Kunal Bhargava Date: Tue, 29 Oct 2024 19:20:55 +0000 Subject: [PATCH 2/2] multiprocess exec_fixtures --- commands.md | 1 + src/test_suite/multiprocessing_utils.py | 20 +++++++++- src/test_suite/test_suite.py | 51 +++++++++++++------------ 3 files changed, 46 insertions(+), 26 deletions(-) diff --git a/commands.md b/commands.md index 315b7b5..8b8d41e 100644 --- a/commands.md +++ b/commands.md @@ -112,6 +112,7 @@ $ solana-test-suite exec-fixtures [OPTIONS] * `-t, --target PATH`: Shared object (.so) target file path to execute [default: impl/firedancer/build/native/clang/lib/libfd_exec_sol_compat.so] * `-r, --randomize-output-buffer`: Randomizes bytes in output buffer before shared library execution * `-l, --log-level INTEGER`: FD logging level [default: 2] +* `-p, --num-processes INTEGER`: Number of processes to use [default: 4] * `--help`: Show this message and exit. ## `solana-test-suite execute` diff --git a/src/test_suite/multiprocessing_utils.py b/src/test_suite/multiprocessing_utils.py index 72d7dcc..4f7ca28 100644 --- a/src/test_suite/multiprocessing_utils.py +++ b/src/test_suite/multiprocessing_utils.py @@ -1,7 +1,7 @@ from dataclasses import dataclass, field from test_suite.constants import OUTPUT_BUFFER_SIZE from test_suite.fuzz_context import ENTRYPOINT_HARNESS_MAP, HarnessCtx -from test_suite.fuzz_interface import ContextType +from test_suite.fuzz_interface import ContextType, EffectsType import test_suite.invoke_pb2 as invoke_pb import test_suite.metadata_pb2 as metadata_pb2 import ctypes @@ -372,3 +372,21 @@ def run_test(test_file: Path) -> tuple[str, int, dict | None]: results = process_single_test_case(harness_ctx, context) pruned_results = harness_ctx.prune_effects_fn(context, results) return test_file.stem, *build_test_results(harness_ctx, pruned_results) + + +def execute_fixture(test_file: Path) -> tuple[str, int | None]: + if test_file.suffix != ".fix": + print(f"File {test_file} is not a fixture") + return test_file.stem, None + + fn_entrypoint = extract_metadata(test_file).fn_entrypoint + harness_ctx = ENTRYPOINT_HARNESS_MAP[fn_entrypoint] + fixture = read_fixture(test_file) + context = fixture.input + output = fixture.output + + effects = process_target( + harness_ctx, globals.target_libraries[globals.reference_shared_library], context + ) + + return test_file.stem, effects == output diff --git a/src/test_suite/test_suite.py b/src/test_suite/test_suite.py index 557026a..bb7136b 100644 --- a/src/test_suite/test_suite.py +++ b/src/test_suite/test_suite.py @@ -19,6 +19,7 @@ ) from test_suite.multiprocessing_utils import ( decode_single_test_case, + execute_fixture, extract_metadata, read_fixture, initialize_process_output_buffers, @@ -94,7 +95,7 @@ def execute( except: set_ld_preload_asan() - files_to_exec = input.iterdir() if input.is_dir() else [input] + files_to_exec = list(input.iterdir()) if input.is_dir() else [input] for file in files_to_exec: print(f"Handling {file}...") if file.suffix == ".fix": @@ -162,7 +163,7 @@ def fix_to_ctx( shutil.rmtree(globals.output_dir) globals.output_dir.mkdir(parents=True, exist_ok=True) - test_cases = input.iterdir() if input.is_dir() else [input] + test_cases = list(input.iterdir()) if input.is_dir() else [input] num_test_cases = len(test_cases) print(f"Converting to Fixture messages...") @@ -955,6 +956,9 @@ def exec_fixtures( "-l", help="FD logging level", ), + num_processes: int = typer.Option( + 4, "--num-processes", "-p", help="Number of processes to use" + ), ): # Initialize output buffers and shared library initialize_process_output_buffers(randomize_output_buffer=randomize_output_buffer) @@ -966,39 +970,36 @@ def exec_fixtures( except: set_ld_preload_asan() + test_cases = list(input.iterdir()) if input.is_dir() else [input] + num_test_cases = len(test_cases) + print("Running tests...") + results = [] + with Pool( + processes=num_processes, + initializer=initialize_process_output_buffers, + initargs=(randomize_output_buffer,), + ) as pool: + for result in tqdm.tqdm( + pool.imap(execute_fixture, test_cases), + total=num_test_cases, + ): + results.append(result) + passed = 0 failed = 0 skipped = 0 failed_tests = [] skipped_tests = [] - files_to_exec = input.iterdir() if input.is_dir() else [input] - for file in files_to_exec: - if file.suffix == ".fix": - fn_entrypoint = extract_metadata(file).fn_entrypoint - harness_ctx = ENTRYPOINT_HARNESS_MAP[fn_entrypoint] - fixture = read_fixture(file) - context = fixture.input - output = fixture.output - else: - print(f"File {file} is not a fixture") + for file, status in results: + if status == None: skipped += 1 - skipped_tests.append(file.stem) - continue - - effects = process_target(harness_ctx, lib, context) - - if not effects: - print(f"No {harness_ctx.effects_type.__name__} returned for file {file}") - skipped += 1 - skipped_tests.append(file.stem) - continue - - if output == effects: + skipped_tests.append(file) + elif status == 1: passed += 1 else: failed += 1 - failed_tests.append(file.stem) + failed_tests.append(file) print(f"Total test cases: {passed + failed + skipped}") print(f"Passed: {passed}, Failed: {failed}, Skipped: {skipped}")