Skip to content

Commit

Permalink
Coverage collection for Firefly (#266)
Browse files Browse the repository at this point in the history 
* iele-coverage.md: Add coverage collection semantics

* VM responds with coverage over protobuf

* Add configuration variable for enabling coverage

* Add coverage test

* proto/proto/VERSION: 1.2 -> 3.0

* vm/c/vm.cpp: Include bytecode in CodeCoverage message
  • Loading branch information
@gtrepta
gtrepta authored Apr 15, 2021
1 parent f0afc0a commit f72acd6
Show file tree
Hide file tree
Showing 16 changed files with 444 additions and 35 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
/tests/VMTests
/tests/BlockchainTests
/tests/**/*.test-assembled
/tests/**/*.coverage-out
/tmp
/.kompile*
/.kprove*
Expand Down
15 changes: 8 additions & 7 deletions Jenkinsfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,14 @@ pipeline {
stage('Test') {
options { timeout(time: 30, unit: 'MINUTES') }
parallel {
stage('EVM Tests') { steps { sh 'make test-vm -j4' } }
stage('IELE Tests') { steps { sh 'make test-iele -j4' } }
stage('IELE Tests (Haskell)') { steps { sh 'make test-iele-haskell -j4' } }
stage('Well Formed Check') { steps { sh 'make test-wellformed -j4' } }
stage('Ill Formed Check') { steps { sh 'make test-illformed -j4' } }
stage('Interactive') { steps { sh 'make test-interactive' } }
stage('Node') { steps { sh 'make test-node -j4 TEST_PORT=9001' } }
stage('EVM Tests') { steps { sh 'make test-vm -j4' } }
stage('IELE Tests') { steps { sh 'make test-iele -j4' } }
stage('IELE Tests (Haskell)') { steps { sh 'make test-iele-haskell -j4' } }
stage('IELE Tests (Coverage)') { steps { sh 'make test-iele-coverage -j4' } }
stage('Well Formed Check') { steps { sh 'make test-wellformed -j4' } }
stage('Ill Formed Check') { steps { sh 'make test-illformed -j4' } }
stage('Interactive') { steps { sh 'make test-interactive' } }
stage('Node') { steps { sh 'make test-node -j4 TEST_PORT=9001' } }
}
}
}
Expand Down
16 changes: 13 additions & 3 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ export PATH:=$(IELE_BIN):$(PATH)
.PHONY: all clean distclean libff protobuf coverage secp256k1 cryptopp \
build build-interpreter build-vm build-check build-haskell build-node build-testnode \
install install-interpreter install-vm install-kiele install-check uninstall \
split-tests split-vm-tests split-blockchain-tests test-node \
split-tests split-vm-tests split-blockchain-tests test-node test-iele-coverage \
test-evm test-vm test-blockchain test-wellformed test-illformed test-bad-packet test-interactive \
test-iele test-iele-haskell test-iele-failing test-iele-slow test-iele-node assemble-iele-test test
.SECONDARY:
Expand Down Expand Up @@ -128,6 +128,7 @@ $(libcryptopp_out): $(PLUGIN)/deps/cryptopp/GNUmakefile
# -----

TEST = kiele
CHECK = git --no-pager diff --no-index --ignore-all-space -R
TEST_ASSEMBLE = $(IELE_DIR)/assemble-iele-test
TEST_BACKEND = standalone
TEST_MODE = NORMAL
Expand All @@ -136,7 +137,7 @@ TEST_PORT = 10000
TEST_ARGS = --no-unparse
TEST_DIR = $(IELE_DIR)/tests

test: split-tests test-vm test-iele test-iele-haskell test-wellformed test-illformed test-interactive test-node
test: split-tests test-vm test-iele test-iele-haskell test-iele-coverage test-wellformed test-illformed test-interactive test-node

split-tests: split-vm-tests split-blockchain-tests

Expand Down Expand Up @@ -164,6 +165,8 @@ skipped_tests=$(failing_tests) \
$(wildcard $(TEST_DIR)/BlockchainTests/GeneralStateTests/stStaticCall/static_Return50000*/*.iele.json) \
$(wildcard $(TEST_DIR)/BlockchainTests/GeneralStateTests/stStaticCall/static_Call1MB1024Calldepth_d1g0v0/*.iele.json) \

coverage_tests=$(TEST_DIR)/iele/danse/ERC20/transfer_Caller-MoreThanBalance.iele.json

passing_tests=$(filter-out $(skipped_tests), $(all_tests))
passing_vm_tests=$(filter-out $(skipped_tests), $(vm_tests))
passing_blockchain_tests=$(filter-out $(skipped_tests), $(blockchain_tests))
Expand All @@ -180,6 +183,7 @@ iele_slow=$(slow_tests:=.test)
iele_failing=$(failing_tests:=.test)
iele_targets=$(iele_passing_tests:=.test)
iele_node_targets=$(iele_tests:=.nodetest)
iele_coverage_tests=$(coverage_tests:=.test-coverage)

iele_contracts=$(wildcard iele-examples/*.iele $(TEST_DIR)/iele/*/*/*.iele)
well_formed_contracts=$(filter-out $(wildcard $(TEST_DIR)/iele/*/ill-formed/*.iele), $(iele_contracts))
Expand All @@ -199,6 +203,7 @@ test-iele-node: $(iele_node_targets)
assemble-iele-test: $(iele_assembled)
test-wellformed: $(well_formedness_targets)
test-illformed: $(ill_formedness_targets)
test-iele-coverage: $(iele_coverage_tests)

test-bad-packet:
netcat 127.0.0.1 $(TEST_PORT) -q 2 < $(TEST_DIR)/bad-packet
Expand Down Expand Up @@ -227,6 +232,11 @@ $(TEST_DIR)/VMTests/%: TEST_MODE = VMTESTS
%.json.test: %.json.test-assembled
$(TEST) interpret --backend $(TEST_BACKEND) --mode $(TEST_MODE) --schedule $(TEST_SCHEDULE) $(TEST_ARGS) $<

%.json.test-coverage: %.json.test-assembled
$(TEST) interpret --backend $(TEST_BACKEND) --mode $(TEST_MODE) --schedule $(TEST_SCHEDULE) --coverage $< | grep -A 30 "<kiele-coverage>" > $*.json.coverage-out
$(CHECK) $*.json.coverage-out $*.json.coverage-expected
rm -rf $*.json.coverage-out

%.json.test-assembled: %.json
$(TEST_ASSEMBLE) $< > $@

Expand All @@ -251,7 +261,7 @@ $(TEST_DIR)/evm-to-iele/evm-to-iele: $(wildcard $(TEST_DIR)/evm-to-iele/*.ml $(T
# Build Source Files
# ------------------

k_files:=$(addprefix $(IELE_DIR)/,iele-testing.md data.md iele.md iele-gas.md iele-binary.md plugin/plugin/krypto.md iele-syntax.md iele-node.md well-formedness.md)
k_files:=$(addprefix $(IELE_DIR)/,iele-testing.md data.md iele.md iele-gas.md iele-binary.md plugin/plugin/krypto.md iele-syntax.md iele-node.md well-formedness.md iele-coverage.md)
checker_files:=$(addprefix $(IELE_DIR)/,iele-syntax.md well-formedness.md data.md)

# LLVM Builds
Expand Down
Loading

0 comments on commit f72acd6

Please sign in to comment.