From 7f4705d8d4600fc35f78f03c1632eb72a0b4df58 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 30 Nov 2022 17:35:44 +0100 Subject: [PATCH 1/9] Move lcov commands to a separate script Signed-off-by: Gilles Peskine --- Makefile | 9 +-------- scripts/lcov.sh | 10 ++++++++++ 2 files changed, 11 insertions(+), 8 deletions(-) create mode 100755 scripts/lcov.sh diff --git a/Makefile b/Makefile index 6a8b230077..e5dddba32d 100644 --- a/Makefile +++ b/Makefile @@ -109,14 +109,7 @@ covtest: tests/ssl-opt.sh lcov: - rm -rf Coverage - lcov --capture --initial --directory library -o files.info - lcov --rc lcov_branch_coverage=1 --capture --directory library -o tests.info - lcov --rc lcov_branch_coverage=1 --add-tracefile files.info --add-tracefile tests.info -o all.info - lcov --rc lcov_branch_coverage=1 --remove all.info -o final.info '*.h' - gendesc tests/Descriptions.txt -o descriptions - genhtml --title "mbed TLS" --description-file descriptions --keep-descriptions --legend --branch-coverage -o Coverage final.info - rm -f files.info tests.info all.info final.info descriptions + scripts/lcov.sh apidoc: mkdir -p apidoc diff --git a/scripts/lcov.sh b/scripts/lcov.sh new file mode 100755 index 0000000000..ab86421f9e --- /dev/null +++ b/scripts/lcov.sh @@ -0,0 +1,10 @@ +#!/bin/sh + + rm -rf Coverage + lcov --capture --initial --directory library -o files.info + lcov --rc lcov_branch_coverage=1 --capture --directory library -o tests.info + lcov --rc lcov_branch_coverage=1 --add-tracefile files.info --add-tracefile tests.info -o all.info + lcov --rc lcov_branch_coverage=1 --remove all.info -o final.info '*.h' + gendesc tests/Descriptions.txt -o descriptions + genhtml --title "mbed TLS" --description-file descriptions --keep-descriptions --legend --branch-coverage -o Coverage final.info + rm -f files.info tests.info all.info final.info descriptions From 862e4a3f8da43eaf41689e3b89fdd25541c50028 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 30 Nov 2022 17:51:44 +0100 Subject: [PATCH 2/9] Add a bit of documentation Signed-off-by: Gilles Peskine --- scripts/lcov.sh | 58 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/scripts/lcov.sh b/scripts/lcov.sh index ab86421f9e..8c8c8468bd 100755 --- a/scripts/lcov.sh +++ b/scripts/lcov.sh @@ -1,10 +1,52 @@ #!/bin/sh - rm -rf Coverage - lcov --capture --initial --directory library -o files.info - lcov --rc lcov_branch_coverage=1 --capture --directory library -o tests.info - lcov --rc lcov_branch_coverage=1 --add-tracefile files.info --add-tracefile tests.info -o all.info - lcov --rc lcov_branch_coverage=1 --remove all.info -o final.info '*.h' - gendesc tests/Descriptions.txt -o descriptions - genhtml --title "mbed TLS" --description-file descriptions --keep-descriptions --legend --branch-coverage -o Coverage final.info - rm -f files.info tests.info all.info final.info descriptions +help () { + cat < Date: Wed, 30 Nov 2022 17:52:01 +0100 Subject: [PATCH 3/9] Call the new lcov script from CMake builds The code in CMakeLists.txt was an old copy of the code in Makefile. This brings in branch coverage, which had only been added to Makefile. Signed-off-by: Gilles Peskine --- CMakeLists.txt | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 72cc737c6e..7d02986855 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -319,14 +319,7 @@ if(ENABLE_TESTING) ) ADD_CUSTOM_TARGET(lcov - COMMAND rm -rf Coverage - COMMAND lcov --capture --initial --directory library/CMakeFiles/mbedtls.dir -o files.info - COMMAND lcov --capture --directory library/CMakeFiles/mbedtls.dir -o tests.info - COMMAND lcov --add-tracefile files.info --add-tracefile tests.info -o all.info - COMMAND lcov --remove all.info -o final.info '*.h' - COMMAND gendesc tests/Descriptions.txt -o descriptions - COMMAND genhtml --title "mbed TLS" --description-file descriptions --keep-descriptions --legend --no-branch-coverage -o Coverage final.info - COMMAND rm -f files.info tests.info all.info final.info descriptions + COMMAND scripts/lcov.sh ) ADD_CUSTOM_TARGET(memcheck From b0f11f1429fe42fff7a6a384fd4d7b7719c2f7c2 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 30 Nov 2022 17:52:54 +0100 Subject: [PATCH 4/9] Fix "make lcov" in CMake out-of-tree builds Signed-off-by: Gilles Peskine --- tests/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index bc02627362..ca477acedd 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -97,6 +97,7 @@ if (NOT ${CMAKE_CURRENT_BINARY_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}) if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/seedfile") link_to_source(seedfile) endif() + link_to_source(Descriptions.txt) link_to_source(compat.sh) link_to_source(context-info.sh) link_to_source(data_files) From f11c33c27c11ddada5e7c5d65c335f4e12e334c6 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 30 Nov 2022 17:56:58 +0100 Subject: [PATCH 5/9] Put temporary files in a temporary directory Signed-off-by: Gilles Peskine --- scripts/lcov.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/lcov.sh b/scripts/lcov.sh index 8c8c8468bd..80ebc79870 100755 --- a/scripts/lcov.sh +++ b/scripts/lcov.sh @@ -34,13 +34,14 @@ set -eu lcov_rebuild_stats () { rm -rf Coverage - lcov --capture --initial --directory library -o files.info - lcov --rc lcov_branch_coverage=1 --capture --directory library -o tests.info - lcov --rc lcov_branch_coverage=1 --add-tracefile files.info --add-tracefile tests.info -o all.info - lcov --rc lcov_branch_coverage=1 --remove all.info -o final.info '*.h' - gendesc tests/Descriptions.txt -o descriptions - genhtml --title "mbed TLS" --description-file descriptions --keep-descriptions --legend --branch-coverage -o Coverage final.info - rm -f files.info tests.info all.info final.info descriptions + mkdir Coverage Coverage/tmp + lcov --capture --initial --directory library -o Coverage/tmp/files.info + lcov --rc lcov_branch_coverage=1 --capture --directory library -o Coverage/tmp/tests.info + lcov --rc lcov_branch_coverage=1 --add-tracefile Coverage/tmp/files.info --add-tracefile Coverage/tmp/tests.info -o Coverage/tmp/all.info + lcov --rc lcov_branch_coverage=1 --remove Coverage/tmp/all.info -o Coverage/tmp/final.info '*.h' + gendesc tests/Descriptions.txt -o Coverage/tmp/descriptions + genhtml --title "mbed TLS" --description-file Coverage/tmp/descriptions --keep-descriptions --legend --branch-coverage -o Coverage Coverage/tmp/final.info + rm -f Coverage/tmp/*.info Coverage/tmp/descriptions echo "Coverage report in: Coverage/index.html" } From 26aae470428f6d48277584aca6761174ebc699c7 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 30 Nov 2022 18:08:14 +0100 Subject: [PATCH 6/9] Add option to reset the traces, to use after rework Signed-off-by: Gilles Peskine --- scripts/lcov.sh | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/scripts/lcov.sh b/scripts/lcov.sh index 80ebc79870..718166ddb2 100755 --- a/scripts/lcov.sh +++ b/scripts/lcov.sh @@ -2,7 +2,7 @@ help () { cat <&1; exit 120;; + esac +done +shift $((OPTIND - 1)) + +"$main" "$@" From bb3d31659b6e66ec678862a6e6532d67b7d6466c Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 1 Dec 2022 17:41:36 +0100 Subject: [PATCH 7/9] You need --coverage when linking as well Signed-off-by: Gilles Peskine --- Makefile | 5 +++-- scripts/lcov.sh | 8 +++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index e5dddba32d..5e0275a16a 100644 --- a/Makefile +++ b/Makefile @@ -100,8 +100,9 @@ check: lib tests test: check ifndef WINDOWS -# note: for coverage testing, build with: -# make CFLAGS='--coverage -g3 -O0' +# Note: for coverage testing, build with: +# make CFLAGS='--coverage -g3 -O0' LDFLAGS='--coverage -g3 -O0' +# For the reference coverage measurement, see tests/scripts/basic-build-test.sh covtest: $(MAKE) check programs/test/selftest diff --git a/scripts/lcov.sh b/scripts/lcov.sh index 718166ddb2..8d141eedf7 100755 --- a/scripts/lcov.sh +++ b/scripts/lcov.sh @@ -6,8 +6,14 @@ Usage: $0 [-r] Collect coverage statistics of library code into an HTML report. General instructions: -1. Build the library with CFLAGS="--coverage -O0 -g3". +1. Build the library with CFLAGS="--coverage -O0 -g3" and link the test + programs with LDFLAGS="--coverage". This can be an out-of-tree build. + For example (in-tree): + make CFLAGS="--coverage -O0 -g3" LDFLAGS="--coverage" + Or (out-of-tree): + mkdir build-coverage && cd build-coverage && + cmake -D CMAKE_BUILD_TYPE=Coverage .. && make 2. Run whatever tests you want. 3. Run this script from the parent of the directory containing the library object files and coverage statistics files. From 141b551df62664d28e02b1b0fa3966f389e205e7 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 1 Dec 2022 18:05:31 +0100 Subject: [PATCH 8/9] No need for -g or -O in LDFLAGS Fix a mistake in the previous commit. Signed-off-by: Gilles Peskine --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5e0275a16a..7aa7bceccd 100644 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ test: check ifndef WINDOWS # Note: for coverage testing, build with: -# make CFLAGS='--coverage -g3 -O0' LDFLAGS='--coverage -g3 -O0' +# make CFLAGS='--coverage -g3 -O0' LDFLAGS='--coverage' # For the reference coverage measurement, see tests/scripts/basic-build-test.sh covtest: $(MAKE) check From eaac1afcf1057e0cfe9739b0d729fa24aecc513e Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 8 Dec 2022 22:18:31 +0100 Subject: [PATCH 9/9] Remove obsolete target covtest This target dates back from before basic-build-test.sh existed. Refer to that script instead. Signed-off-by: Gilles Peskine --- CMakeLists.txt | 14 +++++++------- Makefile | 18 ++++++++---------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7d02986855..ddeb115e85 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -311,13 +311,13 @@ if(ENABLE_TESTING) # additional convenience targets for Unix only if(UNIX) - ADD_CUSTOM_TARGET(covtest - COMMAND make test - COMMAND programs/test/selftest - COMMAND tests/compat.sh - COMMAND tests/ssl-opt.sh - ) - + # For coverage testing: + # 1. Build with: + # cmake -D CMAKE_BUILD_TYPE=Coverage /path/to/source && make + # 2. Run the relevant tests for the part of the code you're interested in. + # For the reference coverage measurement, see + # tests/scripts/basic-build-test.sh + # 3. Run scripts/lcov.sh to generate an HTML report. ADD_CUSTOM_TARGET(lcov COMMAND scripts/lcov.sh ) diff --git a/Makefile b/Makefile index 7aa7bceccd..1efe89dd9f 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ PREFIX=mbedtls_ .SILENT: -.PHONY: all no_test programs lib tests install uninstall clean test check covtest lcov apidoc apidoc_clean +.PHONY: all no_test programs lib tests install uninstall clean test check lcov apidoc apidoc_clean all: programs tests $(MAKE) post_build @@ -100,15 +100,13 @@ check: lib tests test: check ifndef WINDOWS -# Note: for coverage testing, build with: -# make CFLAGS='--coverage -g3 -O0' LDFLAGS='--coverage' -# For the reference coverage measurement, see tests/scripts/basic-build-test.sh -covtest: - $(MAKE) check - programs/test/selftest - tests/compat.sh - tests/ssl-opt.sh - +# For coverage testing: +# 1. Build with: +# make CFLAGS='--coverage -g3 -O0' LDFLAGS='--coverage' +# 2. Run the relevant tests for the part of the code you're interested in. +# For the reference coverage measurement, see +# tests/scripts/basic-build-test.sh +# 3. Run scripts/lcov.sh to generate an HTML report. lcov: scripts/lcov.sh