From 5eb94fa94ff01a10f2c12e6c78e18a135c7514a0 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Sun, 13 Sep 2026 17:59:51 -0300 Subject: [PATCH 1/3] Reach and require 100% function coverage Signed-off-by: Juan Cruz Viotti --- .github/workflows/website.yml | 4 ++++ contrib/website.sh | 26 ++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index 30c4475f1c..63ada7f487 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -49,6 +49,10 @@ jobs: # Container jobs run as root, and a superuser is not constrained by the # permission bits that part of the suite asserts on - name: Build the website + # Every function that the published report measures has to be called by + # the test suite, so that a change cannot quietly lower it + env: + REQUIRE_FULL_FUNCTION_COVERAGE: "1" run: | useradd --create-home builder chown -R builder . diff --git a/contrib/website.sh b/contrib/website.sh index 4839468610..5034bc26a0 100755 --- a/contrib/website.sh +++ b/contrib/website.sh @@ -69,10 +69,9 @@ export CMAKE_BUILD_PARALLEL_LEVEL export CTEST_PARALLEL_LEVEL # Counters are kept in the profile of each program as they change, rather than -# written out once it exits, so that a program that dies on a fatal signal, like -# the ones exercising the crash handler, still reports what it ran. The profile -# file name asks for it, which is all that Apple platforms need, while elsewhere -# the compiler has to arrange for it too +# written out once it exits, so that a program that dies on a fatal signal still +# reports what it ran. The profile file name asks for it, which is all that +# Apple platforms need, while elsewhere the compiler has to arrange for it too PROFILE_FLAGS="-fprofile-instr-generate -fcoverage-mapping" if [ "$(uname)" != "Darwin" ] then @@ -264,6 +263,25 @@ done < "$OBJECT_LIST" "-ignore-filename-regex=$EXCLUDE" \ -show-branches=count +# Optionally require every function under measurement to be reached by the +# suite, judged over the same view as the published report. Only the platform +# that the report is published from is held to it, as a report produced +# elsewhere measures a different set of code +if [ -n "${REQUIRE_FULL_FUNCTION_COVERAGE:-}" ] +then + "$LLVM_COV" report "$MAIN_OBJECT" "$@" \ + "-instr-profile=$PROFILE_DATA" \ + "-ignore-filename-regex=$EXCLUDE" > "$WORK_DIRECTORY/report.txt" + MISSED_FUNCTIONS="$(awk '$1 == "TOTAL" { print $6 }' "$WORK_DIRECTORY/report.txt")" + if [ "$MISSED_FUNCTIONS" != "0" ] + then + echo "The test suite never calls $MISSED_FUNCTIONS function(s) in:" >&2 + awk '$1 != "TOTAL" && $6 ~ /^[0-9]+$/ && $6 != "0" { print " " $1 " (" $6 ")" }' \ + "$WORK_DIRECTORY/report.txt" >&2 + exit 1 + fi +fi + # Whatever the report generator emitted is taken as is rather than named entry # by entry, and only the entries about to be written are cleared, so that the # destination is never removed wholesale From 89d34d9f661b11f01b801e34e5c44230bcadc010 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Sun, 13 Sep 2026 18:19:30 -0300 Subject: [PATCH 2/3] Fix Signed-off-by: Juan Cruz Viotti --- contrib/website.sh | 72 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 51 insertions(+), 21 deletions(-) diff --git a/contrib/website.sh b/contrib/website.sh index 5034bc26a0..96b8fa8be7 100755 --- a/contrib/website.sh +++ b/contrib/website.sh @@ -163,10 +163,29 @@ do done < "$OBJECT_LIST" # Merge the traces line by line, keeping the highest count observed for every -# line and branch, then emit a merged LCOV trace plus a per file summary +# line, branch, and function, then emit a merged LCOV trace plus a per file +# summary. A function is identified by where it is defined, so that every +# instantiation of a template counts towards the same one MERGE_PROGRAM="$WORK_DIRECTORY/merge.awk" cat > "$MERGE_PROGRAM" <<'AWK' /^SF:/ { source = substr($0, 4); files[source] = 1; next } +/^FN:/ { + split(substr($0, 4), record, ",") + function_lines[source SUBSEP substr($0, 5 + length(record[1]))] = record[1] + key = source SUBSEP record[1] + if (!(key in functions)) { + functions[key] = 0 + } + next +} +/^FNDA:/ { + split(substr($0, 6), record, ",") + key = source SUBSEP function_lines[source SUBSEP substr($0, 7 + length(record[1]))] + if (record[1] + 0 > functions[key] + 0) { + functions[key] = record[1] + 0 + } + next +} /^DA:/ { split(substr($0, 4), record, ",") key = source SUBSEP record[1] @@ -238,12 +257,42 @@ END { total_covered, total_lines printf "%8.2f%% %6d/%-6d TOTAL branches\n", branch_percentage, total_branches_covered, total_branches + total_functions = 0 + total_functions_covered = 0 + printf "" > uncovered + for (key in functions) { + total_functions += 1 + if (functions[key] > 0) { + total_functions_covered += 1 + } else { + split(key, parts, SUBSEP) + printf "%s:%s\n", parts[1], parts[2] > uncovered + } + } + close(uncovered) + function_percentage = total_functions > 0 \ + ? (total_functions_covered * 100.0) / total_functions : 100 + printf "%8.2f%% %6d/%-6d TOTAL functions\n", function_percentage, + total_functions_covered, total_functions } AWK -awk -v "merged=$WORK_DIRECTORY/coverage.lcov" -f "$MERGE_PROGRAM" \ +UNCOVERED_FUNCTIONS="$WORK_DIRECTORY/uncovered.txt" +awk -v "merged=$WORK_DIRECTORY/coverage.lcov" \ + -v "uncovered=$UNCOVERED_FUNCTIONS" -f "$MERGE_PROGRAM" \ "$LCOV_DIRECTORY"/*.lcov > "$WORK_DIRECTORY/summary.txt" +# Optionally require every function under measurement to be reached by the +# suite, judged over the merged traces for the reason given above. Only the +# platform that the report is published from is held to it, as a report +# produced elsewhere measures a different set of code +if [ -n "${REQUIRE_FULL_FUNCTION_COVERAGE:-}" ] && [ -s "$UNCOVERED_FUNCTIONS" ] +then + echo "The test suite never calls the functions defined at:" >&2 + sort -t : -k 1,1 -k 2,2n "$UNCOVERED_FUNCTIONS" >&2 + exit 1 +fi + # The browsable report keeps the combined view. Its annotated sources can still # under count the header inline cases described above, so the summary file # carries the authoritative numbers @@ -263,25 +312,6 @@ done < "$OBJECT_LIST" "-ignore-filename-regex=$EXCLUDE" \ -show-branches=count -# Optionally require every function under measurement to be reached by the -# suite, judged over the same view as the published report. Only the platform -# that the report is published from is held to it, as a report produced -# elsewhere measures a different set of code -if [ -n "${REQUIRE_FULL_FUNCTION_COVERAGE:-}" ] -then - "$LLVM_COV" report "$MAIN_OBJECT" "$@" \ - "-instr-profile=$PROFILE_DATA" \ - "-ignore-filename-regex=$EXCLUDE" > "$WORK_DIRECTORY/report.txt" - MISSED_FUNCTIONS="$(awk '$1 == "TOTAL" { print $6 }' "$WORK_DIRECTORY/report.txt")" - if [ "$MISSED_FUNCTIONS" != "0" ] - then - echo "The test suite never calls $MISSED_FUNCTIONS function(s) in:" >&2 - awk '$1 != "TOTAL" && $6 ~ /^[0-9]+$/ && $6 != "0" { print " " $1 " (" $6 ")" }' \ - "$WORK_DIRECTORY/report.txt" >&2 - exit 1 - fi -fi - # Whatever the report generator emitted is taken as is rather than named entry # by entry, and only the entries about to be written are cleared, so that the # destination is never removed wholesale From 9594aacdffe1ae5ec1e45c914b1d01eab410e644 Mon Sep 17 00:00:00 2001 From: Juan Cruz Viotti Date: Sun, 13 Sep 2026 18:47:46 -0300 Subject: [PATCH 3/3] Fix Signed-off-by: Juan Cruz Viotti --- .github/workflows/website.yml | 2 +- contrib/website.sh | 97 +++++++++++++++++++---------------- 2 files changed, 53 insertions(+), 46 deletions(-) diff --git a/.github/workflows/website.yml b/.github/workflows/website.yml index 63ada7f487..8d92c33bcd 100644 --- a/.github/workflows/website.yml +++ b/.github/workflows/website.yml @@ -29,7 +29,7 @@ jobs: CACHE_KEY: ccache-v2-website-coverage steps: - name: Install dependencies - run: dnf install --assumeyes doxygen cmake gcc-c++ graphviz git clang llvm libstdc++-static diffutils util-linux + run: dnf install --assumeyes doxygen cmake gcc-c++ graphviz git clang llvm libstdc++-static diffutils util-linux python3 - uses: actions/checkout@v7 # The primary key never matches, as it carries the run identifier, so a # restore always walks the fallbacks in order. A previous run of this same diff --git a/contrib/website.sh b/contrib/website.sh index 96b8fa8be7..a3c9cdeb24 100755 --- a/contrib/website.sh +++ b/contrib/website.sh @@ -163,29 +163,10 @@ do done < "$OBJECT_LIST" # Merge the traces line by line, keeping the highest count observed for every -# line, branch, and function, then emit a merged LCOV trace plus a per file -# summary. A function is identified by where it is defined, so that every -# instantiation of a template counts towards the same one +# line and branch, then emit a merged LCOV trace plus a per file summary MERGE_PROGRAM="$WORK_DIRECTORY/merge.awk" cat > "$MERGE_PROGRAM" <<'AWK' /^SF:/ { source = substr($0, 4); files[source] = 1; next } -/^FN:/ { - split(substr($0, 4), record, ",") - function_lines[source SUBSEP substr($0, 5 + length(record[1]))] = record[1] - key = source SUBSEP record[1] - if (!(key in functions)) { - functions[key] = 0 - } - next -} -/^FNDA:/ { - split(substr($0, 6), record, ",") - key = source SUBSEP function_lines[source SUBSEP substr($0, 7 + length(record[1]))] - if (record[1] + 0 > functions[key] + 0) { - functions[key] = record[1] + 0 - } - next -} /^DA:/ { split(substr($0, 4), record, ",") key = source SUBSEP record[1] @@ -257,39 +238,65 @@ END { total_covered, total_lines printf "%8.2f%% %6d/%-6d TOTAL branches\n", branch_percentage, total_branches_covered, total_branches - total_functions = 0 - total_functions_covered = 0 - printf "" > uncovered - for (key in functions) { - total_functions += 1 - if (functions[key] > 0) { - total_functions_covered += 1 - } else { - split(key, parts, SUBSEP) - printf "%s:%s\n", parts[1], parts[2] > uncovered - } - } - close(uncovered) - function_percentage = total_functions > 0 \ - ? (total_functions_covered * 100.0) / total_functions : 100 - printf "%8.2f%% %6d/%-6d TOTAL functions\n", function_percentage, - total_functions_covered, total_functions } AWK -UNCOVERED_FUNCTIONS="$WORK_DIRECTORY/uncovered.txt" -awk -v "merged=$WORK_DIRECTORY/coverage.lcov" \ - -v "uncovered=$UNCOVERED_FUNCTIONS" -f "$MERGE_PROGRAM" \ +awk -v "merged=$WORK_DIRECTORY/coverage.lcov" -f "$MERGE_PROGRAM" \ "$LCOV_DIRECTORY"/*.lcov > "$WORK_DIRECTORY/summary.txt" +# Functions are merged apart from the traces, as a trace only records the line +# that a function starts on, which cannot tell apart two functions starting on +# the same line. The JSON export records the column as well, which is how the +# report itself counts every instantiation of a template as a single function, +# and the highest count across the binaries is kept for the same reason as above +FUNCTIONS_PROGRAM="$WORK_DIRECTORY/functions.py" +cat > "$FUNCTIONS_PROGRAM" <<'PYTHON' +import json +import re +import subprocess +import sys + +llvm_cov, profile_data, exclude, object_list, uncovered = sys.argv[1:] +excluded = re.compile(exclude) + +counts = {} +with open(object_list, encoding="utf-8") as objects: + for binary in objects.read().splitlines(): + export = subprocess.run( + [llvm_cov, "export", binary, f"-instr-profile={profile_data}", + "-format=text", "-skip-expansions", + f"-ignore-filename-regex={exclude}"], + check=True, stdout=subprocess.PIPE) + for data in json.loads(export.stdout)["data"]: + for function in data["functions"]: + filename = function["filenames"][0] + if excluded.search(filename): + continue + start = function["regions"][0] + key = (filename, start[0], start[1]) + counts[key] = max(counts.get(key, 0), function["count"]) + +missed = sorted(key for key, count in counts.items() if count == 0) +with open(uncovered, "w", encoding="utf-8") as output: + for filename, line, column in missed: + output.write(f"{filename}:{line}:{column}\n") + +covered = len(counts) - len(missed) +percentage = covered * 100 / len(counts) if counts else 100 +print(f"{percentage:8.2f}% {covered:6d}/{len(counts):<6d} TOTAL functions") +PYTHON + +UNCOVERED_FUNCTIONS="$WORK_DIRECTORY/uncovered.txt" +python3 "$FUNCTIONS_PROGRAM" "$LLVM_COV" "$PROFILE_DATA" "$EXCLUDE" \ + "$OBJECT_LIST" "$UNCOVERED_FUNCTIONS" >> "$WORK_DIRECTORY/summary.txt" + # Optionally require every function under measurement to be reached by the -# suite, judged over the merged traces for the reason given above. Only the -# platform that the report is published from is held to it, as a report -# produced elsewhere measures a different set of code +# suite. Only the platform that the report is published from is held to it, as +# a report produced elsewhere measures a different set of code if [ -n "${REQUIRE_FULL_FUNCTION_COVERAGE:-}" ] && [ -s "$UNCOVERED_FUNCTIONS" ] then - echo "The test suite never calls the functions defined at:" >&2 - sort -t : -k 1,1 -k 2,2n "$UNCOVERED_FUNCTIONS" >&2 + echo "The test suite never calls the functions starting at:" >&2 + cat "$UNCOVERED_FUNCTIONS" >&2 exit 1 fi