From 9aedafb1bd627ffa236ea87e3619a20208c2f5ea Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Fri, 25 Sep 2026 03:02:49 +0800 Subject: [PATCH 1/2] ci(coverage): compact profiles with bounded memory --- .github/workflows/coverage-ut.yaml | 35 +--- scripts/compact_coverage_profile.go | 212 +++++++++++++++++++++++ scripts/compact_coverage_profile_test.go | 128 ++++++++++++++ 3 files changed, 345 insertions(+), 30 deletions(-) create mode 100644 scripts/compact_coverage_profile.go create mode 100644 scripts/compact_coverage_profile_test.go diff --git a/.github/workflows/coverage-ut.yaml b/.github/workflows/coverage-ut.yaml index 733b2ab..3ae558d 100644 --- a/.github/workflows/coverage-ut.yaml +++ b/.github/workflows/coverage-ut.yaml @@ -185,6 +185,7 @@ jobs: sparse-checkout: | scripts/prepare_coverage_cgo.sh scripts/run_coverage_ut.py + scripts/compact_coverage_profile.go scripts/summarize_coverage_ut.py sparse-checkout-cone-mode: false path: .ci-workflow @@ -514,41 +515,15 @@ jobs: test -s "${coverage_report}" test -s "${coverage_profile}" phase_start compact_profile - compact_profile() { - test -s "${coverage_profile}" || return 1 - raw_coverage_bytes=$(wc -c < "${coverage_profile}") - # -coverpkg emits the same instrumented block once per tested - # package. Collapse those duplicate records before crossing the - # artifact boundary; the coverage gate only needs hit/not-hit - # semantics and parse_coverage.py applies the same max(hit) merge. - LC_ALL=C awk ' - NR == 1 { next } - $1 ~ /pkg\/pb|pkg\/sql\/parsers\/goyacc|yaccpar/ { next } - NF >= 3 { - key = $1 " " $2 - hit = ($3 + 0 > 0) ? 1 : 0 - if (!(key in coverage) || hit > coverage[key]) { - coverage[key] = hit - } - } - END { - print "mode: set" - for (key in coverage) { - print key, coverage[key] - } - } - ' "${coverage_profile}" > "${coverage_profile}.compacted" || return 1 - test "$(wc -l < "${coverage_profile}.compacted")" -gt 1 || return 1 - mv "${coverage_profile}.compacted" "${coverage_profile}" || return 1 - compacted_coverage_bytes=$(wc -c < "${coverage_profile}") - } + # -coverpkg repeats instrumented blocks for every test package. The + # Go helper ORs exact block hits with memory proportional to unique + # blocks, then atomically replaces the raw profile after success. set +e - compact_profile + go run "$GITHUB_WORKSPACE/.ci-workflow/scripts/compact_coverage_profile.go" "${coverage_profile}" phase_status=$? set -e phase_end "${phase_status}" (( phase_status == 0 )) || exit "${phase_status}" - echo "UT coverage profile compacted from ${raw_coverage_bytes} to ${compacted_coverage_bytes} bytes" - name: Print failed coverage UT packages and tests if: ${{ always() && !cancelled() && steps.coverage_ut.outcome == 'failure' }} timeout-minutes: 5 diff --git a/scripts/compact_coverage_profile.go b/scripts/compact_coverage_profile.go new file mode 100644 index 0000000..b1df6a6 --- /dev/null +++ b/scripts/compact_coverage_profile.go @@ -0,0 +1,212 @@ +// Command compact_coverage_profile merges duplicate mode:set coverage records. +// Go's all-package -coverpkg profile contains one copy of each instrumented +// block per tested package. Keep only each block's logical OR before upload. +package main + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "time" +) + +const maxCoverageLine = 1 << 20 + +var excludedPaths = [][]byte{ + []byte("pkg/pb"), + []byte("pkg/sql/parsers/goyacc"), + []byte("yaccpar"), +} + +// compact retains one key per distinct (source range, statement count), not +// one key per input row. The common duplicate lookup does not retain a copy of +// the reader's buffer. +func compact(in io.Reader, out io.Writer) (int, error) { + reader := bufio.NewReaderSize(in, maxCoverageLine) + header, err := reader.ReadSlice('\n') + if err != nil || !bytes.Equal(header, []byte("mode: set\n")) { + return 0, fmt.Errorf("invalid mode:set coverage header: %w", errOrInvalid(err)) + } + + hits := make(map[string]bool, 65536) + lineNumber := 1 + for { + line, readErr := reader.ReadSlice('\n') + if errors.Is(readErr, io.EOF) && len(line) == 0 { + break + } + if readErr != nil { + return 0, fmt.Errorf("coverage line %d is incomplete or exceeds %d bytes: %w", lineNumber+1, maxCoverageLine, readErr) + } + lineNumber++ + line = line[:len(line)-1] + firstSpace := bytes.IndexByte(line, ' ') + if firstSpace <= 0 { + return 0, fmt.Errorf("malformed coverage line %d", lineNumber) + } + secondRel := bytes.IndexByte(line[firstSpace+1:], ' ') + if secondRel <= 0 { + return 0, fmt.Errorf("malformed coverage line %d", lineNumber) + } + secondSpace := firstSpace + 1 + secondRel + if !decimal(line[firstSpace+1 : secondSpace]) { + return 0, fmt.Errorf("invalid statement count on coverage line %d", lineNumber) + } + count, valid := positiveDecimal(line[secondSpace+1:]) + if !valid { + return 0, fmt.Errorf("invalid hit count on coverage line %d", lineNumber) + } + + path := line[:firstSpace] + excluded := false + for _, fragment := range excludedPaths { + if bytes.Contains(path, fragment) { + excluded = true + break + } + } + if excluded { + continue + } + keyBytes := line[:secondSpace] + if previous, exists := hits[string(keyBytes)]; exists { + if count && !previous { + hits[string(keyBytes)] = true + } + continue + } + hits[string(keyBytes)] = count + } + if len(hits) == 0 { + return 0, errors.New("empty compacted coverage profile") + } + + keys := make([]string, 0, len(hits)) + for key := range hits { + keys = append(keys, key) + } + sort.Strings(keys) + writer := bufio.NewWriterSize(out, 1<<20) + if _, err := writer.WriteString("mode: set\n"); err != nil { + return 0, err + } + for _, key := range keys { + if _, err := writer.WriteString(key); err != nil { + return 0, err + } + if hits[key] { + if _, err := writer.WriteString(" 1\n"); err != nil { + return 0, err + } + } else { + if _, err := writer.WriteString(" 0\n"); err != nil { + return 0, err + } + } + } + if err := writer.Flush(); err != nil { + return 0, err + } + return len(hits), nil +} + +func decimal(value []byte) bool { + if len(value) == 0 { + return false + } + for _, digit := range value { + if digit < '0' || digit > '9' { + return false + } + } + return true +} + +func positiveDecimal(value []byte) (bool, bool) { + if !decimal(value) { + return false, false + } + for _, digit := range value { + if digit != '0' { + return true, true + } + } + return false, true +} + +func errOrInvalid(err error) error { + if err != nil { + return err + } + return errors.New("unexpected header") +} + +func compactFile(path string) error { + started := time.Now() + input, err := os.Open(path) + if err != nil { + return err + } + defer func() { + if input != nil { + _ = input.Close() + } + }() + inputInfo, err := input.Stat() + if err != nil { + return err + } + if !inputInfo.Mode().IsRegular() { + return fmt.Errorf("coverage input is not a regular file: %s", path) + } + + temporary, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".compacted.*") + if err != nil { + return err + } + temporaryPath := temporary.Name() + defer os.Remove(temporaryPath) + defer func() { + if temporary != nil { + _ = temporary.Close() + } + }() + blocks, err := compact(input, temporary) + if err != nil { + return err + } + if err := input.Close(); err != nil { + return err + } + input = nil + if err := temporary.Close(); err != nil { + return err + } + temporary = nil + outputInfo, err := os.Stat(temporaryPath) + if err != nil { + return err + } + if err := os.Rename(temporaryPath, path); err != nil { + return err + } + fmt.Printf("UT coverage profile compacted from %d to %d bytes; blocks=%d elapsed_seconds=%.1f\n", + inputInfo.Size(), outputInfo.Size(), blocks, time.Since(started).Seconds()) + return nil +} + +func main() { + if len(os.Args) != 2 { + fmt.Fprintln(os.Stderr, "usage: compact_coverage_profile.go COVERAGE_PROFILE") + os.Exit(2) + } + if err := compactFile(os.Args[1]); err != nil { + fmt.Fprintf(os.Stderr, "coverage profile compaction failed: %v\n", err) + os.Exit(1) + } +} diff --git a/scripts/compact_coverage_profile_test.go b/scripts/compact_coverage_profile_test.go new file mode 100644 index 0000000..d059d7d --- /dev/null +++ b/scripts/compact_coverage_profile_test.go @@ -0,0 +1,128 @@ +package main + +import ( + "bytes" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" +) + +const coverageHeader = "mode: set\n" + +func TestCompactUnionsExactBlocksAndExclusions(t *testing.T) { + input := coverageHeader + + "example.com/a.go:1.1,1.2 1 0\n" + + "example.com/b.go:2.1,2.2 2 1\n" + + "example.com/a.go:1.1,1.2 1 1\n" + + "example.com/a.go:1.1,1.2 2 0\n" + + "example.com/b.go:2.1,2.2 2 0\n" + + "example.com/pkg/pb/p.go:1.1,1.2 1 1\n" + + "example.com/pkg/sql/parsers/goyacc/g.go:1.1,1.2 1 1\n" + + "example.com/yaccparser/y.go:1.1,1.2 1 1\n" + var output bytes.Buffer + blocks, err := compact(strings.NewReader(input), &output) + if err != nil { + t.Fatal(err) + } + if blocks != 3 { + t.Fatalf("blocks=%d, want 3", blocks) + } + want := coverageHeader + + "example.com/a.go:1.1,1.2 1 1\n" + + "example.com/a.go:1.1,1.2 2 0\n" + + "example.com/b.go:2.1,2.2 2 1\n" + if output.String() != want { + t.Fatalf("output:\n%s\nwant:\n%s", output.String(), want) + } +} + +func TestCompactCrossesReaderBufferWithoutLosingRecords(t *testing.T) { + var input strings.Builder + input.WriteString(coverageHeader) + for i := 0; i < 40000; i++ { + input.WriteString("example.com/a.go:1.1,1.2 1 0\n") + } + input.WriteString("example.com/a.go:1.1,1.2 1 1\n") + var output bytes.Buffer + blocks, err := compact(strings.NewReader(input.String()), &output) + if err != nil || blocks != 1 { + t.Fatalf("blocks=%d error=%v", blocks, err) + } + if output.String() != coverageHeader+"example.com/a.go:1.1,1.2 1 1\n" { + t.Fatalf("last duplicate was lost: %q", output.String()) + } +} + +func TestCompactRejectsMalformedInputs(t *testing.T) { + cases := []string{ + "mode: count\nexample.com/a.go:1.1,1.2 1 1\n", + coverageHeader, + coverageHeader + "example.com/pkg/pb/a.go:1.1,1.2 1 1\n", + coverageHeader + "example.com/a.go:1.1,1.2 1 1", + coverageHeader + "example.com/a.go:1.1,1.2 1 X\n", + coverageHeader + "example.com/a.go:1.1,1.2 X 1\n", + coverageHeader + "example.com/a.go:1.1,1.2 1 1 extra\n", + coverageHeader + strings.Repeat("x", maxCoverageLine) + "\n", + } + for _, input := range cases { + var output bytes.Buffer + if _, err := compact(strings.NewReader(input), &output); err == nil { + t.Fatalf("accepted malformed input of length %d", len(input)) + } + } +} + +type failureReader struct{} + +func (failureReader) Read([]byte) (int, error) { return 0, errors.New("injected read failure") } + +type failureWriter struct{} + +func (failureWriter) Write([]byte) (int, error) { return 0, errors.New("injected write failure") } + +func TestCompactPropagatesReadAndWriteFailures(t *testing.T) { + input := io.MultiReader(strings.NewReader(coverageHeader), failureReader{}) + if _, err := compact(input, io.Discard); err == nil { + t.Fatal("read failure was lost") + } + input = strings.NewReader(coverageHeader + "example.com/a.go:1.1,1.2 1 1\n") + if _, err := compact(input, failureWriter{}); err == nil { + t.Fatal("write failure was lost") + } +} + +func TestCompactFileIsAtomic(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "coverage.out") + invalid := []byte(coverageHeader + "example.com/a.go:1.1,1.2 1 1") + if err := os.WriteFile(path, invalid, 0600); err != nil { + t.Fatal(err) + } + if err := compactFile(path); err == nil { + t.Fatal("accepted incomplete profile") + } + got, err := os.ReadFile(path) + if err != nil || !bytes.Equal(got, invalid) { + t.Fatalf("raw input was changed after failure: data=%q error=%v", got, err) + } + files, err := filepath.Glob(path + ".compacted.*") + if err != nil || len(files) != 0 { + t.Fatalf("temporary files after failure: %v, %v", files, err) + } + valid := []byte(coverageHeader + + "example.com/a.go:1.1,1.2 1 0\n" + + "example.com/a.go:1.1,1.2 1 1\n") + if err := os.WriteFile(path, valid, 0600); err != nil { + t.Fatal(err) + } + if err := compactFile(path); err != nil { + t.Fatal(err) + } + got, err = os.ReadFile(path) + if err != nil || string(got) != coverageHeader+"example.com/a.go:1.1,1.2 1 1\n" { + t.Fatalf("wrong compacted profile: data=%q error=%v", got, err) + } +} From 42d796704a001f6a1d5a5e5160dcc5e879acc3eb Mon Sep 17 00:00:00 2001 From: XuPeng-SH Date: Fri, 25 Sep 2026 04:03:15 +0800 Subject: [PATCH 2/2] ci: validate coverage compactor in workflow checks --- .github/workflows/check-action-file.yaml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-action-file.yaml b/.github/workflows/check-action-file.yaml index bce21bd..f76b18e 100644 --- a/.github/workflows/check-action-file.yaml +++ b/.github/workflows/check-action-file.yaml @@ -17,6 +17,8 @@ on: - 'scripts/select_coverage_artifacts.py' - 'scripts/test_select_coverage_artifacts.py' - 'scripts/test_merge_trigger_tke_subject.py' + - 'scripts/compact_coverage_profile.go' + - 'scripts/compact_coverage_profile_test.go' - 'scripts/*race_seed*.py' permissions: @@ -28,7 +30,7 @@ jobs: permissions: contents: read runs-on: arm64-mo-shanghai-4c8g - timeout-minutes: 10 + timeout-minutes: 20 steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: @@ -61,6 +63,24 @@ jobs: # Keep this gate focused on workflow/action semantics. The repository # has a separate backlog of legacy shellcheck and pyflakes findings. run: ${{ steps.install-actionlint.outputs.executable }} -shellcheck= -pyflakes= -color + - name: Set up Go for coverage helper checks + timeout-minutes: 5 + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version: '1.26.4' + cache: false + - name: Test and vet coverage profile compactor + timeout-minutes: 3 + env: + GOTOOLCHAIN: local + run: | + set -euo pipefail + go test -count=1 -timeout=60s \ + scripts/compact_coverage_profile.go \ + scripts/compact_coverage_profile_test.go + go vet \ + scripts/compact_coverage_profile.go \ + scripts/compact_coverage_profile_test.go - name: Test coverage artifact generation selection run: python3 scripts/test_select_coverage_artifacts.py -v - name: Test TKE merge subject identity contract