diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9722ba36..e6566ec2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,6 +4,9 @@ on: pull_request: branches: - main + merge_group: + types: + - checks_requested push: branches: - main @@ -35,8 +38,8 @@ jobs: env: EVENT_NAME: ${{ github.event_name }} EVENT_BEFORE: ${{ github.event.before }} - EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha }} - EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} run: bash scripts/detect-changes.sh quality: @@ -136,6 +139,10 @@ jobs: run: bash scripts/build.sh - name: Diagnose installation run: "build/bin/peeper${{ matrix.exe }} doctor --json" + - name: Test native object-cache publication + env: + CCACHE_DISABLE: "1" + run: go test -count=1 -run '^TestCompileObject' -v ./cmd - name: Run source fixtures env: PEEPER_BIN: ${{ github.workspace }}/build/bin/peeper${{ matrix.exe }} diff --git a/cmd/build.go b/cmd/build.go index 05283c50..c2dcec31 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -84,36 +84,10 @@ func buildExecutable(ctx *project.CompilerContext, entry *module.Module, outputP if module == nil { continue } - ir := strings.TrimSpace(module.LLVMIR) - if ir == "" { - return fmt.Errorf("empty LLVM IR for module %s", module.ID.ImportPath) - } - cachePath := objectCachePath(ctx, profile, ir) - if cachePath != "" { - if _, err := os.Stat(cachePath); err == nil { - objectPaths = append(objectPaths, cachePath) - continue - } else if !os.IsNotExist(err) { - return fmt.Errorf("inspect cached object for %s: %w", module.ID.ImportPath, err) - } - } - llPath := filepath.Join(artifactDir, fmt.Sprintf("mod_%d.ll", i)) - if err := os.WriteFile(llPath, []byte(ir), 0o644); err != nil { - return fmt.Errorf("write llvm ir: %w", err) - } - objectPath := filepath.Join(artifactDir, fmt.Sprintf("mod_%d.o", i)) - if err := runCompilerTool(profile.ClangPath, profile.ObjectArgs(llPath, objectPath, ctx.Config.IsDebugBuild), "compile LLVM module "+module.ID.ImportPath); err != nil { + objectPath, err := compileObject(ctx, profile, module, filepath.Join(artifactDir, fmt.Sprintf("mod_%d", i))) + if err != nil { return err } - if cachePath != "" { - if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil { - return fmt.Errorf("create object cache directory: %w", err) - } - if err := os.Rename(objectPath, cachePath); err != nil { - return fmt.Errorf("publish cached object: %w", err) - } - objectPath = cachePath - } objectPaths = append(objectPaths, objectPath) } if len(objectPaths) == 0 { @@ -144,6 +118,71 @@ func buildExecutable(ctx *project.CompilerContext, entry *module.Module, outputP return replacePath(stagedPath, outputPath) } +// compileObject owns one module's cache staging lifetime, ending before linking. +func compileObject(ctx *project.CompilerContext, profile toolchain.Profile, module *module.Module, objectBase string) (string, error) { + ir := strings.TrimSpace(module.LLVMIR) + if ir == "" { + return "", fmt.Errorf("empty LLVM IR for module %s", module.ID.ImportPath) + } + cachePath := objectCachePath(ctx, profile, ir) + if cachePath != "" { + if exists, err := inspectCachedObject(cachePath); err != nil { + return "", fmt.Errorf("inspect cached object for %s: %w", module.ID.ImportPath, err) + } else if exists { + return cachePath, nil + } + } + llPath := objectBase + ".ll" + if err := os.WriteFile(llPath, []byte(ir), 0o644); err != nil { + return "", fmt.Errorf("write llvm ir: %w", err) + } + objectPath := objectBase + ".o" + if cachePath != "" { + cacheDir := filepath.Dir(cachePath) + if err := os.MkdirAll(cacheDir, 0o755); err != nil { + return "", fmt.Errorf("create object cache directory: %w", err) + } + // Clang must finish on the destination filesystem before publication. + stageDir, err := os.MkdirTemp(cacheDir, "."+filepath.Base(cachePath)+"-stage-*") + if err != nil { + return "", fmt.Errorf("stage cached object for %s: %w", module.ID.ImportPath, err) + } + defer os.RemoveAll(stageDir) + objectPath = filepath.Join(stageDir, filepath.Base(cachePath)) + } + if err := runCompilerTool(profile.ClangPath, profile.ObjectArgs(llPath, objectPath, ctx.Config.IsDebugBuild), "compile LLVM module "+module.ID.ImportPath); err != nil { + return "", err + } + if cachePath != "" { + // A hard link installs the completed object without replacing a winner. + if linkErr := os.Link(objectPath, cachePath); linkErr != nil { + // Any failed link may race with another publisher. Reuse only a + // regular directory entry and preserve the publication error otherwise. + if exists, err := inspectCachedObject(cachePath); err != nil { + return "", fmt.Errorf("publish cached object: %w; inspect cache winner: %w", linkErr, err) + } else if !exists { + return "", fmt.Errorf("publish cached object: %w", linkErr) + } + } + return cachePath, nil + } + return objectPath, nil +} + +func inspectCachedObject(path string) (bool, error) { + info, err := os.Lstat(path) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + if !info.Mode().IsRegular() { + return false, fmt.Errorf("%s is not a regular file", path) + } + return true, nil +} + func objectCachePath(ctx *project.CompilerContext, profile toolchain.Profile, ir string) string { if ctx == nil || !profile.IsManaged || ctx.Config.RootDir == "" { return "" diff --git a/cmd/build_linux_test.go b/cmd/build_linux_test.go new file mode 100644 index 00000000..887d27bc --- /dev/null +++ b/cmd/build_linux_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "encoding/binary" + "errors" + "os" + "path/filepath" + "strings" + "syscall" + "testing" +) + +func TestCompileObjectNeverMovesPublishedCache(t *testing.T) { + ctx, profile, mod, base := objectCompileFixture(t, t.TempDir()) + cachePath := objectCachePath(ctx, profile, strings.TrimSpace(mod.LLVMIR)) + if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil { + t.Fatal(err) + } + watch, err := syscall.InotifyInit1(syscall.IN_CLOEXEC | syscall.IN_NONBLOCK) + if err != nil { + t.Fatal(err) + } + defer syscall.Close(watch) + if _, err := syscall.InotifyAddWatch(watch, filepath.Dir(cachePath), syscall.IN_MOVED_FROM|syscall.IN_DELETE); err != nil { + t.Fatal(err) + } + // The fixture publishes a winner after our cache miss, before our publication. + // Watching directory events catches even a gap too short for polling readers. + t.Setenv("PEEPER_TEST_OBJECT_MODE", "winner") + if _, err := compileObject(ctx, profile, mod, base); err != nil { + t.Fatal(err) + } + var events [65536]byte + n, err := syscall.Read(watch, events[:]) + if errors.Is(err, syscall.EAGAIN) { + return + } + if err != nil { + t.Fatal(err) + } + for offset := 0; offset < n; { + nameLen := int(binary.NativeEndian.Uint32(events[offset+12 : offset+16])) + name := strings.TrimRight(string(events[offset+16:offset+16+nameLen]), "\x00") + if name == filepath.Base(cachePath) { + t.Fatalf("published cache object was moved away or deleted: %s", name) + } + offset += syscall.SizeofInotifyEvent + nameLen + } +} diff --git a/cmd/build_test.go b/cmd/build_test.go index e8fb13b8..a407f42c 100644 --- a/cmd/build_test.go +++ b/cmd/build_test.go @@ -1,12 +1,111 @@ package main import ( + "errors" + "fmt" "os" + "os/exec" "path/filepath" "runtime" + "strings" + "sync" + "syscall" "testing" + "time" + + "compiler/internal/module" + "compiler/internal/moduleid" + "compiler/internal/project" + "compiler/internal/target" + "compiler/internal/toolchain" ) +// Run the test executable as a controlled compiler subprocess without shell or +// production hooks. Normal test invocations still run through testing.M. +func TestMain(m *testing.M) { + if os.Getenv("PEEPER_TEST_OBJECT_TOOL") == "1" && len(os.Args) > 1 && os.Args[1] == "-target" { + if err := objectCompilerFixture(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(23) + } + os.Exit(0) + } + os.Exit(m.Run()) +} + +func objectCompilerFixture() error { + inputPath, outputPath := os.Args[len(os.Args)-3], os.Args[len(os.Args)-1] + ir, err := os.ReadFile(inputPath) + if err != nil { + return err + } + if err := os.WriteFile(outputPath, []byte("partial"), 0o640); err != nil { + return err + } + if err := os.WriteFile(inputPath+".trace", []byte(outputPath), 0o600); err != nil { + return err + } + cachePath := os.Getenv("PEEPER_TEST_OBJECT_CACHE") + mode := os.Getenv("PEEPER_TEST_OBJECT_MODE") + switch mode { + case "compiler-failure": + return fmt.Errorf("fixture compiler failed after partial output") + case "wait": + if err := waitForObjectFile(inputPath + ".release"); err != nil { + return err + } + case "directory-collision": + if err := os.Mkdir(cachePath, 0o755); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(cachePath, "keep"), []byte("keep"), 0o600); err != nil { + return err + } + case "winner", "winner-missing-output": + // Another compiler publishes the same key after our initial cache miss. + winner := outputPath + ".winner" + if err := os.WriteFile(winner, append([]byte("object:"), ir...), 0o640); err != nil { + return err + } + if err := os.Rename(winner, cachePath); err != nil { + return err + } + } + if mode == "missing-output" || mode == "winner-missing-output" { + return os.Remove(outputPath) + } + return os.WriteFile(outputPath, append([]byte("object:"), ir...), 0o640) +} + +func waitForObjectFile(path string) error { + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(path); err == nil { + return nil + } else if !os.IsNotExist(err) { + return err + } + time.Sleep(time.Millisecond) + } + return fmt.Errorf("timed out waiting for %s", path) +} + +func objectCompileFixture(t *testing.T, root string) (*project.CompilerContext, toolchain.Profile, *module.Module, string) { + t.Helper() + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + host := target.Host() + ctx := &project.CompilerContext{Config: project.Config{RootDir: root, IsDebugBuild: true}} + profile := toolchain.Profile{IsManaged: true, ClangPath: executable, TargetOS: host.OS, TargetArch: host.Arch, LLVMTriple: host.LLVMTriple, DebugFormat: "dwarf"} + mod := &module.Module{ID: moduleid.ID{Origin: "local", ImportPath: "cache_test"}, LLVMIR: "\n define i32 @main() { ret i32 0 } \n"} + t.Setenv("PEEPER_TEST_OBJECT_TOOL", "1") + t.Setenv("PEEPER_TEST_OBJECT_MODE", "") + t.Setenv("PEEPER_TEST_OBJECT_CACHE", objectCachePath(ctx, profile, strings.TrimSpace(mod.LLVMIR))) + return ctx, profile, mod, filepath.Join(t.TempDir(), "mod_0") +} + func TestValidateNativeLinkTarget(t *testing.T) { if err := validateNativeLinkTarget(runtime.GOOS, runtime.GOARCH); err != nil { t.Fatalf("host target rejected: %v", err) @@ -45,3 +144,284 @@ func TestReplacePathReplacesExistingFile(t *testing.T) { t.Fatalf("staged path still exists: %v", err) } } + +func TestCompileObjectLifecycle(t *testing.T) { + for _, mode := range []string{"success", "unmanaged", "cache-hit", "compiler-failure", "missing-output", "directory-collision", "winner", "winner-missing-output"} { + t.Run(mode, func(t *testing.T) { + ctx, profile, mod, base := objectCompileFixture(t, t.TempDir()) + if mode == "unmanaged" { + profile.IsManaged = false + } + cachePath := objectCachePath(ctx, profile, strings.TrimSpace(mod.LLVMIR)) + want := "object:" + strings.TrimSpace(mod.LLVMIR) + t.Setenv("PEEPER_TEST_OBJECT_MODE", mode) + if mode == "cache-hit" { + if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(cachePath, []byte(want), 0o640); err != nil { + t.Fatal(err) + } + t.Setenv("PEEPER_TEST_OBJECT_MODE", "compiler-failure") + } + objectPath, err := compileObject(ctx, profile, mod, base) + switch mode { + case "compiler-failure": + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 23 || !strings.Contains(err.Error(), "compile LLVM module cache_test") || !strings.Contains(err.Error(), "fixture compiler failed") { + t.Fatalf("compiler error = %v, want wrapped exit 23 with module and tool output", err) + } + case "missing-output", "directory-collision": + var linkErr *os.LinkError + if !errors.As(err, &linkErr) || !strings.Contains(err.Error(), "publish cached object") { + t.Fatalf("publish error = %v, want wrapped LinkError", err) + } + if mode == "missing-output" && !errors.Is(err, os.ErrNotExist) { + t.Fatalf("publish error lost missing-file identity: %v", err) + } + default: + if err != nil { + t.Fatal(err) + } + wantPath := cachePath + if mode == "unmanaged" { + wantPath = base + ".o" + } + if objectPath != wantPath { + t.Fatalf("object path = %q, want %q", objectPath, wantPath) + } + data, err := os.ReadFile(objectPath) + if err != nil || string(data) != want { + t.Fatalf("object = %q, %v; want %q", data, err, want) + } + info, err := os.Stat(objectPath) + if err != nil { + t.Fatal(err) + } + if runtime.GOOS != "windows" && info.Mode().Perm() != 0o640 { + t.Fatalf("object mode = %o, want 640", info.Mode().Perm()) + } + } + if mode == "cache-hit" { + if _, err := os.Stat(base + ".ll"); !os.IsNotExist(err) { + t.Fatalf("cache hit wrote LLVM input: %v", err) + } + return + } + trace, err := os.ReadFile(base + ".ll.trace") + if err != nil { + t.Fatal(err) + } + if mode == "unmanaged" { + if string(trace) != base+".o" { + t.Fatalf("uncached compiler output = %q", trace) + } + return + } + stageDir := filepath.Dir(string(trace)) + if filepath.Dir(stageDir) != filepath.Dir(cachePath) { + t.Fatalf("stage %q not on cache filesystem beside %q", stageDir, cachePath) + } + if _, err := os.Stat(stageDir); !os.IsNotExist(err) { + t.Fatalf("compileObject left stage directory: %v", err) + } + entries, err := os.ReadDir(filepath.Dir(cachePath)) + if err != nil { + t.Fatal(err) + } + if mode == "compiler-failure" || mode == "missing-output" { + if len(entries) != 0 { + t.Fatalf("failed compilation/publication left cache entries: %v", entries) + } + } else if len(entries) != 1 || entries[0].Name() != filepath.Base(cachePath) { + t.Fatalf("unexpected cache entries: %v", entries) + } + if mode == "directory-collision" { + if data, err := os.ReadFile(filepath.Join(cachePath, "keep")); err != nil || string(data) != "keep" { + t.Fatalf("publish failure damaged destination: %q, %v", data, err) + } + } + }) + } +} + +func TestCompileObjectRejectsPreexistingDirectory(t *testing.T) { + ctx, profile, mod, base := objectCompileFixture(t, t.TempDir()) + cachePath := objectCachePath(ctx, profile, strings.TrimSpace(mod.LLVMIR)) + // Unlike directory-collision, this entry exists before the initial lookup. + if err := os.MkdirAll(cachePath, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(cachePath, "keep"), []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + + objectPath, err := compileObject(ctx, profile, mod, base) + if err == nil || objectPath != "" { + t.Fatalf("compileObject() = %q, %v; want empty path and invalid-cache error", objectPath, err) + } + for _, text := range []string{"inspect cached object for " + mod.ID.ImportPath, cachePath, "not a regular file"} { + if !strings.Contains(err.Error(), text) { + t.Errorf("cache error = %v, want %q", err, text) + } + } + if data, err := os.ReadFile(filepath.Join(cachePath, "keep")); err != nil || string(data) != "keep" { + t.Fatalf("cache inspection damaged destination: %q, %v", data, err) + } + entries, err := os.ReadDir(filepath.Dir(cachePath)) + if err != nil || len(entries) != 1 || entries[0].Name() != filepath.Base(cachePath) || !entries[0].IsDir() { + t.Fatalf("cache inspection changed cache entries: %v, %v", entries, err) + } + entries, err = os.ReadDir(filepath.Dir(base)) + if err != nil || len(entries) != 0 { + t.Fatalf("cache inspection wrote compiler artifacts: %v, %v", entries, err) + } +} + +func TestCompileObjectRejectsPreexistingSymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows CI does not guarantee symlink privileges") + } + ctx, profile, mod, base := objectCompileFixture(t, t.TempDir()) + cachePath := objectCachePath(ctx, profile, strings.TrimSpace(mod.LLVMIR)) + if err := os.MkdirAll(filepath.Dir(cachePath), 0o755); err != nil { + t.Fatal(err) + } + targetPath := filepath.Join(t.TempDir(), "object.o") + if err := os.WriteFile(targetPath, []byte("not cached"), 0o640); err != nil { + t.Fatal(err) + } + if err := os.Symlink(targetPath, cachePath); err != nil { + t.Fatal(err) + } + + objectPath, err := compileObject(ctx, profile, mod, base) + if err == nil || objectPath != "" { + t.Fatalf("compileObject() = %q, %v; want empty path and invalid-cache error", objectPath, err) + } + for _, text := range []string{"inspect cached object for " + mod.ID.ImportPath, cachePath, "not a regular file"} { + if !strings.Contains(err.Error(), text) { + t.Errorf("cache error = %v, want %q", err, text) + } + } + info, err := os.Lstat(cachePath) + if err != nil || info.Mode()&os.ModeSymlink == 0 { + t.Fatalf("cache inspection changed symlink: %v, %v", info, err) + } + if data, err := os.ReadFile(targetPath); err != nil || string(data) != "not cached" { + t.Fatalf("cache inspection changed symlink target: %q, %v", data, err) + } + entries, err := os.ReadDir(filepath.Dir(base)) + if err != nil || len(entries) != 0 { + t.Fatalf("cache inspection wrote compiler artifacts: %v, %v", entries, err) + } +} + +func TestCompileObjectConcurrentReaders(t *testing.T) { + ctx, profile, mod, base := objectCompileFixture(t, t.TempDir()) + t.Setenv("PEEPER_TEST_OBJECT_MODE", "wait") + cachePath := objectCachePath(ctx, profile, strings.TrimSpace(mod.LLVMIR)) + var compilers sync.WaitGroup + results := []chan error{make(chan error, 1), make(chan error, 1)} + for i := range results { + compilers.Add(1) + go func() { + defer compilers.Done() + _, err := compileObject(ctx, profile, mod, fmt.Sprintf("%s_%d", base, i)) + results[i] <- err + }() + } + t.Cleanup(func() { + for i := range results { + _ = os.WriteFile(fmt.Sprintf("%s_%d.ll.release", base, i), nil, 0o600) + } + compilers.Wait() + }) + for i := range results { + if err := waitForObjectFile(fmt.Sprintf("%s_%d.ll.trace", base, i)); err != nil { + t.Fatal(err) + } + } + if _, err := os.Stat(cachePath); !os.IsNotExist(err) { + t.Fatalf("partial compiler output visible in cache: %v", err) + } + if err := os.WriteFile(base+"_0.ll.release", nil, 0o600); err != nil { + t.Fatal(err) + } + if err := <-results[0]; err != nil { + t.Fatal(err) + } + publishedInfo, err := os.Lstat(cachePath) + if err != nil { + t.Fatal(err) + } + stopReader := make(chan struct{}) + readerDone := make(chan error, 1) + go func() { + for { + data, err := os.ReadFile(cachePath) + if err != nil || string(data) != "object:"+strings.TrimSpace(mod.LLVMIR) { + readerDone <- fmt.Errorf("cache reader observed %q, %v", data, err) + return + } + select { + case <-stopReader: + readerDone <- nil + return + default: + } + } + }() + t.Cleanup(func() { + close(stopReader) + if err := <-readerDone; err != nil { + t.Error(err) + } + }) + if err := os.WriteFile(base+"_1.ll.release", nil, 0o600); err != nil { + t.Fatal(err) + } + if err := <-results[1]; err != nil { + t.Fatal(err) + } + currentInfo, err := os.Lstat(cachePath) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(publishedInfo, currentInfo) { + t.Fatal("second publisher replaced the published cache object") + } + entries, err := os.ReadDir(filepath.Dir(cachePath)) + if err != nil || len(entries) != 1 || entries[0].Name() != filepath.Base(cachePath) || entries[0].IsDir() { + t.Fatalf("duplicate publisher left cache entries: %v, %v", entries, err) + } +} + +func TestCompileObjectAcrossFilesystems(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("cross-filesystem fixture requires Linux /dev/shm") + } + cacheRoot, err := os.MkdirTemp("/dev/shm", "peeper-object-cache-") + if err != nil { + t.Skipf("/dev/shm unavailable: %v", err) + } + defer os.RemoveAll(cacheRoot) + + ctx, profile, mod, base := objectCompileFixture(t, cacheRoot) + sourcePath := base + ".probe" + if err := os.WriteFile(sourcePath, []byte("object"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Rename(sourcePath, filepath.Join(cacheRoot, "probe")); err == nil { + t.Skip("test temporary directory and /dev/shm share a filesystem") + } else if !errors.Is(err, syscall.EXDEV) { + t.Fatalf("cross-filesystem probe = %v, want EXDEV", err) + } + objectPath, err := compileObject(ctx, profile, mod, base) + if err != nil { + t.Fatal(err) + } + if data, err := os.ReadFile(objectPath); err != nil || string(data) != "object:"+strings.TrimSpace(mod.LLVMIR) { + t.Fatalf("cached object = %q, %v", data, err) + } +} diff --git a/cmd/build_windows_test.go b/cmd/build_windows_test.go new file mode 100644 index 00000000..96763260 --- /dev/null +++ b/cmd/build_windows_test.go @@ -0,0 +1,67 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "syscall" + "testing" +) + +func TestCompileObjectReusesLockedWinner(t *testing.T) { + ctx, profile, mod, base := objectCompileFixture(t, t.TempDir()) + t.Setenv("PEEPER_TEST_OBJECT_MODE", "wait") + cachePath := objectCachePath(ctx, profile, strings.TrimSpace(mod.LLVMIR)) + result := make(chan error, 1) + done := make(chan struct{}) + go func() { + defer close(done) + _, err := compileObject(ctx, profile, mod, base) + result <- err + }() + t.Cleanup(func() { + _ = os.WriteFile(base+".ll.release", nil, 0o600) + <-done + }) + if err := waitForObjectFile(base + ".ll.trace"); err != nil { + t.Fatal(err) + } + // Model a linker holding a competing publisher's complete object without + // FILE_SHARE_DELETE. The probe proves the old replacement path is blocked. + want := "object:" + strings.TrimSpace(mod.LLVMIR) + if err := os.WriteFile(cachePath, []byte(want), 0o640); err != nil { + t.Fatal(err) + } + name, err := syscall.UTF16PtrFromString(cachePath) + if err != nil { + t.Fatal(err) + } + handle, err := syscall.CreateFile(name, syscall.GENERIC_READ, syscall.FILE_SHARE_READ, nil, syscall.OPEN_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0) + if err != nil { + t.Fatal(err) + } + defer syscall.CloseHandle(handle) + probe := filepath.Join(filepath.Dir(cachePath), "replacement-probe.o") + if err := os.WriteFile(probe, []byte(want), 0o640); err != nil { + t.Fatal(err) + } + if err := os.Rename(probe, cachePath); err == nil { + t.Fatal("replacement probe succeeded despite locked destination") + } + if err := os.Remove(probe); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(base+".ll.release", nil, 0o600); err != nil { + t.Fatal(err) + } + if err := <-result; err != nil { + t.Fatalf("failed to reuse locked winner: %v", err) + } + if data, err := os.ReadFile(cachePath); err != nil || string(data) != want { + t.Fatalf("winner changed: %q, %v", data, err) + } + entries, err := os.ReadDir(filepath.Dir(cachePath)) + if err != nil || len(entries) != 1 || entries[0].Name() != filepath.Base(cachePath) { + t.Fatalf("duplicate stage not cleaned: %v, %v", entries, err) + } +} diff --git a/scripts/detect-changes.sh b/scripts/detect-changes.sh index 660f9a66..e60d6aaf 100644 --- a/scripts/detect-changes.sh +++ b/scripts/detect-changes.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -if [ "${EVENT_NAME:-}" = pull_request ]; then +if [ "${EVENT_NAME:-}" = pull_request ] || [ "${EVENT_NAME:-}" = merge_group ]; then base="$EVENT_BASE_SHA" head="$EVENT_HEAD_SHA" elif [ "${EVENT_NAME:-}" = push ] && [ -n "${EVENT_BEFORE:-}" ] && [[ "$EVENT_BEFORE" != 0000000000000000000000000000000000000000 ]]; then diff --git a/x_test/cache_linux_test.go b/x_test/cache_linux_test.go new file mode 100644 index 00000000..10601a35 --- /dev/null +++ b/x_test/cache_linux_test.go @@ -0,0 +1,148 @@ +package xtest_test + +import ( + "encoding/json" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "testing" + + "compiler/internal/target" + "compiler/internal/toolchain" + "compiler/pkg/peeper" +) + +func TestManagedObjectCacheFixture(t *testing.T) { + binary := os.Getenv("PEEPER_BIN") + if binary == "" { + t.Skip("PEEPER_BIN not set; managed cache fixture requires bundled compiler") + } + clang, err := exec.LookPath("clang") + if err != nil { + t.Fatal(err) + } + tempRoot, err := os.MkdirTemp("/dev/shm", "peeper-link-fixture-") + if err != nil { + t.Skipf("/dev/shm unavailable: %v", err) + } + defer os.RemoveAll(tempRoot) + projectRoot := t.TempDir() + probe := filepath.Join(tempRoot, "probe.o") + if err := os.WriteFile(probe, nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.Rename(probe, filepath.Join(projectRoot, "probe.o")); err == nil { + t.Skip("test project and /dev/shm share a filesystem") + } else if !errors.Is(err, syscall.EXDEV) { + t.Fatalf("cross-device probe = %v, want EXDEV", err) + } + if err := os.Remove(probe); err != nil { + t.Fatal(err) + } + if err := os.CopyFS(projectRoot, os.DirFS("managed_object_cache")); err != nil { + t.Fatal(err) + } + installation := t.TempDir() + profileDir := filepath.Join(installation, "toolchains", "native") + for _, dir := range []string{filepath.Join(installation, "bin"), filepath.Join(profileDir, "link")} { + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + } + binary, err = filepath.Abs(binary) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(binary) + if err != nil { + t.Fatal(err) + } + installedBinary := filepath.Join(installation, "bin", "peeper") + if err := os.WriteFile(installedBinary, data, 0o755); err != nil { + t.Fatal(err) + } + for _, dir := range []string{"libs", "runtime"} { + if err := os.Symlink(filepath.Join(peeper.InstallationRootForExecutable(binary), dir), filepath.Join(installation, dir)); err != nil { + t.Fatal(err) + } + } + host := target.Host() + systemTriple, err := target.SystemLLVMTriple(host.OS, host.Arch) + if err != nil { + t.Fatal(err) + } + // Installed profiles require the canonical musl triple, but this cache-only + // fixture uses system Clang/libc. Select its host ABI so output does not depend + // on a musl loader being installed on the runner. The final --target wins. + // This tests real compilation, cross-filesystem caching, and execution with + // the system ABI; it does not validate the managed musl ABI/toolchain. + driver := "#!/bin/sh\nexec '" + strings.ReplaceAll(clang, "'", "'\"'\"'") + "' \"$@\" --target=" + systemTriple + "\n" + for _, tool := range []string{"clang", filepath.Join("link", "clang")} { + if err := os.WriteFile(filepath.Join(profileDir, tool), []byte(driver), 0o755); err != nil { + t.Fatal(err) + } + } + profile := toolchain.Profile{ + SchemaVersion: toolchain.ProfileSchemaVersion, ProfileID: "cache-fixture", + TargetOS: host.OS, TargetArch: host.Arch, LLVMTriple: host.LLVMTriple, + ClangPath: "toolchains/native/clang", LinkerPath: "toolchains/native/link/clang", + RuntimeArchive: "runtime/libpeeper_rt_v1.a", RuntimeABI: toolchain.RuntimeABIVersion, + LinkMode: "system", DebugFormat: "dwarf", + } + data, err = json.Marshal(profile) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(profileDir, "profile.json"), data, 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("TMPDIR", tempRoot) + t.Setenv("CCACHE_DISABLE", "1") + expectation := readFixtureExpectation(t, filepath.Join("managed_object_cache", "peeper.toml")) + expectation.Dir = projectRoot + runFixture(t, installedBinary, expectation) + objects, err := filepath.Glob(filepath.Join(projectRoot, "build", "*", "*", "artifacts", "*", "*.o")) + if err != nil || len(objects) == 0 { + t.Fatalf("managed build did not publish objects: %v, %v", objects, err) + } + // Keep the profile/key and linker identical, but make object compilation + // unavailable. A successful second build must consume published cache hits. + if err := os.Remove(filepath.Join(profileDir, "clang")); err != nil { + t.Fatal(err) + } + runFixture(t, installedBinary, expectation) + entries, err := os.ReadDir(tempRoot) + if err != nil || len(entries) != 0 { + t.Fatalf("link artifacts leaked: %v, %v", entries, err) + } + for _, object := range objects { + entries, err := os.ReadDir(filepath.Dir(object)) + if err != nil { + t.Fatal(err) + } + for _, entry := range entries { + if entry.IsDir() { + t.Fatalf("cache staging directory leaked: %s", entry.Name()) + } + } + } + + // A non-regular entry present before the next build must fail cache inspection, + // rather than reach the compiler/linker or be removed as part of cache repair. + if err := os.Remove(objects[0]); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(objects[0], 0o755); err != nil { + t.Fatal(err) + } + expectation.Mode = "build" + expectation.Outcome = "failure" + expectation.StderrContains = []string{"inspect cached object for", objects[0], "not a regular file"} + runFixture(t, installedBinary, expectation) + if info, err := os.Stat(objects[0]); err != nil || !info.IsDir() { + t.Fatalf("cache inspection changed invalid entry: %v, %v", info, err) + } +} diff --git a/x_test/managed_object_cache/peeper.toml b/x_test/managed_object_cache/peeper.toml new file mode 100644 index 00000000..8ee969a2 --- /dev/null +++ b/x_test/managed_object_cache/peeper.toml @@ -0,0 +1,6 @@ +name = "managed_object_cache" +build = "program" + +[test] +mode = "run" +outcome = "success" diff --git a/x_test/managed_object_cache/src/main.peep b/x_test/managed_object_cache/src/main.peep new file mode 100644 index 00000000..c9fccbf7 --- /dev/null +++ b/x_test/managed_object_cache/src/main.peep @@ -0,0 +1,8 @@ +fn Compute(value: i32) -> i32 { + return value * 3 + 5; +} + +fn main() -> i32 { + if Compute(7) != 26 { return 1; } + return 0; +}