vm: keep qcow2 overlays buffered so a shared base stays in the page cache

docs/vm.md

@@ -87,7 +87,7 @@ With `cgroup_cpus=0-14`, the reserved core 15 has no VM competition and acts as
 
 - **Hugepages** (Cloud Hypervisor only): opt-in via `vm create --hugepages`; VM memory is backed by 2 MiB hugepages for reduced TLB pressure, and in exchange snapshots of that VM restore via eager copy only (the mmap fast path needs plain private-anon memory). Firecracker rejects `--hugepages`: FC cannot restore a hugetlbfs-backed snapshot, which would break hibernate/clone
 - **Mergeable memory / KSM** (Cloud Hypervisor only): opt-in via `--mergeable` at golden creation; guest memory is madvised `MADV_MERGEABLE` so host KSM can dedup identical pages across VMs — the flag persists through snapshot/clone/restore (it lives in the snapshot's CH config, not the CLI), so build the golden with it or rebuild. cocoon only sets the madvise: enabling and tuning the scanner (`/sys/kernel/mm/ksm/run`, `pages_to_scan`) is the operator's. Excludes `--hugepages`/`--shared-memory` (KSM merges only plain private pages); mmap-cloned siblings already share untouched pages via the page cache, so KSM's gain is dirtied-but-equal and cross-golden pages — measure density on your fleet, and weigh ksmd CPU plus the cross-VM dedup timing side channel in multi-tenant setups
-- **Disk I/O**: multi-queue virtio-blk; readonly base disks keep host page cache (`direct=off`), while writable raw/qcow2 COW disks use O_DIRECT (`direct=on`) to avoid host cache buildup and guest flush storms
+- **Disk I/O**: multi-queue virtio-blk; readonly base disks keep host page cache (`direct=off`), writable raw COW and data disks use O_DIRECT (`direct=on`) to avoid host cache buildup and guest flush storms, and qcow2 overlays stay buffered — Cloud Hypervisor applies the disk's `direct` flag to the backing file too, and O_DIRECT there would give every VM its own read of the shared base instead of one page-cache copy
 - **Balloon**: 25% of memory auto-returned via virtio-balloon with deflate-on-OOM and free-page reporting (VMs with < 256 MiB memory skip balloon)
 - **Watchdog**: hardware watchdog enabled by default for automatic guest reset on hang
 

hypervisor/cloudhypervisor/args.go

@@ -178,10 +178,18 @@ func serialConsoleFor(directBoot bool, consoleSock string) (serial, console *chR
 	return &chRuntimeFile{Mode: "Socket", Socket: consoleSock}, &chRuntimeFile{Mode: "Off"}
 }
 
+func qcow2Overlay(sc *types.StorageConfig) bool {
+	return !sc.RO && filepath.Ext(sc.Path) == ".qcow2"
+}
+
 func effectiveDirectIO(sc *types.StorageConfig, noDirectIO bool) bool {
 	if sc.DirectIO != nil {
 		return *sc.DirectIO
 	}
+	// CH applies this flag to the backing file too, so O_DIRECT would stop one page-cache copy of the shared base serving every VM.
+	if qcow2Overlay(sc) {
+		return false
+	}
 	return !sc.RO && !noDirectIO
 }
 
@@ -199,7 +207,7 @@ func storageConfigToDisk(storageConfig *types.StorageConfig, cpuCount, diskQueue
 	switch {
 	case filepath.Ext(storageConfig.Path) == ".qcow2":
 		d.ImageType = "Qcow2"
-		d.BackingFiles = !storageConfig.RO
+		d.BackingFiles = qcow2Overlay(storageConfig)
 	case storageConfig.RO:
 		d.ImageType = "Raw"
 	default:

hypervisor/cloudhypervisor/args_test.go

@@ -30,3 +30,39 @@ func TestMemoryCLIArg(t *testing.T) {
 		})
 	}
 }
+
+func TestEffectiveDirectIO(t *testing.T) {
+	tests := []struct {
+		name       string
+		sc         types.StorageConfig
+		noDirectIO bool
+		want       bool
+	}{
+		{name: "raw cow", sc: types.StorageConfig{Path: "/v/cow.raw", Role: types.StorageRoleCOW}, want: true},
+		{name: "raw cow with no-direct-io", sc: types.StorageConfig{Path: "/v/cow.raw", Role: types.StorageRoleCOW}, noDirectIO: true},
+		{name: "readonly layer", sc: types.StorageConfig{Path: "/v/base.raw", RO: true, Role: types.StorageRoleLayer}},
+		{name: "qcow2 overlay stays buffered", sc: types.StorageConfig{Path: "/v/overlay.qcow2", Role: types.StorageRoleCOW}},
+		{name: "readonly qcow2 has no backing chain", sc: types.StorageConfig{Path: "/v/base.qcow2", RO: true, Role: types.StorageRoleLayer}},
+		{name: "explicit override wins", sc: types.StorageConfig{Path: "/v/data.raw", Role: types.StorageRoleData, DirectIO: ptr(false)}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := effectiveDirectIO(&tt.sc, tt.noDirectIO); got != tt.want {
+				t.Errorf("got %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func TestQcow2OverlayDiskArgs(t *testing.T) {
+	sc := &types.StorageConfig{Path: "/v/overlay.qcow2", Role: types.StorageRoleCOW}
+	got := diskToCLIArg(storageConfigToDisk(sc, 1, 0, false, nil))
+	if strings.Contains(got, "direct=on") {
+		t.Errorf("qcow2 overlay must stay buffered so the shared base keeps one page-cache copy: %s", got)
+	}
+	if !strings.Contains(got, "backing_files=on") {
+		t.Errorf("qcow2 overlay must keep backing_files=on: %s", got)
+	}
+}
+
+func ptr[T any](v T) *T { return &v }

hypervisor/teardown.go

@@ -102,21 +102,18 @@ func (b *Backend) recoverVMTombstone(ctx context.Context, id string) (done bool,
 	var (
 		rec     *tombstone.Record
 		leaseID string
+		cl      vmCleanup
 	)
 	if err := b.update(ctx, func(t *vmTx) error {
 		var err error
-		rec, leaseID, err = ts.Resume(ctx, t.w, id)
+		rec, leaseID, err = ts.Recover(ctx, t.w, id, &cl)
 		return err
 	}); err != nil {
 		return false, err
 	}
-	if rec == nil || rec.Phase == tombstone.PhaseLeased {
+	if rec == nil {
 		return false, nil
 	}
-	var cl vmCleanup
-	if err := json.Unmarshal(rec.Payload.Cleanup, &cl); err != nil {
-		return false, fmt.Errorf("tombstone %s payload: %w", id, err)
-	}
 	if err := b.finishVMTeardown(ctx, id, leaseID, cl); err != nil {
 		return false, err
 	}

meta/tombstone/tombstone.go

@@ -164,6 +164,18 @@ func (t *Table) Acquire(ctx context.Context, w meta.Writer, id string, build fun
 	return leaseID, nil, err
 }
 
+// Recover is the shared recover skeleton: Resume, then decode a deleting-phase payload into cl. A nil Record means nothing to roll forward (no tombstone, or leased — rolled back in place).
+func (t *Table) Recover(ctx context.Context, w meta.Writer, id string, cl any) (*Record, string, error) {
+	rec, leaseID, err := t.Resume(ctx, w, id)
+	if err != nil || rec == nil || rec.Phase == PhaseLeased {
+		return nil, "", err
+	}
+	if err := json.Unmarshal(rec.Payload.Cleanup, cl); err != nil {
+		return nil, "", fmt.Errorf("tombstone %s payload: %w", id, err)
+	}
+	return rec, leaseID, nil
+}
+
 // Resume takes over id's tombstone for recovery under the held entity lock: a leased entry rolls back in place; a deleting one gets a fresh lease for the caller to roll forward.
 func (t *Table) Resume(ctx context.Context, w meta.Writer, id string) (rec *Record, leaseID string, err error) {
 	rec, err = t.Get(ctx, w, id)

network/cni/teardown.go

@@ -131,21 +131,18 @@ func (c *CNI) recoverTombstone(ctx context.Context, vmID string) (rolledForward
 	var (
 		rec     *tombstone.Record
 		leaseID string
+		cl      netCleanup
 	)
 	if err := c.update(ctx, func(t *netTx) error {
 		var err error
-		rec, leaseID, err = ts.Resume(ctx, t.Writer(), vmID)
+		rec, leaseID, err = ts.Recover(ctx, t.Writer(), vmID, &cl)
 		return err
 	}); err != nil {
 		return false, err
 	}
-	if rec == nil || rec.Phase == tombstone.PhaseLeased {
+	if rec == nil {
 		return false, nil
 	}
-	var cl netCleanup
-	if err := json.Unmarshal(rec.Payload.Cleanup, &cl); err != nil {
-		return false, fmt.Errorf("tombstone %s payload: %w", vmID, err)
-	}
 	// Subset teardown (vm net remove) creates its TAPs independently of the netns lifetime, so recovery restores Remove's deleteTAP; an aggregate's TAPs die with the netns.
 	deleteTAP := rec.Payload.Mode == tombstone.ModeSubset
 	if err := c.finishTeardown(ctx, vmID, leaseID, rec.Payload.Mode, cl, deleteTAP); err != nil {

snapshot/localfile/teardown.go

@@ -119,21 +119,18 @@ func (lf *LocalFile) recoverSnapTombstoneLocked(ctx context.Context, id string)
 	var (
 		rec     *tombstone.Record
 		leaseID string
+		cl      snapCleanup
 	)
 	if err := lf.update(ctx, func(t *snapTx) error {
 		var err error
-		rec, leaseID, err = ts.Resume(ctx, t.Writer(), id)
+		rec, leaseID, err = ts.Recover(ctx, t.Writer(), id, &cl)
 		return err
 	}); err != nil {
 		return err
 	}
-	if rec == nil || rec.Phase == tombstone.PhaseLeased {
+	if rec == nil {
 		return nil
 	}
-	var cl snapCleanup
-	if err := json.Unmarshal(rec.Payload.Cleanup, &cl); err != nil {
-		return fmt.Errorf("tombstone %s payload: %w", id, err)
-	}
 	if err := lf.finishSnapTeardown(ctx, id, leaseID, cl); err != nil {
 		return err
 	}