From 56606fee66090c59022da7e06eb6b1d816506d85 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 12 Aug 2026 16:24:21 +0800 Subject: [PATCH 1/3] vm: keep qcow2 overlays buffered so a shared base stays in the page cache Cloud Hypervisor used to open a qcow2 backing file with buffered I/O no matter what the disk asked for, so the shared base image behind every cloudimg and Windows VM was served from one host page-cache copy. It now opens the backing file with the disk's own direct flag, and cocoon sets direct=on for every writable disk, so each VM started reading the base image straight from storage instead. Default direct=off for a qcow2 that layers over a base image. The overlay itself loses O_DIRECT, so its writes land in the host page cache as well as the guest one. That memory is reclaimable, and the residual cost is one extra copy per written byte, which is the cheaper half of this trade: without it every VM refetches the shared base from storage. Raw COW disks, read-only layers and data disks are unchanged, and an explicit direct_io on a data disk still wins. Expressing the split the old Cloud Hypervisor gave us by accident, a direct overlay above a buffered base, needs a per-backing-file cache policy that no released version has. That is requested upstream in cloud-hypervisor/cloud-hypervisor#8718. Measured on a 16-core host with a 700 MiB base image and 8 VMs reading 1 GiB each: base image resident in the page cache 310 MiB with the fix against 0 MiB without it, aggregate read 1.86s against 2.47-2.67s. --- hypervisor/cloudhypervisor/args.go | 10 ++++++- hypervisor/cloudhypervisor/args_test.go | 36 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/hypervisor/cloudhypervisor/args.go b/hypervisor/cloudhypervisor/args.go index 78366822..09a0c0e1 100644 --- a/hypervisor/cloudhypervisor/args.go +++ b/hypervisor/cloudhypervisor/args.go @@ -178,10 +178,18 @@ func serialConsoleFor(directBoot bool, consoleSock string) (serial, console *chR return &chRuntimeFile{Mode: "Socket", Socket: consoleSock}, &chRuntimeFile{Mode: "Off"} } +func qcow2Overlay(sc *types.StorageConfig) bool { + return !sc.RO && filepath.Ext(sc.Path) == ".qcow2" +} + func effectiveDirectIO(sc *types.StorageConfig, noDirectIO bool) bool { if sc.DirectIO != nil { return *sc.DirectIO } + // CH applies this flag to the backing file too, so O_DIRECT would stop one page-cache copy of the shared base serving every VM. + if qcow2Overlay(sc) { + return false + } return !sc.RO && !noDirectIO } @@ -199,7 +207,7 @@ func storageConfigToDisk(storageConfig *types.StorageConfig, cpuCount, diskQueue switch { case filepath.Ext(storageConfig.Path) == ".qcow2": d.ImageType = "Qcow2" - d.BackingFiles = !storageConfig.RO + d.BackingFiles = qcow2Overlay(storageConfig) case storageConfig.RO: d.ImageType = "Raw" default: diff --git a/hypervisor/cloudhypervisor/args_test.go b/hypervisor/cloudhypervisor/args_test.go index 8283adbd..5b1d6ed1 100644 --- a/hypervisor/cloudhypervisor/args_test.go +++ b/hypervisor/cloudhypervisor/args_test.go @@ -30,3 +30,39 @@ func TestMemoryCLIArg(t *testing.T) { }) } } + +func TestEffectiveDirectIO(t *testing.T) { + tests := []struct { + name string + sc types.StorageConfig + noDirectIO bool + want bool + }{ + {name: "raw cow", sc: types.StorageConfig{Path: "/v/cow.raw", Role: types.StorageRoleCOW}, want: true}, + {name: "raw cow with no-direct-io", sc: types.StorageConfig{Path: "/v/cow.raw", Role: types.StorageRoleCOW}, noDirectIO: true}, + {name: "readonly layer", sc: types.StorageConfig{Path: "/v/base.raw", RO: true, Role: types.StorageRoleLayer}}, + {name: "qcow2 overlay stays buffered", sc: types.StorageConfig{Path: "/v/overlay.qcow2", Role: types.StorageRoleCOW}}, + {name: "readonly qcow2 has no backing chain", sc: types.StorageConfig{Path: "/v/base.qcow2", RO: true, Role: types.StorageRoleLayer}}, + {name: "explicit override wins", sc: types.StorageConfig{Path: "/v/data.raw", Role: types.StorageRoleData, DirectIO: ptr(false)}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := effectiveDirectIO(&tt.sc, tt.noDirectIO); got != tt.want { + t.Errorf("got %v, want %v", got, tt.want) + } + }) + } +} + +func TestQcow2OverlayDiskArgs(t *testing.T) { + sc := &types.StorageConfig{Path: "/v/overlay.qcow2", Role: types.StorageRoleCOW} + got := diskToCLIArg(storageConfigToDisk(sc, 1, 0, false, nil)) + if strings.Contains(got, "direct=on") { + t.Errorf("qcow2 overlay must stay buffered so the shared base keeps one page-cache copy: %s", got) + } + if !strings.Contains(got, "backing_files=on") { + t.Errorf("qcow2 overlay must keep backing_files=on: %s", got) + } +} + +func ptr[T any](v T) *T { return &v } From cb5ac094b4ce9dd91e049c22c18f7b2d98971028 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 19 Aug 2026 15:57:51 +0800 Subject: [PATCH 2/3] review: hoist the tombstone recover skeleton into Table.Recover --- hypervisor/teardown.go | 9 +++------ meta/tombstone/tombstone.go | 12 ++++++++++++ network/cni/teardown.go | 9 +++------ snapshot/localfile/teardown.go | 9 +++------ 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/hypervisor/teardown.go b/hypervisor/teardown.go index 083a1117..3b7b6179 100644 --- a/hypervisor/teardown.go +++ b/hypervisor/teardown.go @@ -102,21 +102,18 @@ func (b *Backend) recoverVMTombstone(ctx context.Context, id string) (done bool, var ( rec *tombstone.Record leaseID string + cl vmCleanup ) if err := b.update(ctx, func(t *vmTx) error { var err error - rec, leaseID, err = ts.Resume(ctx, t.w, id) + rec, leaseID, err = ts.Recover(ctx, t.w, id, &cl) return err }); err != nil { return false, err } - if rec == nil || rec.Phase == tombstone.PhaseLeased { + if rec == nil { return false, nil } - var cl vmCleanup - if err := json.Unmarshal(rec.Payload.Cleanup, &cl); err != nil { - return false, fmt.Errorf("tombstone %s payload: %w", id, err) - } if err := b.finishVMTeardown(ctx, id, leaseID, cl); err != nil { return false, err } diff --git a/meta/tombstone/tombstone.go b/meta/tombstone/tombstone.go index 9559ef37..9009b6ec 100644 --- a/meta/tombstone/tombstone.go +++ b/meta/tombstone/tombstone.go @@ -164,6 +164,18 @@ func (t *Table) Acquire(ctx context.Context, w meta.Writer, id string, build fun return leaseID, nil, err } +// Recover is the shared recover skeleton: Resume, then decode a deleting-phase payload into cl. A nil Record means nothing to roll forward (no tombstone, or leased — rolled back in place). +func (t *Table) Recover(ctx context.Context, w meta.Writer, id string, cl any) (*Record, string, error) { + rec, leaseID, err := t.Resume(ctx, w, id) + if err != nil || rec == nil || rec.Phase == PhaseLeased { + return nil, "", err + } + if err := json.Unmarshal(rec.Payload.Cleanup, cl); err != nil { + return nil, "", fmt.Errorf("tombstone %s payload: %w", id, err) + } + return rec, leaseID, nil +} + // Resume takes over id's tombstone for recovery under the held entity lock: a leased entry rolls back in place; a deleting one gets a fresh lease for the caller to roll forward. func (t *Table) Resume(ctx context.Context, w meta.Writer, id string) (rec *Record, leaseID string, err error) { rec, err = t.Get(ctx, w, id) diff --git a/network/cni/teardown.go b/network/cni/teardown.go index a68b9434..9f9614d6 100644 --- a/network/cni/teardown.go +++ b/network/cni/teardown.go @@ -131,21 +131,18 @@ func (c *CNI) recoverTombstone(ctx context.Context, vmID string) (rolledForward var ( rec *tombstone.Record leaseID string + cl netCleanup ) if err := c.update(ctx, func(t *netTx) error { var err error - rec, leaseID, err = ts.Resume(ctx, t.Writer(), vmID) + rec, leaseID, err = ts.Recover(ctx, t.Writer(), vmID, &cl) return err }); err != nil { return false, err } - if rec == nil || rec.Phase == tombstone.PhaseLeased { + if rec == nil { return false, nil } - var cl netCleanup - if err := json.Unmarshal(rec.Payload.Cleanup, &cl); err != nil { - return false, fmt.Errorf("tombstone %s payload: %w", vmID, err) - } // Subset teardown (vm net remove) creates its TAPs independently of the netns lifetime, so recovery restores Remove's deleteTAP; an aggregate's TAPs die with the netns. deleteTAP := rec.Payload.Mode == tombstone.ModeSubset if err := c.finishTeardown(ctx, vmID, leaseID, rec.Payload.Mode, cl, deleteTAP); err != nil { diff --git a/snapshot/localfile/teardown.go b/snapshot/localfile/teardown.go index 79129e32..fff2ece3 100644 --- a/snapshot/localfile/teardown.go +++ b/snapshot/localfile/teardown.go @@ -119,21 +119,18 @@ func (lf *LocalFile) recoverSnapTombstoneLocked(ctx context.Context, id string) var ( rec *tombstone.Record leaseID string + cl snapCleanup ) if err := lf.update(ctx, func(t *snapTx) error { var err error - rec, leaseID, err = ts.Resume(ctx, t.Writer(), id) + rec, leaseID, err = ts.Recover(ctx, t.Writer(), id, &cl) return err }); err != nil { return err } - if rec == nil || rec.Phase == tombstone.PhaseLeased { + if rec == nil { return nil } - var cl snapCleanup - if err := json.Unmarshal(rec.Payload.Cleanup, &cl); err != nil { - return fmt.Errorf("tombstone %s payload: %w", id, err) - } if err := lf.finishSnapTeardown(ctx, id, leaseID, cl); err != nil { return err } From 5a2140f3e9c2eed1fdd26ee9c7cd694cb06cbcf6 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 19 Aug 2026 16:08:10 +0800 Subject: [PATCH 3/3] docs: qcow2 overlays stay buffered so the shared base keeps one page-cache copy --- docs/vm.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/vm.md b/docs/vm.md index 889ab8d8..21563558 100644 --- a/docs/vm.md +++ b/docs/vm.md @@ -87,7 +87,7 @@ With `cgroup_cpus=0-14`, the reserved core 15 has no VM competition and acts as - **Hugepages** (Cloud Hypervisor only): opt-in via `vm create --hugepages`; VM memory is backed by 2 MiB hugepages for reduced TLB pressure, and in exchange snapshots of that VM restore via eager copy only (the mmap fast path needs plain private-anon memory). Firecracker rejects `--hugepages`: FC cannot restore a hugetlbfs-backed snapshot, which would break hibernate/clone - **Mergeable memory / KSM** (Cloud Hypervisor only): opt-in via `--mergeable` at golden creation; guest memory is madvised `MADV_MERGEABLE` so host KSM can dedup identical pages across VMs — the flag persists through snapshot/clone/restore (it lives in the snapshot's CH config, not the CLI), so build the golden with it or rebuild. cocoon only sets the madvise: enabling and tuning the scanner (`/sys/kernel/mm/ksm/run`, `pages_to_scan`) is the operator's. Excludes `--hugepages`/`--shared-memory` (KSM merges only plain private pages); mmap-cloned siblings already share untouched pages via the page cache, so KSM's gain is dirtied-but-equal and cross-golden pages — measure density on your fleet, and weigh ksmd CPU plus the cross-VM dedup timing side channel in multi-tenant setups -- **Disk I/O**: multi-queue virtio-blk; readonly base disks keep host page cache (`direct=off`), while writable raw/qcow2 COW disks use O_DIRECT (`direct=on`) to avoid host cache buildup and guest flush storms +- **Disk I/O**: multi-queue virtio-blk; readonly base disks keep host page cache (`direct=off`), writable raw COW and data disks use O_DIRECT (`direct=on`) to avoid host cache buildup and guest flush storms, and qcow2 overlays stay buffered — Cloud Hypervisor applies the disk's `direct` flag to the backing file too, and O_DIRECT there would give every VM its own read of the shared base instead of one page-cache copy - **Balloon**: 25% of memory auto-returned via virtio-balloon with deflate-on-OOM and free-page reporting (VMs with < 256 MiB memory skip balloon) - **Watchdog**: hardware watchdog enabled by default for automatic guest reset on hang