diff --git a/docs/vm.md b/docs/vm.md index 889ab8d8..21563558 100644 --- a/docs/vm.md +++ b/docs/vm.md @@ -87,7 +87,7 @@ With `cgroup_cpus=0-14`, the reserved core 15 has no VM competition and acts as - **Hugepages** (Cloud Hypervisor only): opt-in via `vm create --hugepages`; VM memory is backed by 2 MiB hugepages for reduced TLB pressure, and in exchange snapshots of that VM restore via eager copy only (the mmap fast path needs plain private-anon memory). Firecracker rejects `--hugepages`: FC cannot restore a hugetlbfs-backed snapshot, which would break hibernate/clone - **Mergeable memory / KSM** (Cloud Hypervisor only): opt-in via `--mergeable` at golden creation; guest memory is madvised `MADV_MERGEABLE` so host KSM can dedup identical pages across VMs — the flag persists through snapshot/clone/restore (it lives in the snapshot's CH config, not the CLI), so build the golden with it or rebuild. cocoon only sets the madvise: enabling and tuning the scanner (`/sys/kernel/mm/ksm/run`, `pages_to_scan`) is the operator's. Excludes `--hugepages`/`--shared-memory` (KSM merges only plain private pages); mmap-cloned siblings already share untouched pages via the page cache, so KSM's gain is dirtied-but-equal and cross-golden pages — measure density on your fleet, and weigh ksmd CPU plus the cross-VM dedup timing side channel in multi-tenant setups -- **Disk I/O**: multi-queue virtio-blk; readonly base disks keep host page cache (`direct=off`), while writable raw/qcow2 COW disks use O_DIRECT (`direct=on`) to avoid host cache buildup and guest flush storms +- **Disk I/O**: multi-queue virtio-blk; readonly base disks keep host page cache (`direct=off`), writable raw COW and data disks use O_DIRECT (`direct=on`) to avoid host cache buildup and guest flush storms, and qcow2 overlays stay buffered — Cloud Hypervisor applies the disk's `direct` flag to the backing file too, and O_DIRECT there would give every VM its own read of the shared base instead of one page-cache copy - **Balloon**: 25% of memory auto-returned via virtio-balloon with deflate-on-OOM and free-page reporting (VMs with < 256 MiB memory skip balloon) - **Watchdog**: hardware watchdog enabled by default for automatic guest reset on hang diff --git a/hypervisor/cloudhypervisor/args.go b/hypervisor/cloudhypervisor/args.go index 78366822..09a0c0e1 100644 --- a/hypervisor/cloudhypervisor/args.go +++ b/hypervisor/cloudhypervisor/args.go @@ -178,10 +178,18 @@ func serialConsoleFor(directBoot bool, consoleSock string) (serial, console *chR return &chRuntimeFile{Mode: "Socket", Socket: consoleSock}, &chRuntimeFile{Mode: "Off"} } +func qcow2Overlay(sc *types.StorageConfig) bool { + return !sc.RO && filepath.Ext(sc.Path) == ".qcow2" +} + func effectiveDirectIO(sc *types.StorageConfig, noDirectIO bool) bool { if sc.DirectIO != nil { return *sc.DirectIO } + // CH applies this flag to the backing file too, so O_DIRECT would stop one page-cache copy of the shared base serving every VM. + if qcow2Overlay(sc) { + return false + } return !sc.RO && !noDirectIO } @@ -199,7 +207,7 @@ func storageConfigToDisk(storageConfig *types.StorageConfig, cpuCount, diskQueue switch { case filepath.Ext(storageConfig.Path) == ".qcow2": d.ImageType = "Qcow2" - d.BackingFiles = !storageConfig.RO + d.BackingFiles = qcow2Overlay(storageConfig) case storageConfig.RO: d.ImageType = "Raw" default: diff --git a/hypervisor/cloudhypervisor/args_test.go b/hypervisor/cloudhypervisor/args_test.go index 8283adbd..5b1d6ed1 100644 --- a/hypervisor/cloudhypervisor/args_test.go +++ b/hypervisor/cloudhypervisor/args_test.go @@ -30,3 +30,39 @@ func TestMemoryCLIArg(t *testing.T) { }) } } + +func TestEffectiveDirectIO(t *testing.T) { + tests := []struct { + name string + sc types.StorageConfig + noDirectIO bool + want bool + }{ + {name: "raw cow", sc: types.StorageConfig{Path: "/v/cow.raw", Role: types.StorageRoleCOW}, want: true}, + {name: "raw cow with no-direct-io", sc: types.StorageConfig{Path: "/v/cow.raw", Role: types.StorageRoleCOW}, noDirectIO: true}, + {name: "readonly layer", sc: types.StorageConfig{Path: "/v/base.raw", RO: true, Role: types.StorageRoleLayer}}, + {name: "qcow2 overlay stays buffered", sc: types.StorageConfig{Path: "/v/overlay.qcow2", Role: types.StorageRoleCOW}}, + {name: "readonly qcow2 has no backing chain", sc: types.StorageConfig{Path: "/v/base.qcow2", RO: true, Role: types.StorageRoleLayer}}, + {name: "explicit override wins", sc: types.StorageConfig{Path: "/v/data.raw", Role: types.StorageRoleData, DirectIO: ptr(false)}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := effectiveDirectIO(&tt.sc, tt.noDirectIO); got != tt.want { + t.Errorf("got %v, want %v", got, tt.want) + } + }) + } +} + +func TestQcow2OverlayDiskArgs(t *testing.T) { + sc := &types.StorageConfig{Path: "/v/overlay.qcow2", Role: types.StorageRoleCOW} + got := diskToCLIArg(storageConfigToDisk(sc, 1, 0, false, nil)) + if strings.Contains(got, "direct=on") { + t.Errorf("qcow2 overlay must stay buffered so the shared base keeps one page-cache copy: %s", got) + } + if !strings.Contains(got, "backing_files=on") { + t.Errorf("qcow2 overlay must keep backing_files=on: %s", got) + } +} + +func ptr[T any](v T) *T { return &v } diff --git a/hypervisor/teardown.go b/hypervisor/teardown.go index 083a1117..3b7b6179 100644 --- a/hypervisor/teardown.go +++ b/hypervisor/teardown.go @@ -102,21 +102,18 @@ func (b *Backend) recoverVMTombstone(ctx context.Context, id string) (done bool, var ( rec *tombstone.Record leaseID string + cl vmCleanup ) if err := b.update(ctx, func(t *vmTx) error { var err error - rec, leaseID, err = ts.Resume(ctx, t.w, id) + rec, leaseID, err = ts.Recover(ctx, t.w, id, &cl) return err }); err != nil { return false, err } - if rec == nil || rec.Phase == tombstone.PhaseLeased { + if rec == nil { return false, nil } - var cl vmCleanup - if err := json.Unmarshal(rec.Payload.Cleanup, &cl); err != nil { - return false, fmt.Errorf("tombstone %s payload: %w", id, err) - } if err := b.finishVMTeardown(ctx, id, leaseID, cl); err != nil { return false, err } diff --git a/meta/tombstone/tombstone.go b/meta/tombstone/tombstone.go index 9559ef37..9009b6ec 100644 --- a/meta/tombstone/tombstone.go +++ b/meta/tombstone/tombstone.go @@ -164,6 +164,18 @@ func (t *Table) Acquire(ctx context.Context, w meta.Writer, id string, build fun return leaseID, nil, err } +// Recover is the shared recover skeleton: Resume, then decode a deleting-phase payload into cl. A nil Record means nothing to roll forward (no tombstone, or leased — rolled back in place). +func (t *Table) Recover(ctx context.Context, w meta.Writer, id string, cl any) (*Record, string, error) { + rec, leaseID, err := t.Resume(ctx, w, id) + if err != nil || rec == nil || rec.Phase == PhaseLeased { + return nil, "", err + } + if err := json.Unmarshal(rec.Payload.Cleanup, cl); err != nil { + return nil, "", fmt.Errorf("tombstone %s payload: %w", id, err) + } + return rec, leaseID, nil +} + // Resume takes over id's tombstone for recovery under the held entity lock: a leased entry rolls back in place; a deleting one gets a fresh lease for the caller to roll forward. func (t *Table) Resume(ctx context.Context, w meta.Writer, id string) (rec *Record, leaseID string, err error) { rec, err = t.Get(ctx, w, id) diff --git a/network/cni/teardown.go b/network/cni/teardown.go index a68b9434..9f9614d6 100644 --- a/network/cni/teardown.go +++ b/network/cni/teardown.go @@ -131,21 +131,18 @@ func (c *CNI) recoverTombstone(ctx context.Context, vmID string) (rolledForward var ( rec *tombstone.Record leaseID string + cl netCleanup ) if err := c.update(ctx, func(t *netTx) error { var err error - rec, leaseID, err = ts.Resume(ctx, t.Writer(), vmID) + rec, leaseID, err = ts.Recover(ctx, t.Writer(), vmID, &cl) return err }); err != nil { return false, err } - if rec == nil || rec.Phase == tombstone.PhaseLeased { + if rec == nil { return false, nil } - var cl netCleanup - if err := json.Unmarshal(rec.Payload.Cleanup, &cl); err != nil { - return false, fmt.Errorf("tombstone %s payload: %w", vmID, err) - } // Subset teardown (vm net remove) creates its TAPs independently of the netns lifetime, so recovery restores Remove's deleteTAP; an aggregate's TAPs die with the netns. deleteTAP := rec.Payload.Mode == tombstone.ModeSubset if err := c.finishTeardown(ctx, vmID, leaseID, rec.Payload.Mode, cl, deleteTAP); err != nil { diff --git a/snapshot/localfile/teardown.go b/snapshot/localfile/teardown.go index 79129e32..fff2ece3 100644 --- a/snapshot/localfile/teardown.go +++ b/snapshot/localfile/teardown.go @@ -119,21 +119,18 @@ func (lf *LocalFile) recoverSnapTombstoneLocked(ctx context.Context, id string) var ( rec *tombstone.Record leaseID string + cl snapCleanup ) if err := lf.update(ctx, func(t *snapTx) error { var err error - rec, leaseID, err = ts.Resume(ctx, t.Writer(), id) + rec, leaseID, err = ts.Recover(ctx, t.Writer(), id, &cl) return err }); err != nil { return err } - if rec == nil || rec.Phase == tombstone.PhaseLeased { + if rec == nil { return nil } - var cl snapCleanup - if err := json.Unmarshal(rec.Payload.Cleanup, &cl); err != nil { - return fmt.Errorf("tombstone %s payload: %w", id, err) - } if err := lf.finishSnapTeardown(ctx, id, leaseID, cl); err != nil { return err }