From 51009e2f029192197eaffe4b6e6163d4a867c507 Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:48:09 +0000 Subject: [PATCH] Add experimental Cloud Hypervisor diff snapshots --- .../cloudhypervisor/cloudhypervisor.go | 26 +++ .../cloudhypervisor/diff_snapshot.go | 55 ++++++ .../cloudhypervisor/diff_snapshot_linux.go | 165 ++++++++++++++++++ .../diff_snapshot_linux_test.go | 52 ++++++ .../cloudhypervisor/diff_snapshot_test.go | 39 +++++ .../diff_snapshot_unsupported.go | 9 + lib/hypervisor/cloudhypervisor/process.go | 3 + lib/instances/firecracker_uffd_test.go | 6 +- lib/instances/snapshot_memfile.go | 24 +-- lib/instances/snapshot_memfile_test.go | 44 +++++ lib/instances/standby.go | 7 +- lib/vmm/vmm.go | 17 +- .../api-v0.3.0/cloud-hypervisor.yaml | 7 + 13 files changed, 434 insertions(+), 20 deletions(-) create mode 100644 lib/hypervisor/cloudhypervisor/diff_snapshot.go create mode 100644 lib/hypervisor/cloudhypervisor/diff_snapshot_linux.go create mode 100644 lib/hypervisor/cloudhypervisor/diff_snapshot_linux_test.go create mode 100644 lib/hypervisor/cloudhypervisor/diff_snapshot_test.go create mode 100644 lib/hypervisor/cloudhypervisor/diff_snapshot_unsupported.go diff --git a/lib/hypervisor/cloudhypervisor/cloudhypervisor.go b/lib/hypervisor/cloudhypervisor/cloudhypervisor.go index 1838fc1ec..75089e90b 100644 --- a/lib/hypervisor/cloudhypervisor/cloudhypervisor.go +++ b/lib/hypervisor/cloudhypervisor/cloudhypervisor.go @@ -72,6 +72,7 @@ func CapabilitiesForVersion(v vmm.CHVersion) hypervisor.Capabilities { case vmm.V51_1: caps.SupportsDiskResize = true caps.SupportsConcurrentForkPrepare = true + caps.SupportsSnapshotBaseReuse = experimentalDiffSnapshotsEnabled() } return caps } @@ -163,8 +164,20 @@ func (c *CloudHypervisor) Resume(ctx context.Context) error { // Snapshot creates a VM snapshot. func (c *CloudHypervisor) Snapshot(ctx context.Context, destPath string) error { + diff, err := prepareDiffSnapshotDestination(destPath) + if err != nil { + return fmt.Errorf("prepare diff snapshot destination: %w", err) + } + snapshotURL := "file://" + destPath snapshotConfig := vmm.VmSnapshotConfig{DestinationUrl: &snapshotURL} + if experimentalDiffSnapshotsEnabled() { + snapshotType := vmm.Full + if diff { + snapshotType = vmm.Diff + } + snapshotConfig.SnapshotType = &snapshotType + } resp, err := c.client.PutVmSnapshotWithResponse(ctx, snapshotConfig) if err != nil { return fmt.Errorf("snapshot: %w", err) @@ -172,6 +185,19 @@ func (c *CloudHypervisor) Snapshot(ctx context.Context, destPath string) error { if resp.StatusCode() != 204 { return fmt.Errorf("snapshot failed with status %d", resp.StatusCode()) } + if diff { + stats, err := mergeCloudHypervisorDiff(destPath) + if err != nil { + return fmt.Errorf("merge diff snapshot: %w", err) + } + logger.FromContext(ctx).InfoContext(ctx, "merged Cloud Hypervisor diff snapshot", + "snapshot_dir", destPath, + "delta_bytes", stats.DeltaBytes, + "extents", stats.ExtentCount, + "reflinked_bytes", stats.ReflinkedBytes, + "copied_bytes", stats.CopiedBytes, + ) + } optimized, err := prepareSnapshotForKernelPaging(destPath) if err != nil { return fmt.Errorf("prepare kernel-paged snapshot: %w", err) diff --git a/lib/hypervisor/cloudhypervisor/diff_snapshot.go b/lib/hypervisor/cloudhypervisor/diff_snapshot.go new file mode 100644 index 000000000..4bc407b70 --- /dev/null +++ b/lib/hypervisor/cloudhypervisor/diff_snapshot.go @@ -0,0 +1,55 @@ +package cloudhypervisor + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +const experimentalDiffSnapshotsEnv = "HYPEMAN_EXPERIMENTAL_CH_DIFF_SNAPSHOTS" + +const cloudHypervisorDiffMemoryFile = cloudHypervisorMemoryFile + ".diff" + +type diffMergeStats struct { + DeltaBytes int64 + ExtentCount int + ReflinkedBytes int64 + CopiedBytes int64 +} + +func experimentalDiffSnapshotsEnabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv(experimentalDiffSnapshotsEnv))) { + case "1", "true", "yes", "on": + return true + default: + return false + } +} + +// prepareDiffSnapshotDestination leaves the retained memory baseline in place +// and removes metadata that Cloud Hypervisor recreates with O_EXCL. +func prepareDiffSnapshotDestination(snapshotDir string) (bool, error) { + if !experimentalDiffSnapshotsEnabled() { + return false, nil + } + memoryPath := filepath.Join(snapshotDir, cloudHypervisorMemoryFile) + if _, err := os.Stat(memoryPath); err != nil { + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + return false, fmt.Errorf("stat retained snapshot memory: %w", err) + } + + for _, name := range []string{ + cloudHypervisorConfigFile, + cloudHypervisorStateFile, + cloudHypervisorDiffMemoryFile, + } { + if err := os.Remove(filepath.Join(snapshotDir, name)); err != nil && !errors.Is(err, os.ErrNotExist) { + return false, fmt.Errorf("remove retained snapshot file %s: %w", name, err) + } + } + return true, nil +} diff --git a/lib/hypervisor/cloudhypervisor/diff_snapshot_linux.go b/lib/hypervisor/cloudhypervisor/diff_snapshot_linux.go new file mode 100644 index 000000000..c16d41ed2 --- /dev/null +++ b/lib/hypervisor/cloudhypervisor/diff_snapshot_linux.go @@ -0,0 +1,165 @@ +//go:build linux + +package cloudhypervisor + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +func mergeCloudHypervisorDiff(snapshotDir string) (diffMergeStats, error) { + var stats diffMergeStats + basePath := filepath.Join(snapshotDir, cloudHypervisorMemoryFile) + diffPath := filepath.Join(snapshotDir, cloudHypervisorDiffMemoryFile) + + src, err := os.Open(diffPath) + if err != nil { + return stats, fmt.Errorf("open diff snapshot memory: %w", err) + } + defer src.Close() + // Resolve delayed allocation before cloning extents. FICLONERANGE can + // otherwise clone the pre-write extent state while dirty pages still live + // only in the source file's page cache. + if err := src.Sync(); err != nil { + return stats, fmt.Errorf("sync diff snapshot memory: %w", err) + } + dst, err := os.OpenFile(basePath, os.O_RDWR, 0) + if err != nil { + return stats, fmt.Errorf("open retained snapshot memory: %w", err) + } + defer dst.Close() + + srcInfo, err := src.Stat() + if err != nil { + return stats, fmt.Errorf("stat diff snapshot memory: %w", err) + } + dstInfo, err := dst.Stat() + if err != nil { + return stats, fmt.Errorf("stat retained snapshot memory: %w", err) + } + if srcInfo.Size() != dstInfo.Size() { + return stats, fmt.Errorf("diff snapshot memory size %d does not match baseline %d", srcInfo.Size(), dstInfo.Size()) + } + + srcFD := int(src.Fd()) + dstFD := int(dst.Fd()) + size := srcInfo.Size() + offset := int64(0) + cloneRanges := true + buf := make([]byte, 1<<20) + + for offset < size { + dataStart, err := unix.Seek(srcFD, offset, unix.SEEK_DATA) + if err != nil { + if errors.Is(err, unix.ENXIO) { + break + } + return stats, fmt.Errorf("seek diff data at %d: %w", offset, err) + } + dataEnd, err := unix.Seek(srcFD, dataStart, unix.SEEK_HOLE) + if err != nil { + if errors.Is(err, unix.ENXIO) { + dataEnd = size + } else { + return stats, fmt.Errorf("seek diff hole at %d: %w", dataStart, err) + } + } + if dataEnd > size { + dataEnd = size + } + if dataEnd <= dataStart { + return stats, fmt.Errorf("invalid diff extent [%d,%d)", dataStart, dataEnd) + } + + length := dataEnd - dataStart + stats.DeltaBytes += length + stats.ExtentCount++ + if cloneRanges { + clone := unix.FileCloneRange{ + Src_fd: int64(srcFD), + Src_offset: uint64(dataStart), + Src_length: uint64(length), + Dest_offset: uint64(dataStart), + } + if err := unix.IoctlFileCloneRange(dstFD, &clone); err == nil { + stats.ReflinkedBytes += length + offset = dataEnd + continue + } else if !diffRangeCloneUnsupported(err) { + return stats, fmt.Errorf("reflink diff extent [%d,%d): %w", dataStart, dataEnd, err) + } + cloneRanges = false + } + + if err := copyDiffExtent(srcFD, dstFD, dataStart, length, buf); err != nil { + return stats, fmt.Errorf("copy diff extent [%d,%d): %w", dataStart, dataEnd, err) + } + stats.CopiedBytes += length + offset = dataEnd + } + + if err := dst.Sync(); err != nil { + return stats, fmt.Errorf("sync merged snapshot memory: %w", err) + } + if err := os.Remove(diffPath); err != nil { + return stats, fmt.Errorf("remove merged diff snapshot: %w", err) + } + dir, err := os.Open(snapshotDir) + if err != nil { + return stats, fmt.Errorf("open snapshot directory: %w", err) + } + if err := dir.Sync(); err != nil { + _ = dir.Close() + return stats, fmt.Errorf("sync snapshot directory: %w", err) + } + if err := dir.Close(); err != nil { + return stats, fmt.Errorf("close snapshot directory: %w", err) + } + return stats, nil +} + +func copyDiffExtent(srcFD, dstFD int, offset, length int64, buf []byte) error { + position := offset + remaining := length + for remaining > 0 { + chunk := int64(len(buf)) + if remaining < chunk { + chunk = remaining + } + n, err := unix.Pread(srcFD, buf[:int(chunk)], position) + if err != nil { + return err + } + if n == 0 { + return io.ErrUnexpectedEOF + } + written := 0 + for written < n { + wn, err := unix.Pwrite(dstFD, buf[written:n], position+int64(written)) + if err != nil { + return err + } + if wn == 0 { + return io.ErrShortWrite + } + written += wn + } + position += int64(n) + remaining -= int64(n) + } + return nil +} + +func diffRangeCloneUnsupported(err error) bool { + return errors.Is(err, unix.EINVAL) || + errors.Is(err, unix.ENOTSUP) || + errors.Is(err, unix.EOPNOTSUPP) || + errors.Is(err, unix.EXDEV) || + errors.Is(err, unix.ENOTTY) || + errors.Is(err, unix.ETXTBSY) +} diff --git a/lib/hypervisor/cloudhypervisor/diff_snapshot_linux_test.go b/lib/hypervisor/cloudhypervisor/diff_snapshot_linux_test.go new file mode 100644 index 000000000..35655e6b8 --- /dev/null +++ b/lib/hypervisor/cloudhypervisor/diff_snapshot_linux_test.go @@ -0,0 +1,52 @@ +//go:build linux + +package cloudhypervisor + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestMergeCloudHypervisorDiff(t *testing.T) { + dir := t.TempDir() + const size = 4 << 20 + base := bytes.Repeat([]byte{0x7b}, size) + basePath := filepath.Join(dir, cloudHypervisorMemoryFile) + diffPath := filepath.Join(dir, cloudHypervisorDiffMemoryFile) + require.NoError(t, os.WriteFile(basePath, base, 0600)) + + diff, err := os.OpenFile(diffPath, os.O_CREATE|os.O_RDWR, 0600) + require.NoError(t, err) + require.NoError(t, diff.Truncate(size)) + changed := bytes.Repeat([]byte{0x2a}, 4096) + zeroed := make([]byte, 4096) + _, err = diff.WriteAt(changed, 64*4096) + require.NoError(t, err) + _, err = diff.WriteAt(zeroed, 512*4096) + require.NoError(t, err) + require.NoError(t, diff.Sync()) + require.NoError(t, diff.Close()) + + stats, err := mergeCloudHypervisorDiff(dir) + if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.ENOTSUP) || errors.Is(err, unix.EOPNOTSUPP) { + t.Skipf("filesystem does not support sparse extent discovery: %v", err) + } + require.NoError(t, err) + assert.GreaterOrEqual(t, stats.DeltaBytes, int64(2*4096)) + assert.Equal(t, stats.DeltaBytes, stats.ReflinkedBytes+stats.CopiedBytes) + assert.NoFileExists(t, diffPath) + + want := append([]byte(nil), base...) + copy(want[64*4096:], changed) + copy(want[512*4096:], zeroed) + got, err := os.ReadFile(basePath) + require.NoError(t, err) + assert.Equal(t, want, got) +} diff --git a/lib/hypervisor/cloudhypervisor/diff_snapshot_test.go b/lib/hypervisor/cloudhypervisor/diff_snapshot_test.go new file mode 100644 index 000000000..d34a9e23d --- /dev/null +++ b/lib/hypervisor/cloudhypervisor/diff_snapshot_test.go @@ -0,0 +1,39 @@ +package cloudhypervisor + +import ( + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/vmm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPrepareDiffSnapshotDestination(t *testing.T) { + t.Setenv(experimentalDiffSnapshotsEnv, "true") + dir := t.TempDir() + for _, name := range []string{ + cloudHypervisorMemoryFile, + cloudHypervisorConfigFile, + cloudHypervisorStateFile, + cloudHypervisorDiffMemoryFile, + } { + require.NoError(t, os.WriteFile(filepath.Join(dir, name), []byte(name), 0600)) + } + + diff, err := prepareDiffSnapshotDestination(dir) + require.NoError(t, err) + assert.True(t, diff) + assert.FileExists(t, filepath.Join(dir, cloudHypervisorMemoryFile)) + assert.NoFileExists(t, filepath.Join(dir, cloudHypervisorConfigFile)) + assert.NoFileExists(t, filepath.Join(dir, cloudHypervisorStateFile)) + assert.NoFileExists(t, filepath.Join(dir, cloudHypervisorDiffMemoryFile)) +} + +func TestExperimentalDiffSnapshotCapability(t *testing.T) { + t.Setenv(experimentalDiffSnapshotsEnv, "true") + assert.True(t, CapabilitiesForVersion(vmm.V51_1).SupportsSnapshotBaseReuse) + t.Setenv(experimentalDiffSnapshotsEnv, "false") + assert.False(t, CapabilitiesForVersion(vmm.V51_1).SupportsSnapshotBaseReuse) +} diff --git a/lib/hypervisor/cloudhypervisor/diff_snapshot_unsupported.go b/lib/hypervisor/cloudhypervisor/diff_snapshot_unsupported.go new file mode 100644 index 000000000..65f33cf2b --- /dev/null +++ b/lib/hypervisor/cloudhypervisor/diff_snapshot_unsupported.go @@ -0,0 +1,9 @@ +//go:build !linux + +package cloudhypervisor + +import "errors" + +func mergeCloudHypervisorDiff(string) (diffMergeStats, error) { + return diffMergeStats{}, errors.New("cloud hypervisor diff snapshots require Linux") +} diff --git a/lib/hypervisor/cloudhypervisor/process.go b/lib/hypervisor/cloudhypervisor/process.go index 674014885..f61ac99e5 100644 --- a/lib/hypervisor/cloudhypervisor/process.go +++ b/lib/hypervisor/cloudhypervisor/process.go @@ -261,6 +261,9 @@ func (s *Starter) RestoreVM(ctx context.Context, p *paths.Paths, version string, SourceUrl: sourceURL, Prefault: ptr(false), } + if experimentalDiffSnapshotsEnabled() { + restoreConfig.TrackDirtyPages = ptr(true) + } resp, err := hv.client.PutVmRestoreWithResponse(ctx, restoreConfig) if err != nil { return 0, nil, fmt.Errorf("restore: %w", err) diff --git a/lib/instances/firecracker_uffd_test.go b/lib/instances/firecracker_uffd_test.go index 73c7306c3..087958008 100644 --- a/lib/instances/firecracker_uffd_test.go +++ b/lib/instances/firecracker_uffd_test.go @@ -263,7 +263,7 @@ func TestEnsureExclusiveSnapshotMemoryOwnershipUnsharesHardlinkedMemory(t *testi forkLinkPath := filepath.Join(root, "fork-memory") require.NoError(t, os.Link(memPath, forkLinkPath)) - require.NoError(t, ensureExclusiveSnapshotMemoryOwnership(context.Background(), snapshotDir)) + require.NoError(t, ensureExclusiveSnapshotMemoryOwnership(context.Background(), snapshotDir, hypervisor.TypeFirecracker)) assertDifferentInode(t, memPath, forkLinkPath) unshared, err := os.ReadFile(memPath) @@ -286,14 +286,14 @@ func TestEnsureExclusiveSnapshotMemoryOwnershipSkipsPrivateMemory(t *testing.T) before, err := os.Stat(memPath) require.NoError(t, err) - require.NoError(t, ensureExclusiveSnapshotMemoryOwnership(context.Background(), snapshotDir)) + require.NoError(t, ensureExclusiveSnapshotMemoryOwnership(context.Background(), snapshotDir, hypervisor.TypeFirecracker)) after, err := os.Stat(memPath) require.NoError(t, err) assert.True(t, os.SameFile(before, after), "private mem-file must not be rewritten") assert.NoFileExists(t, stalePath, "stale unshare tmp must be swept on standby entry") - require.NoError(t, ensureExclusiveSnapshotMemoryOwnership(context.Background(), filepath.Join(t.TempDir(), "missing"))) + require.NoError(t, ensureExclusiveSnapshotMemoryOwnership(context.Background(), filepath.Join(t.TempDir(), "missing"), hypervisor.TypeFirecracker)) } func installOneShotFirecrackerStarter(t *testing.T, mgr *manager) { diff --git a/lib/instances/snapshot_memfile.go b/lib/instances/snapshot_memfile.go index ab391645d..451e0780b 100644 --- a/lib/instances/snapshot_memfile.go +++ b/lib/instances/snapshot_memfile.go @@ -74,16 +74,20 @@ func linkFallbackReason(err error) string { return "unknown" } -// ensureExclusiveSnapshotMemoryOwnership replaces the snapshot mem-file with a -// private copy when other hardlinks to its inode exist (fanout forks). -// Firecracker merges diff snapshots by writing dirty pages into this file in -// place, which must never mutate memory another instance still reads. +// ensureExclusiveSnapshotMemoryOwnership replaces the snapshot mem-file before +// a diff snapshot when another instance shares it. Cloud Hypervisor always gets +// a reflinked inode because its running VM can still map the current base while +// the sparse delta is merged. // // The stat -> copy -> rename sequence is not internally synchronized; callers // must hold the instance's write lock (the standby path does) so no fork can // take a new hardlink between the link-count check and the replacement. -func ensureExclusiveSnapshotMemoryOwnership(ctx context.Context, snapshotDir string) error { - memPath := filepath.Join(snapshotDir, "memory") +func ensureExclusiveSnapshotMemoryOwnership(ctx context.Context, snapshotDir string, hvType hypervisor.Type) error { + relPath, ok := sharedSnapshotMemoryRelPath(hvType) + if !ok { + return fmt.Errorf("snapshot memory ownership is not supported for hypervisor %s", hvType) + } + memPath := filepath.Join(snapshotDir, filepath.Base(relPath)) tmpPath := memPath + ".unshare.tmp" // Sweep any stale tmp from a crash between copy and rename; it is // mem-file-sized and must not linger. @@ -97,20 +101,20 @@ func ensureExclusiveSnapshotMemoryOwnership(ctx context.Context, snapshotDir str return fmt.Errorf("stat snapshot memory: %w", err) } stat, ok := info.Sys().(*syscall.Stat_t) - if !ok || stat.Nlink <= 1 { + if !ok || (stat.Nlink <= 1 && hvType != hypervisor.TypeCloudHypervisor) { return nil } start := time.Now() if err := forkvm.CopyRegularFile(memPath, tmpPath); err != nil { _ = os.Remove(tmpPath) - return fmt.Errorf("copy shared snapshot memory: %w", err) + return fmt.Errorf("copy snapshot memory: %w", err) } if err := os.Rename(tmpPath, memPath); err != nil { _ = os.Remove(tmpPath) - return fmt.Errorf("replace shared snapshot memory: %w", err) + return fmt.Errorf("replace snapshot memory: %w", err) } - logger.FromContext(ctx).InfoContext(ctx, "unshared snapshot memory before diff snapshot", + logger.FromContext(ctx).InfoContext(ctx, "detached snapshot memory before diff snapshot", "path", memPath, "links", stat.Nlink, "duration", time.Since(start).String()) return nil } diff --git a/lib/instances/snapshot_memfile_test.go b/lib/instances/snapshot_memfile_test.go index 81703a222..237aca9d8 100644 --- a/lib/instances/snapshot_memfile_test.go +++ b/lib/instances/snapshot_memfile_test.go @@ -34,6 +34,50 @@ func TestCloneCloudHypervisorSnapshotSharesMemoryInode(t *testing.T) { assert.FileExists(t, filepath.Join(dstDir, "metadata.json")) } +func TestEnsureExclusiveCloudHypervisorSnapshotMemory(t *testing.T) { + t.Parallel() + + snapshotDir := t.TempDir() + memoryPath := filepath.Join(snapshotDir, "memory-ranges") + require.NoError(t, os.WriteFile(memoryPath, []byte("shared memory"), 0600)) + siblingPath := filepath.Join(t.TempDir(), "memory-ranges") + require.NoError(t, os.Link(memoryPath, siblingPath)) + + require.NoError(t, ensureExclusiveSnapshotMemoryOwnership( + context.Background(), snapshotDir, hypervisor.TypeCloudHypervisor, + )) + + memoryInfo, err := os.Stat(memoryPath) + require.NoError(t, err) + siblingInfo, err := os.Stat(siblingPath) + require.NoError(t, err) + assert.False(t, os.SameFile(memoryInfo, siblingInfo)) + contents, err := os.ReadFile(memoryPath) + require.NoError(t, err) + assert.Equal(t, []byte("shared memory"), contents) +} + +func TestEnsureExclusiveCloudHypervisorSnapshotMemoryReplacesPrivateInode(t *testing.T) { + t.Parallel() + + snapshotDir := t.TempDir() + memoryPath := filepath.Join(snapshotDir, "memory-ranges") + require.NoError(t, os.WriteFile(memoryPath, []byte("private memory"), 0600)) + before, err := os.Stat(memoryPath) + require.NoError(t, err) + + require.NoError(t, ensureExclusiveSnapshotMemoryOwnership( + context.Background(), snapshotDir, hypervisor.TypeCloudHypervisor, + )) + + after, err := os.Stat(memoryPath) + require.NoError(t, err) + assert.False(t, os.SameFile(before, after)) + contents, err := os.ReadFile(memoryPath) + require.NoError(t, err) + assert.Equal(t, []byte("private memory"), contents) +} + func assertSameSnapshotMemoryInode(t *testing.T, first, second string) { t.Helper() firstInfo, err := os.Stat(first) diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 6913a9895..61fc22f1b 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -138,10 +138,9 @@ func (m *manager) standbyInstance( } return nil, fmt.Errorf("prepare retained snapshot target: %w", err) } - // The diff snapshot below writes dirty pages into the mem-file in - // place; if fanout forks still hardlink its inode, replace it with a - // private copy first so their memory is never mutated. - if err := ensureExclusiveSnapshotMemoryOwnership(ctx, snapshotDir); err != nil { + // Detach the retained memory before merging a diff so sibling forks + // and the running VM keep their immutable view of the prior base. + if err := ensureExclusiveSnapshotMemoryOwnership(ctx, snapshotDir, stored.HypervisorType); err != nil { if resumeErr := hv.Resume(ctx); resumeErr != nil { log.ErrorContext(ctx, "failed to resume VM after snapshot memory unshare error", "instance_id", id, "error", resumeErr) } diff --git a/lib/vmm/vmm.go b/lib/vmm/vmm.go index 9af758681..8b93caa48 100644 --- a/lib/vmm/vmm.go +++ b/lib/vmm/vmm.go @@ -50,6 +50,12 @@ const ( Shutdown VmInfoState = "Shutdown" ) +// Defines values for VmSnapshotConfigSnapshotType. +const ( + Diff VmSnapshotConfigSnapshotType = "diff" + Full VmSnapshotConfigSnapshotType = "full" +) + // BalloonConfig defines model for BalloonConfig. type BalloonConfig struct { // DeflateOnOom Deflate balloon when the guest is under memory pressure. @@ -314,8 +320,9 @@ type ReceiveMigrationData struct { // RestoreConfig defines model for RestoreConfig. type RestoreConfig struct { - Prefault *bool `json:"prefault,omitempty"` - SourceUrl string `json:"source_url"` + Prefault *bool `json:"prefault,omitempty"` + SourceUrl string `json:"source_url"` + TrackDirtyPages *bool `json:"track_dirty_pages,omitempty"` } // RngConfig defines model for RngConfig. @@ -451,9 +458,13 @@ type VmResizeZone struct { // VmSnapshotConfig defines model for VmSnapshotConfig. type VmSnapshotConfig struct { - DestinationUrl *string `json:"destination_url,omitempty"` + DestinationUrl *string `json:"destination_url,omitempty"` + SnapshotType *VmSnapshotConfigSnapshotType `json:"snapshot_type,omitempty"` } +// VmSnapshotConfigSnapshotType defines model for VmSnapshotConfig.SnapshotType. +type VmSnapshotConfigSnapshotType string + // VmmPingResponse Virtual Machine Monitor information type VmmPingResponse struct { BuildVersion *string `json:"build_version,omitempty"` diff --git a/specs/cloud-hypervisor/api-v0.3.0/cloud-hypervisor.yaml b/specs/cloud-hypervisor/api-v0.3.0/cloud-hypervisor.yaml index 629a6800d..ca7a09773 100644 --- a/specs/cloud-hypervisor/api-v0.3.0/cloud-hypervisor.yaml +++ b/specs/cloud-hypervisor/api-v0.3.0/cloud-hypervisor.yaml @@ -1266,6 +1266,10 @@ components: properties: destination_url: type: string + snapshot_type: + type: string + enum: [full, diff] + default: full VmCoredumpData: type: object @@ -1282,6 +1286,9 @@ components: type: string prefault: type: boolean + track_dirty_pages: + type: boolean + default: false ReceiveMigrationData: required: