From 26feb80ac58ade0150d95f97cb171b2ff71c6e8e Mon Sep 17 00:00:00 2001 From: sjmiller609 <7516283+sjmiller609@users.noreply.github.com> Date: Wed, 12 Aug 2026 02:20:06 +0000 Subject: [PATCH] Experiment with CPU and memory hotplug snapshots --- .../cloudhypervisor/cloudhypervisor.go | 25 ++++++ lib/hypervisor/cloudhypervisor/config.go | 21 +++-- lib/hypervisor/cloudhypervisor/config_test.go | 41 ++++++++++ .../cloudhypervisor/diff_snapshot_linux.go | 2 +- .../cloudhypervisor/hotplug_overlay.go | 29 +++++++ .../cloudhypervisor/kernel_paging.go | 70 +++++++++++++++-- .../cloudhypervisor/kernel_paging_test.go | 78 ++++++++++++++++--- lib/hypervisor/cloudhypervisor/process.go | 8 +- lib/instances/create.go | 4 +- 9 files changed, 249 insertions(+), 29 deletions(-) create mode 100644 lib/hypervisor/cloudhypervisor/hotplug_overlay.go diff --git a/lib/hypervisor/cloudhypervisor/cloudhypervisor.go b/lib/hypervisor/cloudhypervisor/cloudhypervisor.go index 75089e90..57d5607c 100644 --- a/lib/hypervisor/cloudhypervisor/cloudhypervisor.go +++ b/lib/hypervisor/cloudhypervisor/cloudhypervisor.go @@ -208,8 +208,33 @@ func (c *CloudHypervisor) Snapshot(ctx context.Context, destPath string) error { return nil } +func (c *CloudHypervisor) ResizeVCPUs(ctx context.Context, vcpus int) error { + resp, err := c.client.PutVmResizeWithResponse(ctx, vmm.VmResize{DesiredVcpus: &vcpus}) + if err != nil { + return fmt.Errorf("resize vCPUs: %w", err) + } + if resp.StatusCode() != 204 { + return fmt.Errorf("resize vCPUs failed with status %d", resp.StatusCode()) + } + return nil +} + // ResizeMemory changes the VM's memory allocation. func (c *CloudHypervisor) ResizeMemory(ctx context.Context, bytes int64) error { + if ExperimentalHotplugOverlayEnabled() { + resp, err := c.client.PutVmResizeZoneWithResponse(ctx, vmm.VmResizeZone{ + Id: ptr(kernelPagingMemoryZoneID), + DesiredRam: ptr(bytes), + }) + if err != nil { + return fmt.Errorf("resize memory zone: %w", err) + } + if resp.StatusCode() != 204 { + return fmt.Errorf("resize memory zone failed with status %d", resp.StatusCode()) + } + return nil + } + resizeConfig := vmm.VmResize{DesiredRam: &bytes} resp, err := c.client.PutVmResizeWithResponse(ctx, resizeConfig) if err != nil { diff --git a/lib/hypervisor/cloudhypervisor/config.go b/lib/hypervisor/cloudhypervisor/config.go index 360d3c78..2e7f04b4 100644 --- a/lib/hypervisor/cloudhypervisor/config.go +++ b/lib/hypervisor/cloudhypervisor/config.go @@ -37,13 +37,17 @@ func ToVMConfig(cfg hypervisor.VMConfig) vmm.VmConfig { } // CPU configuration + maxVCPUs := cfg.VCPUs + if experimentalMaxVCPUs := ExperimentalMaxVCPUs(); experimentalMaxVCPUs > maxVCPUs { + maxVCPUs = experimentalMaxVCPUs + } cpus := vmm.CpusConfig{ BootVcpus: cfg.VCPUs, - MaxVcpus: cfg.VCPUs, + MaxVcpus: maxVCPUs, } - // Add topology if provided - if cfg.Topology != nil { + // Add topology if provided. + if cfg.Topology != nil && maxVCPUs == cfg.VCPUs { cpus.Topology = &vmm.CpuTopology{ ThreadsPerCore: ptr(cfg.Topology.ThreadsPerCore), CoresPerDie: ptr(cfg.Topology.CoresPerDie), @@ -55,21 +59,26 @@ func ToVMConfig(cfg hypervisor.VMConfig) vmm.VmConfig { // Memory configuration memory := vmm.MemoryConfig{Size: cfg.MemoryBytes} if cfg.MemoryBackingFile != "" { - zones := []vmm.MemoryZoneConfig{{ + zone := vmm.MemoryZoneConfig{ Id: kernelPagingMemoryZoneID, Size: cfg.MemoryBytes, File: ptr(cfg.MemoryBackingFile), Shared: ptr(false), Prefault: ptr(false), Hugepages: ptr(false), - }} + } + if cfg.HotplugBytes > 0 && ExperimentalHotplugOverlayEnabled() { + zone.HotplugSize = &cfg.HotplugBytes + memory.HotplugMethod = ptr("VirtioMem") + } + zones := []vmm.MemoryZoneConfig{zone} memory.Size = 0 memory.Shared = ptr(false) memory.Prefault = ptr(false) memory.Hugepages = ptr(false) memory.Zones = &zones } - if cfg.HotplugBytes > 0 { + if cfg.HotplugBytes > 0 && (cfg.MemoryBackingFile == "" || !ExperimentalHotplugOverlayEnabled()) { memory.HotplugSize = &cfg.HotplugBytes memory.HotplugMethod = ptr("VirtioMem") } diff --git a/lib/hypervisor/cloudhypervisor/config_test.go b/lib/hypervisor/cloudhypervisor/config_test.go index 69e11e4a..f364e374 100644 --- a/lib/hypervisor/cloudhypervisor/config_test.go +++ b/lib/hypervisor/cloudhypervisor/config_test.go @@ -34,6 +34,47 @@ func TestToVMConfigFileBackedMemory(t *testing.T) { assert.False(t, *zone.Hugepages) } +func TestToVMConfigCPUHotplugEnvelope(t *testing.T) { + t.Setenv(experimentalMaxVCPUsEnv, "8") + + vmCfg := ToVMConfig(hypervisor.VMConfig{ + VCPUs: 4, + MemoryBytes: 1024 * 1024 * 1024, + Topology: &hypervisor.CPUTopology{ + ThreadsPerCore: 2, + CoresPerDie: 2, + DiesPerPackage: 1, + Packages: 1, + }, + }) + + require.NotNil(t, vmCfg.Cpus) + assert.Equal(t, 4, vmCfg.Cpus.BootVcpus) + assert.Equal(t, 8, vmCfg.Cpus.MaxVcpus) + assert.Nil(t, vmCfg.Cpus.Topology) +} + +func TestToVMConfigFileBackedHotplugMemory(t *testing.T) { + t.Setenv(experimentalHotplugOverlayEnv, "true") + + vmCfg := ToVMConfig(hypervisor.VMConfig{ + VCPUs: 2, + MemoryBytes: 4 * 1024 * 1024 * 1024, + HotplugBytes: 4 * 1024 * 1024 * 1024, + MemoryBackingFile: "/var/lib/hypeman/memory.raw", + }) + + require.NotNil(t, vmCfg.Memory) + require.NotNil(t, vmCfg.Memory.Zones) + require.Len(t, *vmCfg.Memory.Zones, 1) + zone := (*vmCfg.Memory.Zones)[0] + require.NotNil(t, zone.HotplugSize) + assert.Equal(t, int64(4*1024*1024*1024), *zone.HotplugSize) + assert.Nil(t, vmCfg.Memory.HotplugSize) + require.NotNil(t, vmCfg.Memory.HotplugMethod) + assert.Equal(t, "VirtioMem", *vmCfg.Memory.HotplugMethod) +} + func TestToVMConfig_GuestMemoryBalloon(t *testing.T) { cfg := hypervisor.VMConfig{ VCPUs: 1, diff --git a/lib/hypervisor/cloudhypervisor/diff_snapshot_linux.go b/lib/hypervisor/cloudhypervisor/diff_snapshot_linux.go index c16d41ed..0c58518e 100644 --- a/lib/hypervisor/cloudhypervisor/diff_snapshot_linux.go +++ b/lib/hypervisor/cloudhypervisor/diff_snapshot_linux.go @@ -50,7 +50,7 @@ func mergeCloudHypervisorDiff(snapshotDir string) (diffMergeStats, error) { dstFD := int(dst.Fd()) size := srcInfo.Size() offset := int64(0) - cloneRanges := true + cloneRanges := !ExperimentalHotplugOverlayEnabled() buf := make([]byte, 1<<20) for offset < size { diff --git a/lib/hypervisor/cloudhypervisor/hotplug_overlay.go b/lib/hypervisor/cloudhypervisor/hotplug_overlay.go new file mode 100644 index 00000000..ae7e5489 --- /dev/null +++ b/lib/hypervisor/cloudhypervisor/hotplug_overlay.go @@ -0,0 +1,29 @@ +package cloudhypervisor + +import ( + "os" + "strconv" + "strings" +) + +const ( + experimentalHotplugOverlayEnv = "HYPEMAN_EXPERIMENTAL_CH_HOTPLUG_OVERLAY" + experimentalMaxVCPUsEnv = "HYPEMAN_EXPERIMENTAL_CH_MAX_VCPUS" +) + +func ExperimentalMaxVCPUs() int { + value, err := strconv.Atoi(strings.TrimSpace(os.Getenv(experimentalMaxVCPUsEnv))) + if err != nil || value <= 0 { + return 0 + } + return value +} + +func ExperimentalHotplugOverlayEnabled() bool { + switch strings.ToLower(strings.TrimSpace(os.Getenv(experimentalHotplugOverlayEnv))) { + case "1", "true", "yes", "on": + return true + default: + return false + } +} diff --git a/lib/hypervisor/cloudhypervisor/kernel_paging.go b/lib/hypervisor/cloudhypervisor/kernel_paging.go index cb8817a9..4a785633 100644 --- a/lib/hypervisor/cloudhypervisor/kernel_paging.go +++ b/lib/hypervisor/cloudhypervisor/kernel_paging.go @@ -80,12 +80,16 @@ func prepareSnapshotForKernelPaging(snapshotDir string) (bool, error) { return false, nil } - fileOffsets, totalSize, ok := snapshotMappingFileOffsets(ranges, mappings) - if !ok || totalSize != uint64(memoryInfo.Size()) { - return false, nil - } - for i := range mappings { - mappings[i].FileOffset = fileOffsets[i] + fixedLayout := ExperimentalHotplugOverlayEnabled() && + fixedSnapshotMappingLayout(ranges, mappings, uint64(memoryInfo.Size())) + if !fixedLayout { + fileOffsets, totalSize, ok := snapshotMappingFileOffsets(ranges, mappings) + if !ok || totalSize != uint64(memoryInfo.Size()) { + return false, nil + } + for i := range mappings { + mappings[i].FileOffset = fileOffsets[i] + } } updatedConfig, ok, err := kernelPagingConfig(configData, memoryPath, mappings) @@ -256,6 +260,55 @@ func snapshotMappingFileOffsets( return mappingOffsets, offset, true } +func fixedSnapshotMappingLayout( + ranges []snapshotMemoryRange, + mappings []snapshotGuestRAMMapping, + fileSize uint64, +) bool { + if len(ranges) == 0 || len(mappings) == 0 { + return false + } + + hasVirtioMem := false + var layoutSize uint64 + for _, mapping := range mappings { + if mapping.Size == 0 || mapping.FileOffset > ^uint64(0)-mapping.Size { + return false + } + end := mapping.FileOffset + mapping.Size + if end > layoutSize { + layoutSize = end + } + hasVirtioMem = hasVirtioMem || mapping.VirtioMem + } + if !hasVirtioMem || layoutSize != fileSize { + return false + } + + for _, r := range ranges { + if r.Length == 0 || r.GPA > ^uint64(0)-r.Length { + return false + } + rangeEnd := r.GPA + r.Length + found := false + for _, mapping := range mappings { + if mapping.GPA > ^uint64(0)-mapping.Size { + return false + } + if r.GPA >= mapping.GPA && rangeEnd <= mapping.GPA+mapping.Size { + if found { + return false + } + found = true + } + } + if !found { + return false + } + } + return true +} + func kernelPagingConfig( configData []byte, memoryPath string, @@ -301,8 +354,9 @@ func kernelPagingConfig( } zone := zones[0] + zoneHasHotplug := hasConfiguredDevice(zone["hotplug_size"]) || hasConfiguredDevice(zone["hotplugged_size"]) if rawBool(zone["shared"]) || rawBool(zone["hugepages"]) || - hasConfiguredDevice(zone["hotplug_size"]) || hasConfiguredDevice(zone["hotplugged_size"]) { + (zoneHasHotplug && !ExperimentalHotplugOverlayEnabled()) { return nil, false, nil } var zoneID string @@ -310,7 +364,7 @@ func kernelPagingConfig( return nil, false, nil } for _, mapping := range mappings { - if mapping.ZoneID != zoneID { + if mapping.ZoneID != zoneID || (mapping.VirtioMem && !ExperimentalHotplugOverlayEnabled()) { return nil, false, nil } } diff --git a/lib/hypervisor/cloudhypervisor/kernel_paging_test.go b/lib/hypervisor/cloudhypervisor/kernel_paging_test.go index caf1c0a9..2c538764 100644 --- a/lib/hypervisor/cloudhypervisor/kernel_paging_test.go +++ b/lib/hypervisor/cloudhypervisor/kernel_paging_test.go @@ -70,6 +70,59 @@ func TestPrepareSnapshotForKernelPaging(t *testing.T) { assert.False(t, optimized) } +func TestPrepareSnapshotForKernelPagingWithHotplugOverlay(t *testing.T) { + t.Setenv(experimentalHotplugOverlayEnv, "true") + + dir := writeKernelPagingSnapshot(t, kernelPagingSnapshotFixture{ + Ranges: []snapshotMemoryRange{ + {GPA: 0, Length: 4096}, + {GPA: 1 << 32, Length: 4096}, + }, + Mappings: []snapshotGuestRAMMapping{ + {Slot: 0, GPA: 0, Size: 4096, ZoneID: kernelPagingMemoryZoneID}, + {Slot: 1, GPA: 1 << 32, Size: 8192, ZoneID: kernelPagingMemoryZoneID, VirtioMem: true, FileOffset: 4096}, + }, + MemorySize: 0, + MemoryFileSize: 12288, + Zones: []map[string]any{{ + "id": kernelPagingMemoryZoneID, + "size": 4096, + "file": "/old/memory.raw", + "hotplug_size": 8192, + "hotplugged_size": 4096, + }}, + }) + + optimized, err := prepareSnapshotForKernelPaging(dir) + require.NoError(t, err) + require.True(t, optimized) + + stateData, err := os.ReadFile(filepath.Join(dir, cloudHypervisorStateFile)) + require.NoError(t, err) + _, _, ranges, mappings, err := decodeSnapshotMemoryState(stateData) + require.NoError(t, err) + assert.Empty(t, ranges) + require.Len(t, mappings, 2) + assert.Equal(t, uint64(4096), mappings[1].FileOffset) + + configData, err := os.ReadFile(filepath.Join(dir, cloudHypervisorConfigFile)) + require.NoError(t, err) + var config struct { + Memory struct { + Zones []struct { + File string `json:"file"` + HotplugSize uint64 `json:"hotplug_size"` + HotpluggedSize uint64 `json:"hotplugged_size"` + } `json:"zones"` + } `json:"memory"` + } + require.NoError(t, json.Unmarshal(configData, &config)) + require.Len(t, config.Memory.Zones, 1) + assert.Equal(t, filepath.Join(dir, cloudHypervisorMemoryFile), config.Memory.Zones[0].File) + assert.Equal(t, uint64(8192), config.Memory.Zones[0].HotplugSize) + assert.Equal(t, uint64(4096), config.Memory.Zones[0].HotpluggedSize) +} + func TestPrepareSnapshotForKernelPagingIgnoresUnmarkedMemoryZones(t *testing.T) { t.Parallel() @@ -177,23 +230,26 @@ func TestPrepareSnapshotForKernelPagingLeavesUnsupportedSnapshotsUnchanged(t *te } type kernelPagingSnapshotFixture struct { - Ranges []snapshotMemoryRange - Mappings []snapshotGuestRAMMapping - MemorySize uint64 - Zones []map[string]any - Balloon any - Hugepages bool - Net any - Devices any + Ranges []snapshotMemoryRange + Mappings []snapshotGuestRAMMapping + MemorySize uint64 + MemoryFileSize int64 + Zones []map[string]any + Balloon any + Hugepages bool + Net any + Devices any } func writeKernelPagingSnapshot(t *testing.T, fixture kernelPagingSnapshotFixture) string { t.Helper() dir := t.TempDir() - memorySize := int64(0) - for _, r := range fixture.Ranges { - memorySize += int64(r.Length) + memorySize := fixture.MemoryFileSize + if memorySize == 0 { + for _, r := range fixture.Ranges { + memorySize += int64(r.Length) + } } require.NoError(t, os.WriteFile(filepath.Join(dir, cloudHypervisorMemoryFile), nil, 0600)) require.NoError(t, os.Truncate(filepath.Join(dir, cloudHypervisorMemoryFile), memorySize)) diff --git a/lib/hypervisor/cloudhypervisor/process.go b/lib/hypervisor/cloudhypervisor/process.go index f61ac99e..e86cf6aa 100644 --- a/lib/hypervisor/cloudhypervisor/process.go +++ b/lib/hypervisor/cloudhypervisor/process.go @@ -80,7 +80,7 @@ func (s *Starter) ValidateConfig(config hypervisor.VMConfig) error { if config.MemoryBytes <= 0 { return fmt.Errorf("file-backed memory requires a positive memory size") } - if config.HotplugBytes > 0 { + if config.HotplugBytes > 0 && !ExperimentalHotplugOverlayEnabled() { return fmt.Errorf("file-backed memory does not support hotplug memory") } if config.GuestMemory.EnableBalloon { @@ -138,7 +138,11 @@ func (s *Starter) StartVM(ctx context.Context, p *paths.Paths, version string, s if err := s.ValidateConfig(config); err != nil { return 0, nil, err } - if err := prepareMemoryBackingFile(config.MemoryBackingFile, config.MemoryBytes); err != nil { + memoryBackingSize := config.MemoryBytes + if config.MemoryBackingFile != "" && ExperimentalHotplugOverlayEnabled() { + memoryBackingSize += config.HotplugBytes + } + if err := prepareMemoryBackingFile(config.MemoryBackingFile, memoryBackingSize); err != nil { return 0, nil, fmt.Errorf("prepare memory backing file: %w", err) } diff --git a/lib/instances/create.go b/lib/instances/create.go index 3a541827..8c227dc6 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -13,6 +13,7 @@ import ( "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/guestmemory" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/hypervisor/cloudhypervisor" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/logger" @@ -928,7 +929,8 @@ func (m *manager) buildHypervisorConfig(ctx context.Context, inst *Instance, ima memoryBackingFile := "" if inst.HypervisorType == hypervisor.TypeCloudHypervisor && inst.HypervisorVersion == string(vmm.V51_1) && - inst.HotplugSize == 0 && !guestMemory.EnableBalloon && len(pciDevices) == 0 { + (inst.HotplugSize == 0 || cloudhypervisor.ExperimentalHotplugOverlayEnabled()) && + !guestMemory.EnableBalloon && len(pciDevices) == 0 { memoryBackingFile = filepath.Join(inst.DataDir, cloudHypervisorMemoryBackingFilename) }