From ffcf7dffaac016992344c99e917892a10e0587c2 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 18:57:41 +0000 Subject: [PATCH 01/19] Support vendor VFIO vGPU devices Linux 6.8 hosts with NVIDIA R580 drop the mdev interface: vGPUs are assigned by writing a type ID to a VF's nvidia/current_vgpu_type and passed to QEMU as a plain VFIO PCI device. Add a vendor VFIO backend behind the existing framework dispatch: profile discovery from the capacity-dependent creatable catalogs, least-loaded VF placement, create/verify/rollback, and release. Because the same VF path is reused across assignments (unlike mdev UUIDs), release is guarded: an in-process owner map covers the window before QEMU opens the device, and an open-VFIO-handle scan refuses to clear a VF a running VM still holds. Reconciliation clears orphaned assignments on startup, skipping VFs protected by the caller and failing closed when the protected set is unavailable. Branch the vGPU integration test by discovered framework and extend it to cover release on stop and reacquisition on start. --- integration/vgpu_test.go | 110 ++++-- lib/devices/GPU.md | 46 +-- lib/devices/gpu_mode.go | 30 -- lib/devices/mdev_darwin.go | 13 +- lib/devices/mdev_linux.go | 32 +- lib/devices/types.go | 15 +- lib/devices/vendor_vfio_linux.go | 476 ++++++++++++++++++++++++++ lib/devices/vendor_vfio_linux_test.go | 374 ++++++++++++++++++++ lib/devices/vgpu_linux.go | 116 ++++++- lib/devices/vgpu_linux_test.go | 50 +++ lib/resources/gpu.go | 35 +- lib/resources/monitoring_test.go | 2 +- lib/resources/resource.go | 8 +- 13 files changed, 1159 insertions(+), 148 deletions(-) delete mode 100644 lib/devices/gpu_mode.go create mode 100644 lib/devices/vendor_vfio_linux.go create mode 100644 lib/devices/vendor_vfio_linux_test.go create mode 100644 lib/devices/vgpu_linux_test.go diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 12861a6a7..3f8fdfd65 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "os" + "path/filepath" + "strings" "testing" "time" @@ -21,21 +23,23 @@ import ( "github.com/stretchr/testify/require" ) -// TestVGPU is an integration test that verifies vGPU (SR-IOV mdev) support works. +// TestVGPU is an integration test that verifies vGPU (SR-IOV) support works +// on the host's framework: mdev or NVIDIA's vendor-specific VFIO. // // This test automatically detects vGPU availability and skips if: -// - No SR-IOV VFs are found in /sys/class/mdev_bus/ +// - No vGPU framework (mdev or vendor VFIO) is discovered // - No vGPU profiles are available -// - Not running as root (required for mdev creation) +// - Not running as root (required for sysfs vGPU assignment) // - KVM is not available // // To run manually: // // sudo go test -v -run TestVGPU -timeout 5m ./integration/... // -// Note: This test verifies mdev creation and PCI device visibility inside the VM. -// It does NOT test nvidia-smi or CUDA functionality since that requires NVIDIA -// guest drivers pre-installed in the image. +// Note: This test verifies vGPU assignment, release on stop, reacquisition on +// start, and PCI device visibility inside the VM. It does NOT test nvidia-smi +// or CUDA functionality since that requires NVIDIA guest drivers pre-installed +// in the image. func TestVGPU(t *testing.T) { t.Parallel() if testing.Short() { @@ -159,9 +163,18 @@ func TestVGPU(t *testing.T) { instanceID = inst.Id t.Logf("Instance created: %s", inst.Id) - // Verify mdev UUID was assigned - require.NotEmpty(t, inst.GPUMdevUUID, "Instance should have mdev UUID assigned") - t.Logf("mdev UUID: %s", inst.GPUMdevUUID) + // Verify the assignment matches the host's framework + require.NotEmpty(t, inst.GPUDevicePath, "Instance should have a vGPU device path assigned") + switch inst.GPUFramework { + case devices.VGPUFrameworkMdev: + require.NotEmpty(t, inst.GPUMdevUUID, "mdev instance should have a UUID assigned") + t.Logf("mdev UUID: %s", inst.GPUMdevUUID) + case devices.VGPUFrameworkVendorVFIO: + require.Empty(t, inst.GPUMdevUUID, "vendor VFIO instance should not have an mdev UUID") + t.Logf("vendor VFIO VF: %s", inst.GPUDevicePath) + default: + t.Fatalf("unexpected vGPU framework %q", inst.GPUFramework) + } // Step 5: Check GPU resources AFTER creating instance t.Run("ResourcesDecrementedAfterCreation", func(t *testing.T) { @@ -180,12 +193,9 @@ func TestVGPU(t *testing.T) { assert.Less(t, availableAfter, availableBefore, "available instances should decrease after creating VM") }) - // Step 6: Verify mdev was created in sysfs - t.Run("MdevCreated", func(t *testing.T) { - mdevPath := "/sys/bus/mdev/devices/" + inst.GPUMdevUUID - _, err := os.Stat(mdevPath) - assert.NoError(t, err, "mdev device should exist at %s", mdevPath) - t.Logf("mdev exists at: %s", mdevPath) + // Step 6: Verify the assignment exists in sysfs + t.Run("VGPUAssignedInSysfs", func(t *testing.T) { + assertVGPUAssigned(t, inst.GPUFramework, inst.GPUDevicePath) }) // Step 7: Wait for guest agent to be ready @@ -225,13 +235,68 @@ func TestVGPU(t *testing.T) { require.NoError(t, err) assert.Equal(t, profile, actualInst.GPUProfile, "GPU profile should match") - assert.NotEmpty(t, actualInst.GPUMdevUUID, "mdev UUID should be set") - t.Logf("Instance GPU: profile=%s, mdev=%s", actualInst.GPUProfile, actualInst.GPUMdevUUID) + assert.Equal(t, inst.GPUFramework, actualInst.GPUFramework, "framework should match") + assert.NotEmpty(t, actualInst.GPUDevicePath, "device path should be set") + if inst.GPUFramework == devices.VGPUFrameworkMdev { + assert.NotEmpty(t, actualInst.GPUMdevUUID, "mdev UUID should be set") + } + t.Logf("Instance GPU: profile=%s, framework=%s, device=%s", actualInst.GPUProfile, actualInst.GPUFramework, actualInst.GPUDevicePath) + }) + + t.Log("Step 10: Stopping instance to release the vGPU...") + _, err = instanceManager.StopInstance(ctx, inst.Id) + require.NoError(t, err, "stop should succeed") + + t.Run("VGPUReleasedOnStop", func(t *testing.T) { + stopped, err := instanceManager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") + assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) + }) + + t.Log("Step 11: Starting instance to reacquire a vGPU...") + started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) + require.NoError(t, err, "start should succeed") + + t.Run("VGPUReacquiredOnStart", func(t *testing.T) { + require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") + assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") + assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) }) t.Log("✅ vGPU test PASSED!") } +func assertVGPUAssigned(t *testing.T, framework devices.VGPUFramework, devicePath string) { + t.Helper() + switch framework { + case devices.VGPUFrameworkMdev: + _, err := os.Stat(devicePath) + assert.NoError(t, err, "mdev device should exist at %s", devicePath) + case devices.VGPUFrameworkVendorVFIO: + data, err := os.ReadFile(filepath.Join(devicePath, "nvidia", "current_vgpu_type")) + require.NoError(t, err, "VF should expose current_vgpu_type") + assert.NotEqual(t, "0", strings.TrimSpace(string(data)), "VF should have a vGPU type assigned") + default: + t.Fatalf("unexpected vGPU framework %q", framework) + } +} + +func assertVGPUReleased(t *testing.T, framework devices.VGPUFramework, devicePath string) { + t.Helper() + switch framework { + case devices.VGPUFrameworkMdev: + _, err := os.Stat(devicePath) + assert.True(t, os.IsNotExist(err), "mdev device should be gone from %s", devicePath) + case devices.VGPUFrameworkVendorVFIO: + data, err := os.ReadFile(filepath.Join(devicePath, "nvidia", "current_vgpu_type")) + require.NoError(t, err, "VF should expose current_vgpu_type") + assert.Equal(t, "0", strings.TrimSpace(string(data)), "VF assignment should be released") + default: + t.Fatalf("unexpected vGPU framework %q", framework) + } +} + // checkVGPUTestPrerequisites checks if vGPU test can run. // Returns (skipReason, profileName) - skipReason is empty if all prerequisites are met. func checkVGPUTestPrerequisites() (string, string) { @@ -245,10 +310,13 @@ func checkVGPUTestPrerequisites() (string, string) { return "vGPU test requires root (sudo) for mdev creation", "" } - // Check for vGPU mode (SR-IOV VFs present) - mode := devices.DetectHostGPUMode() - if mode != devices.GPUModeVGPU { - return "vGPU test requires SR-IOV VFs in /sys/class/mdev_bus/", "" + // Check for a vGPU framework (mdev or vendor VFIO) + framework, _, err := devices.DiscoverVGPU() + if err != nil { + return "vGPU test failed to discover vGPU framework: " + err.Error(), "" + } + if framework == devices.VGPUFrameworkNone { + return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", "" } // Check for available profiles diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 54a19c472..cdd269dc8 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -8,16 +8,17 @@ hypeman supports two GPU modes, automatically detected based on host configurati | Mode | Description | Use Case | |------|-------------|----------| -| **vGPU (SR-IOV)** | Virtual GPUs via mdev on SR-IOV VFs | Multi-tenant, shared GPU resources | +| **vGPU (SR-IOV)** | Virtual GPUs on SR-IOV VFs via mdev or vendor VFIO | Multi-tenant, shared GPU resources | | **Passthrough** | Whole GPU VFIO passthrough | Dedicated GPU per instance | The host's GPU mode is determined by the host driver configuration: -- If `/sys/class/mdev_bus/` contains VFs → vGPU mode -- If NVIDIA GPUs are available for VFIO → passthrough mode +- If `/sys/class/mdev_bus/` contains VFs → mdev vGPU mode +- If VFs expose `/sys/bus/pci/devices//nvidia/current_vgpu_type` → vendor VFIO vGPU mode +- If NVIDIA GPUs are available for whole-device VFIO → passthrough mode ## vGPU Mode (Recommended) -vGPU mode uses NVIDIA's SR-IOV technology to create Virtual Functions (VFs), each capable of hosting an mdev (mediated device) representing a vGPU. +vGPU mode uses NVIDIA's SR-IOV technology to create Virtual Functions (VFs). Hosts on older kernels represent each vGPU as an mdev. Hosts using NVIDIA's vendor VFIO framework assign the profile directly to the VF through `current_vgpu_type`. ### How It Works @@ -74,7 +75,7 @@ curl -X POST http://localhost:4973/instances \ }' ``` -The response includes the assigned mdev UUID: +On an mdev host, the response also includes the assigned mdev UUID: ```json { @@ -87,19 +88,16 @@ The response includes the assigned mdev UUID: } ``` -### Ephemeral mdev Lifecycle +### Ephemeral vGPU Lifecycle -mdev devices are **ephemeral**: created on instance start, destroyed on instance delete. +vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts and writes the profile ID/`0` to `current_vgpu_type` on vendor VFIO hosts. ``` -Instance Create → Create mdev → Attach to VM → Instance Running -Instance Delete → Stop VM → Destroy mdev → VF available again +Instance Create → Assign profile to VF → Attach VF to VM → Instance Running +Instance Stop/Delete → Release profile → VF available again ``` -This ensures: -- **Security**: No VRAM data leakage between instances -- **Clean state**: Fresh vGPU for each instance -- **Automatic cleanup**: Orphaned mdevs cleaned up on server restart +Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. ## Passthrough Mode @@ -241,7 +239,8 @@ To upgrade the NVIDIA driver version: 1. Check host GPU mode detection: ```bash - ls /sys/class/mdev_bus/ # Should show VFs for vGPU mode + ls /sys/class/mdev_bus/ + find /sys/bus/pci/devices -path '*/nvidia/current_vgpu_type' ``` 2. Verify NVIDIA drivers are loaded on host: @@ -265,17 +264,18 @@ curl -s http://localhost:4973/resources | jq '.gpu.profiles' curl http://localhost:4973/instances//logs?source=app ``` -### mdev creation fails +### vGPU assignment fails -1. Check if VFs are available: - ```bash - ls /sys/class/mdev_bus/ - ``` +Check the files for the framework detected on the host: -2. Verify mdev types: - ```bash - cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances - ``` +```bash +# mdev +cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances + +# vendor VFIO +cat /sys/bus/pci/devices/*/nvidia/creatable_vgpu_types +cat /sys/bus/pci/devices/*/nvidia/current_vgpu_type +``` ## Performance Tuning diff --git a/lib/devices/gpu_mode.go b/lib/devices/gpu_mode.go deleted file mode 100644 index 40b3b2ba0..000000000 --- a/lib/devices/gpu_mode.go +++ /dev/null @@ -1,30 +0,0 @@ -package devices - -import ( - "os" -) - -// DetectHostGPUMode determines the host's GPU configuration mode. -// -// Returns: -// - GPUModeVGPU if /sys/class/mdev_bus has entries (SR-IOV VFs present) -// - GPUModePassthrough if NVIDIA GPUs are available for VFIO passthrough -// - GPUModeNone if no GPUs are available -// -// Note: A host is configured for either vGPU or passthrough, not both, -// because the host driver determines which mode is available. -func DetectHostGPUMode() GPUMode { - // Check for vGPU mode first (SR-IOV VFs present) - entries, err := os.ReadDir("/sys/class/mdev_bus") - if err == nil && len(entries) > 0 { - return GPUModeVGPU - } - - // Check for passthrough mode (physical GPUs available) - gpus, err := DiscoverAvailableDevices() - if err == nil && len(gpus) > 0 { - return GPUModePassthrough - } - - return GPUModeNone -} diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 1427a5095..4274063ed 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -12,10 +12,9 @@ func SetGPUProfileCacheTTL(ttl string) { // No-op on macOS } -// DiscoverVFs returns an empty list on macOS. -// SR-IOV Virtual Functions are not available on macOS. -func DiscoverVFs() ([]VirtualFunction, error) { - return []VirtualFunction{}, nil +// DiscoverVGPU reports no vGPU framework on macOS. +func DiscoverVGPU() (VGPUFramework, []VirtualFunction, error) { + return VGPUFrameworkNone, nil, nil } // ListGPUProfiles returns an empty list on macOS. @@ -24,7 +23,7 @@ func ListGPUProfiles() ([]GPUProfile, error) { } // ListGPUProfilesWithVFs returns an empty list on macOS. -func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) { return []GPUProfile{}, nil } @@ -59,6 +58,10 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { return nil } +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { + return nil +} + // ReconcileMdevs is a no-op on macOS. func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error { return nil diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 1a398a418..e2891efc9 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -89,14 +89,13 @@ func getCachedProfiles(firstVF string) []profileMetadata { return cachedProfiles } -// DiscoverVFs returns all SR-IOV Virtual Functions available for vGPU. -// These are discovered by scanning /sys/class/mdev_bus/ which contains -// VFs that can host mdev devices. -func DiscoverVFs() ([]VirtualFunction, error) { +// discoverMdevVFs returns all SR-IOV Virtual Functions available for vGPU, +// discovered by scanning /sys/class/mdev_bus/. +func discoverMdevVFs() ([]VirtualFunction, error) { entries, err := os.ReadDir(mdevBusPath) if err != nil { if os.IsNotExist(err) { - return nil, nil // No mdev_bus means no vGPU support + return nil, nil // No mdev_bus means no mdev vGPU support } return nil, fmt.Errorf("read mdev_bus: %w", err) } @@ -133,20 +132,9 @@ func DiscoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// ListGPUProfiles returns available vGPU profiles with availability counts. -// Profiles are discovered from the first VF's mdev_supported_types directory. -func ListGPUProfiles() ([]GPUProfile, error) { - vfs, err := DiscoverVFs() - if err != nil { - return nil, err - } - return ListGPUProfilesWithVFs(vfs) -} - -// ListGPUProfilesWithVFs returns available vGPU profiles using pre-discovered VFs. -// This avoids redundant VF discovery when the caller already has the list. -// Uses parallel sysfs reads for fast availability counting. -func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { +// listMdevGPUProfilesWithVFs returns available vGPU profiles using +// pre-discovered VFs. Uses parallel sysfs reads for fast availability counting. +func listMdevGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { if len(vfs) == 0 { return nil, nil } @@ -305,7 +293,7 @@ func countAvailableForSingleProfile(freeVFsByParent map[string][]VirtualFunction // findProfileType finds the internal type name (e.g., "nvidia-556") for a profile name (e.g., "L40S-1Q") func findProfileType(profileName string) (string, error) { - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil || len(vfs) == 0 { return "", fmt.Errorf("no VFs available") } @@ -531,7 +519,7 @@ func CreateMdev(ctx context.Context, profileName, instanceID string) (*MdevDevic } // Discover all VFs - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil { return nil, fmt.Errorf("discover VFs: %w", err) } @@ -697,7 +685,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log := logger.FromContext(ctx) _ = instanceInfos - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil { return fmt.Errorf("discover managed VFs: %w", err) } diff --git a/lib/devices/types.go b/lib/devices/types.go index 809d669fe..c76d239ed 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -63,12 +63,13 @@ type GPUMode string type VGPUFramework string const ( - VGPUFrameworkNone VGPUFramework = "" - VGPUFrameworkMdev VGPUFramework = "mdev" + VGPUFrameworkNone VGPUFramework = "" + VGPUFrameworkMdev VGPUFramework = "mdev" + VGPUFrameworkVendorVFIO VGPUFramework = "vendor-vfio" // GPUModePassthrough indicates whole GPU VFIO passthrough GPUModePassthrough GPUMode = "passthrough" - // GPUModeVGPU indicates SR-IOV + mdev based vGPU + // GPUModeVGPU indicates vGPU mode GPUModeVGPU GPUMode = "vgpu" // GPUModeNone indicates no GPU available GPUModeNone GPUMode = "none" @@ -76,9 +77,10 @@ const ( // VirtualFunction represents an SR-IOV Virtual Function for vGPU type VirtualFunction struct { - PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4" - ParentGPU string `json:"parent_gpu"` // e.g., "0000:82:00.0" - Allocated bool `json:"allocated"` // true if a vGPU is assigned to this VF + PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4" + ParentGPU string `json:"parent_gpu"` // e.g., "0000:82:00.0" + Allocated bool `json:"allocated"` // true if a vGPU is assigned to this VF + ProfileType string `json:"profile_type,omitempty"` } // VGPUAssignment identifies an existing vGPU assignment to release. @@ -86,6 +88,7 @@ type VGPUAssignment struct { Framework VGPUFramework DevicePath string MdevUUID string + InstanceID string } type VGPUDevice struct { diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go new file mode 100644 index 000000000..23ab1e89a --- /dev/null +++ b/lib/devices/vendor_vfio_linux.go @@ -0,0 +1,476 @@ +//go:build linux + +package devices + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + + "github.com/kernel/hypeman/lib/logger" +) + +const ( + pciDevicesPath = "/sys/bus/pci/devices" + vfioDevicesPath = "/dev/vfio/devices" +) + +type vendorVFIOSysfs struct { + pciDevicesPath string + procPath string + vfioDevicesPath string + owners map[string]string +} + +var ( + hostVendorVFIO = vendorVFIOSysfs{ + pciDevicesPath: pciDevicesPath, + procPath: procPath, + vfioDevicesPath: vfioDevicesPath, + owners: make(map[string]string), + } + vendorVFIOMu sync.Mutex +) + +func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { + entries, err := os.ReadDir(s.pciDevicesPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read PCI devices: %w", err) + } + + vfs := make([]VirtualFunction, 0) + for _, entry := range entries { + vfPath := filepath.Join(s.pciDevicesPath, entry.Name()) + nvidiaPath := filepath.Join(vfPath, "nvidia") + if _, err := os.Stat(filepath.Join(nvidiaPath, "creatable_vgpu_types")); err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("stat creatable vGPU types for VF %s: %w", entry.Name(), err) + } + + currentType, err := readCurrentVGPUType(filepath.Join(nvidiaPath, "current_vgpu_type")) + if err != nil { + return nil, fmt.Errorf("read current vGPU type for VF %s: %w", entry.Name(), err) + } + + parentGPU := "" + if target, err := os.Readlink(filepath.Join(vfPath, "physfn")); err == nil { + parentGPU = filepath.Base(target) + } + vfs = append(vfs, VirtualFunction{ + PCIAddress: entry.Name(), + ParentGPU: parentGPU, + Allocated: currentType != "0", + ProfileType: currentType, + }) + } + + sort.Slice(vfs, func(i, j int) bool { return vfs[i].PCIAddress < vfs[j].PCIAddress }) + return vfs, nil +} + +func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { + profilesByType := make(map[string]profileMetadata) + availability := make(map[string]int) + for _, vf := range vfs { + creatable, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + return nil, err + } + for _, profile := range creatable { + profilesByType[profile.TypeName] = profile + if !vf.Allocated { + availability[profile.TypeName]++ + } + } + } + + metadata := make([]profileMetadata, 0, len(profilesByType)) + for _, profile := range profilesByType { + metadata = append(metadata, profile) + } + sort.Slice(metadata, func(i, j int) bool { return metadata[i].Name < metadata[j].Name }) + + profiles := make([]GPUProfile, 0, len(metadata)) + for _, profile := range metadata { + profiles = append(profiles, GPUProfile{ + Name: profile.Name, + FramebufferMB: profile.FramebufferMB, + Available: availability[profile.TypeName], + }) + } + return profiles, nil +} + +func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + + vfs, err := s.discoverVFs() + if err != nil { + return nil, err + } + metadata, err := s.profileMetadata(vfs) + if err != nil { + return nil, err + } + + var requested profileMetadata + found := false + for _, profile := range metadata { + if profile.Name == profileName { + requested = profile + found = true + break + } + } + if !found { + if len(metadata) == 0 && len(vfs) > 0 { + return nil, fmt.Errorf("no creatable vGPU profiles on any VF, GPUs may be at capacity: profile %q", profileName) + } + return nil, fmt.Errorf("profile %q is not creatable on any VF (unknown profile or insufficient capacity)", profileName) + } + + targetVF, err := s.selectLeastLoadedVF(vfs, metadata, requested.TypeName) + if err != nil { + return nil, err + } + if targetVF == "" { + return nil, fmt.Errorf("no available VF for profile %q", profileName) + } + + currentTypePath := filepath.Join(s.pciDevicesPath, targetVF, "nvidia", "current_vgpu_type") + if err := os.WriteFile(currentTypePath, []byte(requested.TypeName), 0200); err != nil { + return nil, fmt.Errorf("create vGPU on VF %s: %w", targetVF, err) + } + currentType, err := readCurrentVGPUType(currentTypePath) + if err != nil { + verifyErr := fmt.Errorf("verify vGPU on VF %s: %w", targetVF, err) + return nil, rollbackVendorVFIOCreate(currentTypePath, targetVF, verifyErr) + } + if currentType != requested.TypeName { + verifyErr := fmt.Errorf("verify vGPU on VF %s: type is %s, want %s", targetVF, currentType, requested.TypeName) + return nil, rollbackVendorVFIOCreate(currentTypePath, targetVF, verifyErr) + } + s.owners[targetVF] = instanceID + + logger.FromContext(ctx).InfoContext(ctx, "created vendor VFIO vGPU", + "profile", profileName, + "vf", targetVF, + "instance_id", instanceID, + ) + return &VGPUDevice{ + Framework: VGPUFrameworkVendorVFIO, + VFAddress: targetVF, + ProfileType: requested.TypeName, + ProfileName: profileName, + SysfsPath: filepath.Join(s.pciDevicesPath, targetVF), + }, nil +} + +func (s vendorVFIOSysfs) destroy(ctx context.Context, vfAddress, instanceID string) error { + return s.destroyWithOpenPaths(ctx, vfAddress, instanceID, nil) +} + +func (s vendorVFIOSysfs) destroyWithOpenPaths(ctx context.Context, vfAddress, instanceID string, openPaths map[string]struct{}) error { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + + log := logger.FromContext(ctx) + currentTypePath := filepath.Join(s.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type") + currentType, err := readCurrentVGPUType(currentTypePath) + if err != nil { + if os.IsNotExist(err) { + delete(s.owners, vfAddress) + return nil + } + return fmt.Errorf("read current vGPU type for VF %s: %w", vfAddress, err) + } + if currentType == "0" { + delete(s.owners, vfAddress) + return nil + } + + if owner, ok := s.owners[vfAddress]; ok && (instanceID == "" || owner != instanceID) { + log.WarnContext(ctx, "skipping vendor VFIO vGPU release owned by another instance", + "vf", vfAddress, + "owner_instance_id", owner, + "requesting_instance_id", instanceID, + ) + return nil + } + + if openPaths == nil { + if openPaths, err = s.openVFIOPaths(); err != nil { + return fmt.Errorf("scan open VFIO handles: %w", err) + } + } + inUse, err := s.vfioDeviceInUse(vfAddress, openPaths) + if err != nil { + return fmt.Errorf("check vendor VFIO vGPU usage for VF %s: %w", vfAddress, err) + } + if inUse { + return fmt.Errorf("vendor VFIO vGPU on VF %s is still in use", vfAddress) + } + + if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { + return fmt.Errorf("destroy vGPU on VF %s: %w", vfAddress, err) + } + delete(s.owners, vfAddress) + log.InfoContext(ctx, "destroyed vendor VFIO vGPU", "vf", vfAddress) + return nil +} + +func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map[string]struct{}) error { + vfs, err := s.discoverVFs() + if err != nil { + return err + } + log := logger.FromContext(ctx) + protectedVFs := make(map[string]struct{}, len(protectedDevicePaths)) + for path := range protectedDevicePaths { + protectedVFs[filepath.Base(path)] = struct{}{} + } + var openPaths map[string]struct{} + for _, vf := range vfs { + if !vf.Allocated { + continue + } + if _, ok := protectedVFs[vf.PCIAddress]; ok { + log.DebugContext(ctx, "skipping vendor VFIO vGPU held by a live instance", "vf", vf.PCIAddress) + continue + } + if openPaths == nil { + if openPaths, err = s.openVFIOPaths(); err != nil { + return fmt.Errorf("scan open VFIO handles: %w", err) + } + } + inUse, err := s.vfioDeviceInUse(vf.PCIAddress, openPaths) + if err != nil { + log.WarnContext(ctx, "failed to check vendor VFIO vGPU usage", "vf", vf.PCIAddress, "error", err) + continue + } + if inUse { + continue + } + if err := s.destroyWithOpenPaths(ctx, vf.PCIAddress, "", openPaths); err != nil { + log.WarnContext(ctx, "failed to destroy orphaned vendor VFIO vGPU", "vf", vf.PCIAddress, "error", err) + } + } + return nil +} + +func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, metadata []profileMetadata, profileType string) (string, error) { + framebufferByType := make(map[string]int, len(metadata)) + for _, profile := range metadata { + framebufferByType[profile.TypeName] = profile.FramebufferMB + } + + usageByGPU := make(map[string]int) + freeByGPU := make(map[string][]VirtualFunction) + for _, vf := range vfs { + if vf.Allocated { + usageByGPU[vf.ParentGPU] += framebufferByType[vf.ProfileType] + continue + } + profiles, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + return "", err + } + for _, profile := range profiles { + if profile.TypeName == profileType { + freeByGPU[vf.ParentGPU] = append(freeByGPU[vf.ParentGPU], vf) + break + } + } + } + + gpus := make([]string, 0, len(freeByGPU)) + for gpu := range freeByGPU { + gpus = append(gpus, gpu) + } + sort.Slice(gpus, func(i, j int) bool { + if usageByGPU[gpus[i]] == usageByGPU[gpus[j]] { + return gpus[i] < gpus[j] + } + return usageByGPU[gpus[i]] < usageByGPU[gpus[j]] + }) + if len(gpus) == 0 { + return "", nil + } + return freeByGPU[gpus[0]][0].PCIAddress, nil +} + +func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetadata, error) { + profilesByType := make(map[string]profileMetadata) + for _, vf := range vfs { + profiles, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + return nil, err + } + for _, profile := range profiles { + profilesByType[profile.TypeName] = profile + } + } + profiles := make([]profileMetadata, 0, len(profilesByType)) + for _, profile := range profilesByType { + profiles = append(profiles, profile) + } + sort.Slice(profiles, func(i, j int) bool { return profiles[i].Name < profiles[j].Name }) + return profiles, nil +} + +func (s vendorVFIOSysfs) readCreatableProfiles(vfAddress string) ([]profileMetadata, error) { + path := filepath.Join(s.pciDevicesPath, vfAddress, "nvidia", "creatable_vgpu_types") + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read creatable vGPU types for VF %s: %w", vfAddress, err) + } + return parseCreatableVGPUTypes(string(data)) +} + +func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string]struct{}) (bool, error) { + devicePaths := make([]string, 0, 2) + probeErrs := make([]error, 0, 2) + + vfioDevices, err := os.ReadDir(filepath.Join(s.pciDevicesPath, vfAddress, "vfio-dev")) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + probeErrs = append(probeErrs, fmt.Errorf("read VFIO devices for VF %s: %w", vfAddress, err)) + } else { + for _, device := range vfioDevices { + devicePaths = append(devicePaths, filepath.Join(s.vfioDevicesPath, device.Name())) + } + } + + target, err := os.Readlink(filepath.Join(s.pciDevicesPath, vfAddress, "iommu_group")) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + probeErrs = append(probeErrs, fmt.Errorf("read IOMMU group for VF %s: %w", vfAddress, err)) + } else { + devicePaths = append(devicePaths, filepath.Join(filepath.Dir(s.vfioDevicesPath), filepath.Base(target))) + } + + for _, path := range devicePaths { + if _, ok := openPaths[path]; ok { + return true, nil + } + } + if len(probeErrs) > 0 { + return false, errors.Join(probeErrs...) + } + return false, nil +} + +func (s vendorVFIOSysfs) openVFIOPaths() (map[string]struct{}, error) { + processes, err := os.ReadDir(s.procPath) + if err != nil { + return nil, err + } + prefix := filepath.Dir(s.vfioDevicesPath) + string(filepath.Separator) + open := make(map[string]struct{}) + for _, process := range processes { + if _, err := strconv.Atoi(process.Name()); err != nil { + continue + } + fdPath := filepath.Join(s.procPath, process.Name(), "fd") + fds, err := os.ReadDir(fdPath) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("read process %s file descriptors: %w", process.Name(), err) + } + for _, fd := range fds { + target, err := os.Readlink(filepath.Join(fdPath, fd.Name())) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("read process %s file descriptor %s: %w", process.Name(), fd.Name(), err) + } + if strings.HasPrefix(target, prefix) { + open[target] = struct{}{} + } + } + } + return open, nil +} + +func parseCreatableVGPUTypes(value string) ([]profileMetadata, error) { + profiles := make([]profileMetadata, 0) + for lineNumber, line := range strings.Split(value, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + typeID, name, found := strings.Cut(line, ":") + typeID = strings.TrimSpace(typeID) + name = strings.TrimSpace(name) + if typeID == "ID" { + continue + } + if !found || name == "" { + return nil, fmt.Errorf("parse creatable vGPU types line %d: %q", lineNumber+1, line) + } + if _, err := strconv.Atoi(typeID); err != nil { + return nil, fmt.Errorf("parse vGPU type ID %q: %w", typeID, err) + } + profiles = append(profiles, profileMetadata{ + TypeName: typeID, + Name: name, + FramebufferMB: framebufferFromProfileName(name), + }) + } + return profiles, nil +} + +func framebufferFromProfileName(name string) int { + series := strings.LastIndexAny(name, "ABCQ") + if series <= 0 { + return 0 + } + dash := strings.LastIndex(name[:series], "-") + if dash < 0 { + return 0 + } + gb, err := strconv.Atoi(name[dash+1 : series]) + if err != nil { + return 0 + } + return gb * 1024 +} + +func rollbackVendorVFIOCreate(currentTypePath, vfAddress string, verifyErr error) error { + if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { + return errors.Join(verifyErr, fmt.Errorf("roll back vGPU on VF %s: %w", vfAddress, err)) + } + return verifyErr +} + +func readCurrentVGPUType(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + value := strings.TrimSpace(string(data)) + if _, err := strconv.Atoi(value); err != nil { + return "", fmt.Errorf("invalid current vGPU type %q", value) + } + return value, nil +} diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go new file mode 100644 index 000000000..8e79a7750 --- /dev/null +++ b/lib/devices/vendor_vfio_linux_test.go @@ -0,0 +1,374 @@ +//go:build linux + +package devices + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testCreatableTypes = `ID : vGPU Name +1147 : NVIDIA L40S-1Q +1148 : NVIDIA L40S-2Q +1159 : NVIDIA L40S-48Q +` + +func TestParseCreatableVGPUTypes(t *testing.T) { + t.Parallel() + + profiles, err := parseCreatableVGPUTypes(testCreatableTypes) + require.NoError(t, err) + require.Len(t, profiles, 3) + assert.Equal(t, profileMetadata{TypeName: "1147", Name: "NVIDIA L40S-1Q", FramebufferMB: 1024}, profiles[0]) + assert.Equal(t, profileMetadata{TypeName: "1159", Name: "NVIDIA L40S-48Q", FramebufferMB: 48 * 1024}, profiles[2]) +} + +func TestVendorVFIOCreateAndDestroy(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + require.Len(t, vfs, 1) + assert.False(t, vfs[0].Allocated) + assert.Equal(t, "0000:82:00.0", vfs[0].ParentGPU) + + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-2Q")) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, VGPUFrameworkVendorVFIO, device.Framework) + assert.Equal(t, "0000:82:00.4", device.VFAddress) + assert.Equal(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4"), device.SysfsPath) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "1148") + + require.NoError(t, sysfs.destroy(context.Background(), device.VFAddress, "instance-1")) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIODestroySkipsAssignmentOwnedByAnotherInstance(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.NoError(t, err) + + require.NoError(t, sysfs.destroy(context.Background(), device.VFAddress, "stale-instance")) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "1148") + + require.NoError(t, sysfs.destroy(context.Background(), device.VFAddress, "instance-1")) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIODestroyRetainsAssignmentInUse(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + activeDevice := filepath.Join(sysfs.vfioDevicesPath, "vfio42") + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(activeDevice, filepath.Join(fdDir, "5"))) + + err := sysfs.destroy(context.Background(), vfAddress, "instance-1") + require.Error(t, err) + assert.ErrorContains(t, err, "still in use") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIODestroyReleasesUnboundVF(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + unbind func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) + }{ + { + name: "missing vfio device directory", + unbind: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.RemoveAll(filepath.Join(sysfs.pciDevicesPath, vfAddress, "vfio-dev"))) + }, + }, + { + name: "missing iommu group symlink", + unbind: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.Remove(filepath.Join(sysfs.pciDevicesPath, vfAddress, "iommu_group"))) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + tt.unbind(t, sysfs, vfAddress) + + require.NoError(t, sysfs.destroy(context.Background(), vfAddress, "instance-1")) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "0") + }) + } +} + +func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "ID : vGPU Name\n") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "1159", "") + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Empty(t, profiles) + + _, err = sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.Error(t, err) + assert.ErrorContains(t, err, "GPUs may be at capacity") +} + +func TestVendorVFIOCreateReportsAmbiguousMissingProfile(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n") + + _, err := sysfs.create(context.Background(), "NVIDIA L40S-48Q", "instance-1") + require.Error(t, err) + assert.ErrorContains(t, err, "not creatable on any VF") + assert.ErrorContains(t, err, "unknown profile or insufficient capacity") +} + +func TestVendorVFIOSelectsLeastLoadedGPU(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) +} + +func TestVendorVFIOReconcile(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "1148", "") + + activeDevice := filepath.Join(sysfs.vfioDevicesPath, "vfio43") + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(activeDevice, filepath.Join(fdDir, "5"))) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "0") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcileSkipsProtectedVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "1148", "") + + protected := map[string]struct{}{ + filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4"): {}, + } + require.NoError(t, sysfs.reconcile(context.Background(), protected)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "1148") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIOReconcilePreservesLegacyGroupFD(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "1148", "") + + legacyGroup := filepath.Join(filepath.Dir(sysfs.vfioDevicesPath), "43") + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(legacyGroup, filepath.Join(fdDir, "5"))) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "0") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenVFIODeviceProbeFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + vfioDevPath := filepath.Join(sysfs.pciDevicesPath, vfAddress, "vfio-dev") + require.NoError(t, os.RemoveAll(vfioDevPath)) + require.NoError(t, os.WriteFile(vfioDevPath, nil, 0644)) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenIOMMUGroupProbeFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + iommuGroupPath := filepath.Join(sysfs.pciDevicesPath, vfAddress, "iommu_group") + require.NoError(t, os.Remove(iommuGroupPath)) + require.NoError(t, os.WriteFile(iommuGroupPath, nil, 0644)) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenProcFDDirectoryScanFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + processPath := filepath.Join(sysfs.procPath, "123") + require.NoError(t, os.MkdirAll(processPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(processPath, "fd"), nil, 0644)) + + err := sysfs.reconcile(context.Background(), nil) + require.Error(t, err) + assert.ErrorContains(t, err, "read process 123 file descriptors") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenProcFDLinkScanFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + fdPath := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(fdPath, "5"), nil, 0644)) + + err := sysfs.reconcile(context.Background(), nil) + require.Error(t, err) + assert.ErrorContains(t, err, "read process 123 file descriptor 5") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestParseCreatableVGPUTypesHeaderOnly(t *testing.T) { + t.Parallel() + + profiles, err := parseCreatableVGPUTypes("ID : vGPU Name\n") + require.NoError(t, err) + assert.Empty(t, profiles) +} + +func TestParseCreatableVGPUTypesRejectsMalformedLine(t *testing.T) { + t.Parallel() + + _, err := parseCreatableVGPUTypes("NVIDIA") + require.Error(t, err) + + _, err = parseCreatableVGPUTypes("not-an-id : NVIDIA L40S-1Q") + require.Error(t, err) +} + +func TestRollbackVendorVFIOCreate(t *testing.T) { + t.Parallel() + + verifyErr := errors.New("verification failed") + t.Run("preserves verification error", func(t *testing.T) { + currentTypePath := filepath.Join(t.TempDir(), "current_vgpu_type") + require.NoError(t, os.WriteFile(currentTypePath, []byte("1148"), 0644)) + + err := rollbackVendorVFIOCreate(currentTypePath, "0000:82:00.4", verifyErr) + require.ErrorIs(t, err, verifyErr) + assertFileValue(t, currentTypePath, "0") + }) + + t.Run("surfaces rollback error", func(t *testing.T) { + currentTypePath := filepath.Join(t.TempDir(), "missing", "current_vgpu_type") + + err := rollbackVendorVFIOCreate(currentTypePath, "0000:82:00.4", verifyErr) + require.ErrorIs(t, err, verifyErr) + assert.ErrorContains(t, err, "roll back vGPU on VF 0000:82:00.4") + }) +} + +func profileAvailability(profiles []GPUProfile, name string) int { + for _, profile := range profiles { + if profile.Name == name { + return profile.Available + } + } + return -1 +} + +type testVendorVFIOSysfs struct { + vendorVFIOSysfs +} + +func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { + t.Helper() + root := t.TempDir() + pci := filepath.Join(root, "sys", "bus", "pci", "devices") + proc := filepath.Join(root, "proc") + vfio := filepath.Join(root, "dev", "vfio", "devices") + require.NoError(t, os.MkdirAll(pci, 0755)) + require.NoError(t, os.MkdirAll(proc, 0755)) + require.NoError(t, os.MkdirAll(vfio, 0755)) + return testVendorVFIOSysfs{vendorVFIOSysfs{ + pciDevicesPath: pci, + procPath: proc, + vfioDevicesPath: vfio, + owners: make(map[string]string), + }} +} + +func (s testVendorVFIOSysfs) addVF(t *testing.T, parent, address, vfioID, currentType, creatableTypes string) { + t.Helper() + parentPath := filepath.Join(s.pciDevicesPath, parent) + vfPath := filepath.Join(s.pciDevicesPath, address) + nvidiaPath := filepath.Join(vfPath, "nvidia") + require.NoError(t, os.MkdirAll(parentPath, 0755)) + require.NoError(t, os.MkdirAll(nvidiaPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "current_vgpu_type"), []byte(currentType), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "creatable_vgpu_types"), []byte(creatableTypes), 0444)) + require.NoError(t, os.Symlink(parentPath, filepath.Join(vfPath, "physfn"))) + vfioName := "vfio" + vfioID + require.NoError(t, os.MkdirAll(filepath.Join(vfPath, "vfio-dev", vfioName), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(s.vfioDevicesPath, vfioName), nil, 0600)) + require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "kernel", "iommu_groups", vfioID), filepath.Join(vfPath, "iommu_group"))) +} + +func assertFileValue(t *testing.T, path, expected string) { + t.Helper() + value, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, expected, string(value)) +} diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index eaf210b42..a4ccd2db6 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -8,31 +8,115 @@ import ( "path/filepath" ) +// DiscoverVGPU returns the host's active vGPU framework and virtual functions. +func DiscoverVGPU() (VGPUFramework, []VirtualFunction, error) { + return discoverVGPUWith(discoverMdevVFs, hostVendorVFIO.discoverVFs) +} + +func discoverVGPUWith(discoverMdev, discoverVendorVFIO func() ([]VirtualFunction, error)) (VGPUFramework, []VirtualFunction, error) { + vfs, err := discoverMdev() + if err != nil { + return VGPUFrameworkNone, nil, fmt.Errorf("discover mdev VFs: %w", err) + } + if len(vfs) > 0 { + return VGPUFrameworkMdev, vfs, nil + } + + vfs, err = discoverVendorVFIO() + if err != nil { + return VGPUFrameworkNone, nil, fmt.Errorf("discover vendor VFIO VFs: %w", err) + } + if len(vfs) == 0 { + return VGPUFrameworkNone, nil, nil + } + return VGPUFrameworkVendorVFIO, vfs, nil +} + +// ListGPUProfiles returns available vGPU profiles with availability counts. +func ListGPUProfiles() ([]GPUProfile, error) { + framework, vfs, err := DiscoverVGPU() + if err != nil { + return nil, err + } + return ListGPUProfilesWithVFs(framework, vfs) +} + +// ListGPUProfilesWithVFs returns available profiles for discovered VFs. +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) { + switch framework { + case VGPUFrameworkMdev: + return listMdevGPUProfilesWithVFs(vfs) + case VGPUFrameworkVendorVFIO: + return hostVendorVFIO.listProfiles(vfs) + default: + return nil, nil + } +} + func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { - mdev, err := CreateMdev(ctx, profileName, instanceID) + framework, _, err := DiscoverVGPU() if err != nil { return nil, err } - return &VGPUDevice{ - Framework: VGPUFrameworkMdev, - VFAddress: mdev.VFAddress, - ProfileType: mdev.ProfileType, - ProfileName: mdev.ProfileName, - SysfsPath: mdev.SysfsPath, - MdevUUID: mdev.UUID, - }, nil + switch framework { + case VGPUFrameworkMdev: + mdev, err := CreateMdev(ctx, profileName, instanceID) + if err != nil { + return nil, err + } + return &VGPUDevice{ + Framework: VGPUFrameworkMdev, + VFAddress: mdev.VFAddress, + ProfileType: mdev.ProfileType, + ProfileName: mdev.ProfileName, + SysfsPath: mdev.SysfsPath, + MdevUUID: mdev.UUID, + }, nil + case VGPUFrameworkVendorVFIO: + return hostVendorVFIO.create(ctx, profileName, instanceID) + default: + return nil, fmt.Errorf("vGPU framework not available") + } } func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { - if assignment.Framework != VGPUFrameworkNone && assignment.Framework != VGPUFrameworkMdev { - return fmt.Errorf("unknown vGPU framework %q", assignment.Framework) + framework := assignment.Framework + if framework == VGPUFrameworkNone && assignment.MdevUUID != "" { + framework = VGPUFrameworkMdev + } + + switch framework { + case VGPUFrameworkMdev: + mdevUUID := assignment.MdevUUID + if mdevUUID == "" { + mdevUUID = filepath.Base(assignment.DevicePath) + } + return DestroyMdev(ctx, mdevUUID) + case VGPUFrameworkVendorVFIO: + return hostVendorVFIO.destroy(ctx, filepath.Base(assignment.DevicePath), assignment.InstanceID) + case VGPUFrameworkNone: + return nil + default: + return fmt.Errorf("unknown vGPU framework %q", framework) } - mdevUUID := assignment.MdevUUID - if mdevUUID == "" { - if assignment.DevicePath == "" { +} + +// ReconcileVGPUs releases orphaned vGPU assignments. +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { + framework, _, err := DiscoverVGPU() + if err != nil { + return err + } + + switch framework { + case VGPUFrameworkMdev: + return ReconcileMdevs(ctx, nil) + case VGPUFrameworkVendorVFIO: + if protectedDevicePaths == nil { return nil } - mdevUUID = filepath.Base(assignment.DevicePath) + return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) + default: + return nil } - return DestroyMdev(ctx, mdevUUID) } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go new file mode 100644 index 000000000..805f8da95 --- /dev/null +++ b/lib/devices/vgpu_linux_test.go @@ -0,0 +1,50 @@ +//go:build linux + +package devices + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { + t.Parallel() + + discoveryErr := errors.New("mdev discovery failed") + vendorCalled := false + framework, vfs, err := discoverVGPUWith( + func() ([]VirtualFunction, error) { + return nil, discoveryErr + }, + func() ([]VirtualFunction, error) { + vendorCalled = true + return []VirtualFunction{{PCIAddress: "0000:82:00.4"}}, nil + }, + ) + + require.ErrorIs(t, err, discoveryErr) + assert.Equal(t, VGPUFrameworkNone, framework) + assert.Nil(t, vfs) + assert.False(t, vendorCalled) +} + +func TestDiscoverVGPUWithPropagatesVendorVFIOError(t *testing.T) { + t.Parallel() + + discoveryErr := errors.New("vendor VFIO discovery failed") + framework, vfs, err := discoverVGPUWith( + func() ([]VirtualFunction, error) { + return nil, nil + }, + func() ([]VirtualFunction, error) { + return nil, discoveryErr + }, + ) + + require.ErrorIs(t, err, discoveryErr) + assert.Equal(t, VGPUFrameworkNone, framework) + assert.Nil(t, vfs) +} diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 78788412e..4069692c9 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -1,7 +1,10 @@ package resources import ( + "context" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" ) // GPUResourceStatus represents the GPU resource status for the API response. @@ -16,31 +19,22 @@ type GPUResourceStatus struct { // GetGPUStatus returns the current GPU resource status. // Returns nil if no GPU is available or the mode is "none". -func GetGPUStatus() *GPUResourceStatus { - mode := devices.DetectHostGPUMode() - if mode == devices.GPUModeNone { +func GetGPUStatus(ctx context.Context) *GPUResourceStatus { + framework, vfs, err := devices.DiscoverVGPU() + if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) return nil } - - switch mode { - case devices.GPUModeVGPU: - return getVGPUStatus() - case devices.GPUModePassthrough: - return getPassthroughStatus() - default: - return nil + if framework != devices.VGPUFrameworkNone { + return getVGPUStatus(ctx, framework, vfs) } + return getPassthroughStatus() } -// getVGPUStatus returns GPU status for vGPU mode (SR-IOV + mdev). -func getVGPUStatus() *GPUResourceStatus { - vfs, err := devices.DiscoverVFs() - if err != nil || len(vfs) == 0 { - return nil - } - - // Count used VFs (those with mdevs) +// getVGPUStatus returns GPU status for vGPU mode (SR-IOV). +func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) *GPUResourceStatus { usedSlots := 0 + // Count used VFs (those with a vGPU assigned) for _, vf := range vfs { if vf.Allocated { usedSlots++ @@ -48,8 +42,9 @@ func getVGPUStatus() *GPUResourceStatus { } // Get available profiles (reuse VFs to avoid redundant discovery) - profiles, err := devices.ListGPUProfilesWithVFs(vfs) + profiles, err := devices.ListGPUProfilesWithVFs(framework, vfs) if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to list vGPU profiles; reporting none", "framework", framework, "error", err) profiles = nil } diff --git a/lib/resources/monitoring_test.go b/lib/resources/monitoring_test.go index ab6c3e4dd..bef0740dc 100644 --- a/lib/resources/monitoring_test.go +++ b/lib/resources/monitoring_test.go @@ -198,7 +198,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { mgr, _, _ := monitoringTestManager(t) originalProvider := currentGPUStatusProvider() - setGPUStatusProvider(func() *GPUResourceStatus { + setGPUStatusProvider(func(context.Context) *GPUResourceStatus { return &GPUResourceStatus{ Mode: "vgpu", TotalSlots: 8, diff --git a/lib/resources/resource.go b/lib/resources/resource.go index caaf5ba50..86f644bda 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -37,13 +37,13 @@ var ( gpuStatusProvider = GetGPUStatus ) -func currentGPUStatusProvider() func() *GPUResourceStatus { +func currentGPUStatusProvider() func(context.Context) *GPUResourceStatus { gpuStatusProviderMu.RLock() defer gpuStatusProviderMu.RUnlock() return gpuStatusProvider } -func setGPUStatusProvider(fn func() *GPUResourceStatus) { +func setGPUStatusProvider(fn func(context.Context) *GPUResourceStatus) { if fn == nil { fn = GetGPUStatus } @@ -427,7 +427,7 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error } // Get GPU status - gpuStatus := currentGPUStatusProvider()() + gpuStatus := currentGPUStatusProvider()(ctx) return &FullResourceStatus{ CPU: *cpuStatus, @@ -691,7 +691,7 @@ func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string // Check GPU if needed if req.GPUSlots > 0 { - gpuStatus := currentGPUStatusProvider()() + gpuStatus := currentGPUStatusProvider()(ctx) if gpuStatus == nil { return fmt.Errorf("insufficient GPU: no GPU available on this host") } From eca8d4ed64c3534f15857e3b529919dcc0243406 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:25:31 +0000 Subject: [PATCH 02/19] Account for consumed vGPU profiles --- lib/devices/vendor_vfio_linux.go | 34 ++++++++++++++------------- lib/devices/vendor_vfio_linux_test.go | 32 +++++++++++++++++++++---- 2 files changed, 46 insertions(+), 20 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 23ab1e89a..4d056db9e 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -22,18 +22,20 @@ const ( ) type vendorVFIOSysfs struct { - pciDevicesPath string - procPath string - vfioDevicesPath string - owners map[string]string + pciDevicesPath string + procPath string + vfioDevicesPath string + owners map[string]string + framebufferByType map[string]int } var ( hostVendorVFIO = vendorVFIOSysfs{ - pciDevicesPath: pciDevicesPath, - procPath: procPath, - vfioDevicesPath: vfioDevicesPath, - owners: make(map[string]string), + pciDevicesPath: pciDevicesPath, + procPath: procPath, + vfioDevicesPath: vfioDevicesPath, + owners: make(map[string]string), + framebufferByType: make(map[string]int), } vendorVFIOMu sync.Mutex ) @@ -141,7 +143,7 @@ func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID str return nil, fmt.Errorf("profile %q is not creatable on any VF (unknown profile or insufficient capacity)", profileName) } - targetVF, err := s.selectLeastLoadedVF(vfs, metadata, requested.TypeName) + targetVF, err := s.selectLeastLoadedVF(vfs, requested.TypeName) if err != nil { return nil, err } @@ -270,17 +272,16 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map return nil } -func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, metadata []profileMetadata, profileType string) (string, error) { - framebufferByType := make(map[string]int, len(metadata)) - for _, profile := range metadata { - framebufferByType[profile.TypeName] = profile.FramebufferMB - } - +func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) { usageByGPU := make(map[string]int) freeByGPU := make(map[string][]VirtualFunction) for _, vf := range vfs { if vf.Allocated { - usageByGPU[vf.ParentGPU] += framebufferByType[vf.ProfileType] + framebuffer, ok := s.framebufferByType[vf.ProfileType] + if !ok { + return "", fmt.Errorf("framebuffer size for allocated vGPU type %s is unknown", vf.ProfileType) + } + usageByGPU[vf.ParentGPU] += framebuffer continue } profiles, err := s.readCreatableProfiles(vf.PCIAddress) @@ -320,6 +321,7 @@ func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetada } for _, profile := range profiles { profilesByType[profile.TypeName] = profile + s.framebufferByType[profile.TypeName] = profile.FramebufferMB } } profiles := make([]profileMetadata, 0, len(profilesByType)) diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 8e79a7750..9afd56292 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -169,6 +169,29 @@ func TestVendorVFIOSelectsLeastLoadedGPU(t *testing.T) { assert.Equal(t, "0000:e3:00.4", device.VFAddress) } +func TestVendorVFIOSelectsLeastLoadedGPUWithConsumedType(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + _, err = sysfs.profileMetadata(vfs) + require.NoError(t, err) + for _, vfAddress := range []string{"0000:82:00.5", "0000:e3:00.4"} { + creatableTypesPath := filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "creatable_vgpu_types") + require.NoError(t, os.Chmod(creatableTypesPath, 0644)) + require.NoError(t, os.WriteFile(creatableTypesPath, []byte("1147 : NVIDIA L40S-1Q\n"), 0444)) + } + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) +} + func TestVendorVFIOReconcile(t *testing.T) { t.Parallel() @@ -343,10 +366,11 @@ func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { require.NoError(t, os.MkdirAll(proc, 0755)) require.NoError(t, os.MkdirAll(vfio, 0755)) return testVendorVFIOSysfs{vendorVFIOSysfs{ - pciDevicesPath: pci, - procPath: proc, - vfioDevicesPath: vfio, - owners: make(map[string]string), + pciDevicesPath: pci, + procPath: proc, + vfioDevicesPath: vfio, + owners: make(map[string]string), + framebufferByType: make(map[string]int), }} } From fed4d10a1be7c63c948e9a22ae7dd2cbd845ee43 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:25:31 +0000 Subject: [PATCH 03/19] Reject unowned vendor VFIO releases --- lib/devices/vendor_vfio_linux.go | 19 ++++++++++++------- lib/devices/vendor_vfio_linux_test.go | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 4d056db9e..f702f4591 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -203,13 +203,18 @@ func (s vendorVFIOSysfs) destroyWithOpenPaths(ctx context.Context, vfAddress, in return nil } - if owner, ok := s.owners[vfAddress]; ok && (instanceID == "" || owner != instanceID) { - log.WarnContext(ctx, "skipping vendor VFIO vGPU release owned by another instance", - "vf", vfAddress, - "owner_instance_id", owner, - "requesting_instance_id", instanceID, - ) - return nil + if owner, ok := s.owners[vfAddress]; ok { + if instanceID == "" { + return fmt.Errorf("cannot release vendor VFIO vGPU on VF %s without instance ID", vfAddress) + } + if owner != instanceID { + log.WarnContext(ctx, "skipping vendor VFIO vGPU release owned by another instance", + "vf", vfAddress, + "owner_instance_id", owner, + "requesting_instance_id", instanceID, + ) + return nil + } } if openPaths == nil { diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 9afd56292..10b87b081 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -72,6 +72,20 @@ func TestVendorVFIODestroySkipsAssignmentOwnedByAnotherInstance(t *testing.T) { assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "0") } +func TestVendorVFIODestroyRejectsMissingInstanceIDForOwnedVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.NoError(t, err) + + err = sysfs.destroy(context.Background(), device.VFAddress, "") + require.ErrorContains(t, err, "without instance ID") + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "1148") +} + func TestVendorVFIODestroyRetainsAssignmentInUse(t *testing.T) { t.Parallel() From edcaa094f6e759e920ef882aa28a708eaa66e0a6 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:25:37 +0000 Subject: [PATCH 04/19] Check all vendor VFIO device paths --- lib/devices/vendor_vfio_linux.go | 14 ++++---- lib/devices/vendor_vfio_linux_test.go | 46 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index f702f4591..6841c5294 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -351,11 +351,10 @@ func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string] probeErrs := make([]error, 0, 2) vfioDevices, err := os.ReadDir(filepath.Join(s.pciDevicesPath, vfAddress, "vfio-dev")) - if os.IsNotExist(err) { - return false, nil - } if err != nil { - probeErrs = append(probeErrs, fmt.Errorf("read VFIO devices for VF %s: %w", vfAddress, err)) + if !os.IsNotExist(err) { + probeErrs = append(probeErrs, fmt.Errorf("read VFIO devices for VF %s: %w", vfAddress, err)) + } } else { for _, device := range vfioDevices { devicePaths = append(devicePaths, filepath.Join(s.vfioDevicesPath, device.Name())) @@ -363,11 +362,10 @@ func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string] } target, err := os.Readlink(filepath.Join(s.pciDevicesPath, vfAddress, "iommu_group")) - if os.IsNotExist(err) { - return false, nil - } if err != nil { - probeErrs = append(probeErrs, fmt.Errorf("read IOMMU group for VF %s: %w", vfAddress, err)) + if !os.IsNotExist(err) { + probeErrs = append(probeErrs, fmt.Errorf("read IOMMU group for VF %s: %w", vfAddress, err)) + } } else { devicePaths = append(devicePaths, filepath.Join(filepath.Dir(s.vfioDevicesPath), filepath.Base(target))) } diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 10b87b081..8dc6e6558 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -138,6 +138,52 @@ func TestVendorVFIODestroyReleasesUnboundVF(t *testing.T) { } } +func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + remove func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) + open func(sysfs testVendorVFIOSysfs) string + }{ + { + name: "missing iommu group", + remove: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.Remove(filepath.Join(sysfs.pciDevicesPath, vfAddress, "iommu_group"))) + }, + open: func(sysfs testVendorVFIOSysfs) string { + return filepath.Join(sysfs.vfioDevicesPath, "vfio42") + }, + }, + { + name: "missing vfio device directory", + remove: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.RemoveAll(filepath.Join(sysfs.pciDevicesPath, vfAddress, "vfio-dev"))) + }, + open: func(sysfs testVendorVFIOSysfs) string { + return filepath.Join(filepath.Dir(sysfs.vfioDevicesPath), "42") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + tt.remove(t, sysfs, vfAddress) + + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(tt.open(sysfs), filepath.Join(fdDir, "5"))) + + err := sysfs.destroy(context.Background(), vfAddress, "instance-1") + require.ErrorContains(t, err, "still in use") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") + }) + } +} + func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { t.Parallel() From cedda98bc3ab3abd3596889194d6bdb265c317e0 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:25:37 +0000 Subject: [PATCH 05/19] Require usable mdev types for discovery --- lib/devices/mdev_linux.go | 27 ++++++++++++++++++++++++--- lib/devices/vgpu_linux_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index e2891efc9..61e55b599 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -92,7 +92,11 @@ func getCachedProfiles(firstVF string) []profileMetadata { // discoverMdevVFs returns all SR-IOV Virtual Functions available for vGPU, // discovered by scanning /sys/class/mdev_bus/. func discoverMdevVFs() ([]VirtualFunction, error) { - entries, err := os.ReadDir(mdevBusPath) + return discoverMdevVFsWith(mdevBusPath, pciDevicesPath, ListMdevDevices) +} + +func discoverMdevVFsWith(busPath, pciPath string, listMdevs func() ([]MdevDevice, error)) ([]VirtualFunction, error) { + entries, err := os.ReadDir(busPath) if err != nil { if os.IsNotExist(err) { return nil, nil // No mdev_bus means no mdev vGPU support @@ -101,7 +105,7 @@ func discoverMdevVFs() ([]VirtualFunction, error) { } // List mdevs once and build a lookup map to avoid O(n*m) performance - mdevs, _ := ListMdevDevices() + mdevs, _ := listMdevs() mdevByVF := make(map[string]bool, len(mdevs)) for _, mdev := range mdevs { mdevByVF[mdev.VFAddress] = true @@ -110,10 +114,27 @@ func discoverMdevVFs() ([]VirtualFunction, error) { var vfs []VirtualFunction for _, entry := range entries { vfAddr := entry.Name() + types, err := os.ReadDir(filepath.Join(busPath, vfAddr, "mdev_supported_types")) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("read mdev supported types for VF %s: %w", vfAddr, err) + } + usable := false + for _, typ := range types { + if typ.IsDir() { + usable = true + break + } + } + if !usable { + continue + } // Find parent GPU by checking physfn symlink // VFs have a physfn symlink pointing to their parent Physical Function - physfnPath := filepath.Join("/sys/bus/pci/devices", vfAddr, "physfn") + physfnPath := filepath.Join(pciPath, vfAddr, "physfn") parentGPU := "" if target, err := os.Readlink(physfnPath); err == nil { parentGPU = filepath.Base(target) diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go index 805f8da95..40b987394 100644 --- a/lib/devices/vgpu_linux_test.go +++ b/lib/devices/vgpu_linux_test.go @@ -4,6 +4,8 @@ package devices import ( "errors" + "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -31,6 +33,29 @@ func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { assert.False(t, vendorCalled) } +func TestDiscoverVGPUWithFallsBackFromTypelessMdevBus(t *testing.T) { + t.Parallel() + + root := t.TempDir() + busPath := filepath.Join(root, "sys", "class", "mdev_bus") + require.NoError(t, os.MkdirAll(filepath.Join(busPath, "0000:82:00.4", "mdev_supported_types"), 0755)) + + framework, vfs, err := discoverVGPUWith( + func() ([]VirtualFunction, error) { + return discoverMdevVFsWith(busPath, filepath.Join(root, "sys", "bus", "pci", "devices"), func() ([]MdevDevice, error) { + return nil, nil + }) + }, + func() ([]VirtualFunction, error) { + return []VirtualFunction{{PCIAddress: "0000:82:00.4"}}, nil + }, + ) + + require.NoError(t, err) + assert.Equal(t, VGPUFrameworkVendorVFIO, framework) + assert.Equal(t, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}, vfs) +} + func TestDiscoverVGPUWithPropagatesVendorVFIOError(t *testing.T) { t.Parallel() From 9d4d7b578d0bf1f4969ede33f0b147d8d2c02f90 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:50:10 +0000 Subject: [PATCH 06/19] Test retained vendor VFIO assignments --- integration/vgpu_test.go | 50 ++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 3f8fdfd65..ce0da7fb2 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -89,6 +89,14 @@ func TestVGPU(t *testing.T) { // Cleanup any orphaned instances and mdevs t.Cleanup(func() { if instanceID != "" { + if inst, err := instanceManager.GetInstance(ctx, instanceID); err == nil && inst.GPUFramework == devices.VGPUFrameworkVendorVFIO && inst.GPUDevicePath != "" { + err := devices.DestroyVGPU(ctx, devices.VGPUAssignment{ + Framework: inst.GPUFramework, + DevicePath: inst.GPUDevicePath, + InstanceID: instanceID, + }) + require.NoError(t, err, "cleanup should release vendor VFIO vGPU") + } t.Log("Cleanup: Deleting instance...") instanceManager.DeleteInstance(ctx, instanceID) } @@ -243,26 +251,38 @@ func TestVGPU(t *testing.T) { t.Logf("Instance GPU: profile=%s, framework=%s, device=%s", actualInst.GPUProfile, actualInst.GPUFramework, actualInst.GPUDevicePath) }) - t.Log("Step 10: Stopping instance to release the vGPU...") + t.Log("Step 10: Stopping instance...") _, err = instanceManager.StopInstance(ctx, inst.Id) require.NoError(t, err, "stop should succeed") - t.Run("VGPUReleasedOnStop", func(t *testing.T) { - stopped, err := instanceManager.GetInstance(ctx, inst.Id) - require.NoError(t, err) - assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") - assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) - }) + switch inst.GPUFramework { + case devices.VGPUFrameworkMdev: + t.Run("VGPUReleasedOnStop", func(t *testing.T) { + stopped, err := instanceManager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") + assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) + }) - t.Log("Step 11: Starting instance to reacquire a vGPU...") - started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) - require.NoError(t, err, "start should succeed") + t.Log("Step 11: Starting instance to reacquire a vGPU...") + started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) + require.NoError(t, err, "start should succeed") - t.Run("VGPUReacquiredOnStart", func(t *testing.T) { - require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") - assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") - assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) - }) + t.Run("VGPUReacquiredOnStart", func(t *testing.T) { + require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") + assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") + assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) + }) + case devices.VGPUFrameworkVendorVFIO: + t.Run("VGPUAssignmentRetainedOnStop", func(t *testing.T) { + stopped, err := instanceManager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + // Release requires an instance-owned assignment, so stop retains it. + assert.Equal(t, inst.GPUFramework, stopped.GPUFramework, "assignment framework should be retained on stop") + assert.Equal(t, inst.GPUDevicePath, stopped.GPUDevicePath, "assignment metadata should be retained on stop") + assertVGPUAssigned(t, stopped.GPUFramework, stopped.GPUDevicePath) + }) + } t.Log("✅ vGPU test PASSED!") } From bed15f0b46c63d5631324e9af591d562e1f6f232 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:08:51 +0000 Subject: [PATCH 07/19] Always delete the test instance during vGPU cleanup --- integration/vgpu_test.go | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index ce0da7fb2..059513d6e 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -36,10 +36,10 @@ import ( // // sudo go test -v -run TestVGPU -timeout 5m ./integration/... // -// Note: This test verifies vGPU assignment, release on stop, reacquisition on -// start, and PCI device visibility inside the VM. It does NOT test nvidia-smi -// or CUDA functionality since that requires NVIDIA guest drivers pre-installed -// in the image. +// Note: This test verifies vGPU assignment, stop behavior (mdev releases and +// reacquires on start; vendor VFIO retains the assignment), and PCI device +// visibility inside the VM. It does NOT test nvidia-smi or CUDA functionality +// since that requires NVIDIA guest drivers pre-installed in the image. func TestVGPU(t *testing.T) { t.Parallel() if testing.Short() { @@ -88,18 +88,23 @@ func TestVGPU(t *testing.T) { // Cleanup any orphaned instances and mdevs t.Cleanup(func() { - if instanceID != "" { - if inst, err := instanceManager.GetInstance(ctx, instanceID); err == nil && inst.GPUFramework == devices.VGPUFrameworkVendorVFIO && inst.GPUDevicePath != "" { - err := devices.DestroyVGPU(ctx, devices.VGPUAssignment{ - Framework: inst.GPUFramework, - DevicePath: inst.GPUDevicePath, - InstanceID: instanceID, - }) - require.NoError(t, err, "cleanup should release vendor VFIO vGPU") + if instanceID == "" { + return + } + if _, err := instanceManager.StopInstance(ctx, instanceID); err != nil { + t.Logf("Cleanup: stop instance: %v", err) + } + if inst, err := instanceManager.GetInstance(ctx, instanceID); err == nil && inst.GPUFramework == devices.VGPUFrameworkVendorVFIO && inst.GPUDevicePath != "" { + if err := devices.DestroyVGPU(ctx, devices.VGPUAssignment{ + Framework: inst.GPUFramework, + DevicePath: inst.GPUDevicePath, + InstanceID: instanceID, + }); err != nil { + t.Errorf("cleanup: release vendor VFIO vGPU: %v", err) } - t.Log("Cleanup: Deleting instance...") - instanceManager.DeleteInstance(ctx, instanceID) } + t.Log("Cleanup: Deleting instance...") + instanceManager.DeleteInstance(ctx, instanceID) }) // Step 1: Ensure system files (kernel, initrd) From 4ceca745ee42195fde66309078cac1718afcb4f0 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:27:38 +0000 Subject: [PATCH 08/19] Fall back to passthrough status when vGPU discovery fails --- lib/resources/gpu.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 4069692c9..6a34de5cc 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -22,8 +22,9 @@ type GPUResourceStatus struct { func GetGPUStatus(ctx context.Context) *GPUResourceStatus { framework, vfs, err := devices.DiscoverVGPU() if err != nil { + // A failed vGPU probe must not hide passthrough GPUs from status reporting. logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) - return nil + return getPassthroughStatus() } if framework != devices.VGPUFrameworkNone { return getVGPUStatus(ctx, framework, vfs) From 6a269ed8627bcca2ee73323fc8979cac3c418c9a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:39:31 +0000 Subject: [PATCH 09/19] Keep vGPU placement available when an allocated type is unknown Sort GPUs with unaccountable load last instead of rejecting placement, and stop reporting passthrough capacity when vGPU discovery fails. --- lib/devices/vendor_vfio_linux.go | 12 +++++++++++- lib/devices/vendor_vfio_linux_test.go | 27 +++++++++++++++++++++++++++ lib/resources/gpu.go | 7 +++++-- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 6841c5294..78526a45e 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -279,12 +279,19 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) { usageByGPU := make(map[string]int) + unknownUsageByGPU := make(map[string]bool) freeByGPU := make(map[string][]VirtualFunction) for _, vf := range vfs { if vf.Allocated { + // framebufferByType only covers currently creatable profiles, so + // after a restart an allocated type can be missing when its + // capacity is exhausted. Prefer GPUs whose load is fully known + // instead of rejecting placement outright; the kernel driver + // still enforces real capacity through creatable_vgpu_types. framebuffer, ok := s.framebufferByType[vf.ProfileType] if !ok { - return "", fmt.Errorf("framebuffer size for allocated vGPU type %s is unknown", vf.ProfileType) + unknownUsageByGPU[vf.ParentGPU] = true + continue } usageByGPU[vf.ParentGPU] += framebuffer continue @@ -306,6 +313,9 @@ func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType gpus = append(gpus, gpu) } sort.Slice(gpus, func(i, j int) bool { + if unknownUsageByGPU[gpus[i]] != unknownUsageByGPU[gpus[j]] { + return !unknownUsageByGPU[gpus[i]] + } if usageByGPU[gpus[i]] == usageByGPU[gpus[j]] { return gpus[i] < gpus[j] } diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 8dc6e6558..ad2fc2a35 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -252,6 +252,33 @@ func TestVendorVFIOSelectsLeastLoadedGPUWithConsumedType(t *testing.T) { assert.Equal(t, "0000:e3:00.4", device.VFAddress) } +func TestVendorVFIOPlacementPrefersKnownLoadWhenAllocatedTypeIsUnknown(t *testing.T) { + t.Parallel() + + // Simulates a restart: type 1159 is allocated but no longer creatable + // anywhere, so its framebuffer size is unknown. + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "1147 : NVIDIA L40S-1Q\n") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", "1147 : NVIDIA L40S-1Q\n") + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress, "the GPU with unknown load should be picked last") +} + +func TestVendorVFIOPlacesOnGPUWithUnknownLoadWhenItHasTheOnlyCapacity(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "1147 : NVIDIA L40S-1Q\n") + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + func TestVendorVFIOReconcile(t *testing.T) { t.Parallel() diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 6a34de5cc..054e3744e 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -22,9 +22,12 @@ type GPUResourceStatus struct { func GetGPUStatus(ctx context.Context) *GPUResourceStatus { framework, vfs, err := devices.DiscoverVGPU() if err != nil { - // A failed vGPU probe must not hide passthrough GPUs from status reporting. + // Only report passthrough once vGPU discovery confirms no vGPU + // framework. On a vGPU host a transient probe failure would otherwise + // expose the PFs/VFs as available passthrough slots while active vGPU + // assignments exist. logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) - return getPassthroughStatus() + return nil } if framework != devices.VGPUFrameworkNone { return getVGPUStatus(ctx, framework, vfs) From aa29f11d878e48c6bf91c9c2e22ddf14c050fe8b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sat, 8 Aug 2026 00:53:16 +0000 Subject: [PATCH 10/19] Report vendor VFIO availability per parent GPU and thread instance ownership through releases --- lib/devices/vendor_vfio_linux.go | 17 ++++++++++++++--- lib/devices/vendor_vfio_linux_test.go | 16 ++++++++++++++++ lib/instances/create.go | 1 + lib/instances/start.go | 1 + lib/instances/vgpu.go | 1 + 5 files changed, 33 insertions(+), 3 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 78526a45e..6f59bed3d 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -81,9 +81,13 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } +// listProfiles aggregates creatable profiles per parent GPU. Free VFs on the +// same GPU share its framebuffer, so counting each advertising VF overreports +// availability. The driver only guarantees that a GPU still advertising a +// type can fit one more instance of it, so report that per-GPU lower bound. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { profilesByType := make(map[string]profileMetadata) - availability := make(map[string]int) + creatableGPUs := make(map[string]map[string]struct{}) for _, vf := range vfs { creatable, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { @@ -92,7 +96,14 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro for _, profile := range creatable { profilesByType[profile.TypeName] = profile if !vf.Allocated { - availability[profile.TypeName]++ + gpu := vf.ParentGPU + if gpu == "" { + gpu = vf.PCIAddress + } + if creatableGPUs[profile.TypeName] == nil { + creatableGPUs[profile.TypeName] = make(map[string]struct{}) + } + creatableGPUs[profile.TypeName][gpu] = struct{}{} } } } @@ -108,7 +119,7 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro profiles = append(profiles, GPUProfile{ Name: profile.Name, FramebufferMB: profile.FramebufferMB, - Available: availability[profile.TypeName], + Available: len(creatableGPUs[profile.TypeName]), }) } return profiles, nil diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index ad2fc2a35..51668ed11 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -184,6 +184,22 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T } } +func TestVendorVFIOListProfilesCountsPerGPUNotPerVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"), + "free VFs share their parent GPU's capacity, so availability is per GPU") +} + func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { t.Parallel() diff --git a/lib/instances/create.go b/lib/instances/create.go index 7f1438fbb..f24519a9a 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -301,6 +301,7 @@ func (m *manager) createInstance( Framework: gpuDevice.Framework, DevicePath: gpuDevice.SysfsPath, MdevUUID: gpuDevice.MdevUUID, + InstanceID: id, } if err := devices.DestroyVGPU(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) diff --git a/lib/instances/start.go b/lib/instances/start.go index 7e7855eac..a6c832450 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -177,6 +177,7 @@ func (m *manager) startInstance( Framework: device.Framework, DevicePath: device.SysfsPath, MdevUUID: device.MdevUUID, + InstanceID: id, } if err := devices.DestroyVGPU(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index cffe2ac1d..a8ca6aceb 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -27,6 +27,7 @@ func releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { Framework: stored.GPUFramework, DevicePath: path, MdevUUID: stored.GPUMdevUUID, + InstanceID: stored.Id, } if err := devices.DestroyVGPU(ctx, assignment); err != nil { return err From 6abdd17945e34522bbf006adcbe124a5ae74d9fc Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 06:18:53 +0000 Subject: [PATCH 11/19] Keep vendor VFIO out of the create path until lifecycle integration The instance lifecycle already routes create/start/stop/delete through CreateVGPU/DestroyVGPU, so dispatching vendor VFIO creates here would activate the backend before assignment durability and release guards exist. Reject vendor VFIO creates for now; destroy stays wired so existing assignments remain releasable. The integration test skips on vendor VFIO hosts at this layer and no longer asserts the transitional stop-retention behavior. --- integration/vgpu_test.go | 66 +++++++++++++++------------------------ lib/devices/vgpu_linux.go | 5 ++- 2 files changed, 30 insertions(+), 41 deletions(-) diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 059513d6e..7873aa35c 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -36,10 +36,10 @@ import ( // // sudo go test -v -run TestVGPU -timeout 5m ./integration/... // -// Note: This test verifies vGPU assignment, stop behavior (mdev releases and -// reacquires on start; vendor VFIO retains the assignment), and PCI device -// visibility inside the VM. It does NOT test nvidia-smi or CUDA functionality -// since that requires NVIDIA guest drivers pre-installed in the image. +// Note: This test verifies vGPU assignment, release on stop, reacquisition on +// start, and PCI device visibility inside the VM. It does NOT test nvidia-smi +// or CUDA functionality since that requires NVIDIA guest drivers pre-installed +// in the image. func TestVGPU(t *testing.T) { t.Parallel() if testing.Short() { @@ -94,17 +94,10 @@ func TestVGPU(t *testing.T) { if _, err := instanceManager.StopInstance(ctx, instanceID); err != nil { t.Logf("Cleanup: stop instance: %v", err) } - if inst, err := instanceManager.GetInstance(ctx, instanceID); err == nil && inst.GPUFramework == devices.VGPUFrameworkVendorVFIO && inst.GPUDevicePath != "" { - if err := devices.DestroyVGPU(ctx, devices.VGPUAssignment{ - Framework: inst.GPUFramework, - DevicePath: inst.GPUDevicePath, - InstanceID: instanceID, - }); err != nil { - t.Errorf("cleanup: release vendor VFIO vGPU: %v", err) - } - } t.Log("Cleanup: Deleting instance...") - instanceManager.DeleteInstance(ctx, instanceID) + if err := instanceManager.DeleteInstance(ctx, instanceID); err != nil { + t.Errorf("cleanup: delete instance: %v", err) + } }) // Step 1: Ensure system files (kernel, initrd) @@ -260,34 +253,22 @@ func TestVGPU(t *testing.T) { _, err = instanceManager.StopInstance(ctx, inst.Id) require.NoError(t, err, "stop should succeed") - switch inst.GPUFramework { - case devices.VGPUFrameworkMdev: - t.Run("VGPUReleasedOnStop", func(t *testing.T) { - stopped, err := instanceManager.GetInstance(ctx, inst.Id) - require.NoError(t, err) - assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") - assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) - }) + t.Run("VGPUReleasedOnStop", func(t *testing.T) { + stopped, err := instanceManager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") + assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) + }) - t.Log("Step 11: Starting instance to reacquire a vGPU...") - started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) - require.NoError(t, err, "start should succeed") + t.Log("Step 11: Starting instance to reacquire a vGPU...") + started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) + require.NoError(t, err, "start should succeed") - t.Run("VGPUReacquiredOnStart", func(t *testing.T) { - require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") - assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") - assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) - }) - case devices.VGPUFrameworkVendorVFIO: - t.Run("VGPUAssignmentRetainedOnStop", func(t *testing.T) { - stopped, err := instanceManager.GetInstance(ctx, inst.Id) - require.NoError(t, err) - // Release requires an instance-owned assignment, so stop retains it. - assert.Equal(t, inst.GPUFramework, stopped.GPUFramework, "assignment framework should be retained on stop") - assert.Equal(t, inst.GPUDevicePath, stopped.GPUDevicePath, "assignment metadata should be retained on stop") - assertVGPUAssigned(t, stopped.GPUFramework, stopped.GPUDevicePath) - }) - } + t.Run("VGPUReacquiredOnStart", func(t *testing.T) { + require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") + assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") + assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) + }) t.Log("✅ vGPU test PASSED!") } @@ -343,6 +324,11 @@ func checkVGPUTestPrerequisites() (string, string) { if framework == devices.VGPUFrameworkNone { return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", "" } + if framework == devices.VGPUFrameworkVendorVFIO { + // CreateVGPU rejects vendor VFIO until the instance lifecycle + // integration lands. + return "vGPU test requires the vendor VFIO instance lifecycle integration", "" + } // Check for available profiles profiles, err := devices.ListGPUProfiles() diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index a4ccd2db6..72fe3b944 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -73,7 +73,10 @@ func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevic MdevUUID: mdev.UUID, }, nil case VGPUFrameworkVendorVFIO: - return hostVendorVFIO.create(ctx, profileName, instanceID) + // The instance lifecycle does not yet persist vendor VFIO assignments + // durably or guard their release against live claims, so keep the + // backend out of the create path until that integration lands. + return nil, fmt.Errorf("vendor VFIO vGPU support is not yet integrated with the instance lifecycle") default: return nil, fmt.Errorf("vGPU framework not available") } From 84bb8eea346a31e32531b8f34c521adbd6677b6b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 04:53:56 +0000 Subject: [PATCH 12/19] Report vendor VFIO profile availability per free VF --- lib/devices/vendor_vfio_linux.go | 21 +++++++-------------- lib/devices/vendor_vfio_linux_test.go | 6 +++--- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 6f59bed3d..ceab85ec5 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -81,13 +81,13 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// listProfiles aggregates creatable profiles per parent GPU. Free VFs on the -// same GPU share its framebuffer, so counting each advertising VF overreports -// availability. The driver only guarantees that a GPU still advertising a -// type can fit one more instance of it, so report that per-GPU lower bound. +// listProfiles counts each free VF advertising a type as one creatable +// instance, matching the driver-reported units that mdev sums through +// available_instances. This is a best-effort snapshot because creating on one +// VF may revoke the type from siblings that share its GPU framebuffer. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { profilesByType := make(map[string]profileMetadata) - creatableGPUs := make(map[string]map[string]struct{}) + creatableVFs := make(map[string]int) for _, vf := range vfs { creatable, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { @@ -96,14 +96,7 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro for _, profile := range creatable { profilesByType[profile.TypeName] = profile if !vf.Allocated { - gpu := vf.ParentGPU - if gpu == "" { - gpu = vf.PCIAddress - } - if creatableGPUs[profile.TypeName] == nil { - creatableGPUs[profile.TypeName] = make(map[string]struct{}) - } - creatableGPUs[profile.TypeName][gpu] = struct{}{} + creatableVFs[profile.TypeName]++ } } } @@ -119,7 +112,7 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro profiles = append(profiles, GPUProfile{ Name: profile.Name, FramebufferMB: profile.FramebufferMB, - Available: len(creatableGPUs[profile.TypeName]), + Available: creatableVFs[profile.TypeName], }) } return profiles, nil diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 51668ed11..4555f5c2a 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -184,7 +184,7 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T } } -func TestVendorVFIOListProfilesCountsPerGPUNotPerVF(t *testing.T) { +func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { t.Parallel() sysfs := newTestVendorVFIOSysfs(t) @@ -196,8 +196,8 @@ func TestVendorVFIOListProfilesCountsPerGPUNotPerVF(t *testing.T) { require.NoError(t, err) profiles, err := sysfs.listProfiles(vfs) require.NoError(t, err) - assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"), - "free VFs share their parent GPU's capacity, so availability is per GPU") + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-48Q"), + "each free VF advertising the type counts as one creatable instance") } func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { From 38d8e9b75b2307336a9c3917ee5bde498a56d4e3 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 07:18:33 +0000 Subject: [PATCH 13/19] Clarify vGPU availability semantics --- lib/oapi/oapi.go | 477 ++++++++++++++++++------------------ lib/resources/monitoring.go | 2 +- openapi.yaml | 2 +- 3 files changed, 241 insertions(+), 240 deletions(-) diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index f2c406528..8f02bcf86 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -869,7 +869,7 @@ type GPUConfig struct { // GPUProfile Available vGPU profile type GPUProfile struct { - // Available Number of instances that can be created with this profile + // Available Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer. Available int `json:"available"` // FramebufferMb Frame buffer size in MB @@ -18597,243 +18597,244 @@ var swaggerSpec = []string{ "oz9c6UhpQXdGru6D6JBI1tNs25MMJ3dD8WWRtJmvIX8JTmFBQ9JHsLsg+M2B31R22rniSULCzP/THzKb", "x5L9JM1Fmv7Q0EHNCBWICzql5Y7LftZ7jGW9CWc65ro1dxY/bIpNk9k1oE8GGDyFiQkNNTBfxXRuuyad", "buc8g4a0gqlMmrMMXrNGkTzivDbEl6dvbxqRmgg+oT58YoiQsU+toeZiNV/tDM57m/9josk1+4HGRpmJ", - "qol5WEFytO+3O4henr49bRpThq2IiqOrzSmLg1oGhu0oYq8a7V21NWjcbtDnTNZJroo/86m2E4FjMk4n", - "EyJGscfX9rN+jswLWZL7yfOyeqvV6LZG9GlpccCKnuCAsul6a+p7/HOVaXQL1HzvX64zi9XXBHOglyrD", - "8zMIB330W4ZmiV6evpUoj13zOO4qSAJNaUOns4WkAY5MiwaKirKivw2Ys7XCfJp/aD2THrXZD4LqNgJa", - "m0+TFLbh+Vnv+PW7jTgk825pTBBvNuMR0eNeL0iLuctdz3OcSkJi3uT4MIwh226gAq2yHdyaSIX96qGO", - "4gpHIxlxXwjPG/0QwUO09u5nk3yqR9BFSWkp9e8FKpT4e8+7YwA2pqHbc+iw6kEtbXCvKVmunmG8LYXp", - "lTr1bRWT8VNXeeoIxPyyvND8cjXqrWmkud9Dl5RU8XFbvRKZ3CUEuUsuKgGZT40T23pKJEmwwIpEC6No", - "ZEdfGcWE1C+YyTUJbpAV9UK//skgV6WCjNRMEDnjUTksYbtbRz+XEBo7Jxbw0cyp4IdXHMVYXMLB6PRq", - "lDJDgXJk9fYqNJGZUskNJvXLmzenxthWRMxxVI3Nl7UL9yMS4QUaE3VFCHNTwRJh9JJnyJrVFEHZALon", - "1CghgvIyDTvbnn7PTbgumgocEGS+crUT7JJIODXbktL24kG0DgIiZcP6bi5bX/vpJI3arbFvWJsrS40E", - "N1ngN4enDjwuA+d3ZN6qU/mUiJ7Zcg6lf/nSbsnlMIauK8YZqXcm+JgArqFNiCjmK7pwI8Bt1J+XcgQL", - "wkEWs9psP7ALDKm6Zp+/b6XsVbe77149xiz0FTkwcfAmEXuaxnpB9JBFCldJNDSxSCY33KjlxXB+QXBI", - "GZGyksUapCLqdDu9iZ3V/sZGxAMczbhU+zvbm083lkd1Lg3ntdFLo5AuM/dcjJOJgnHJnqbYAEy6zBIb", - "OElaOMQMHVecDyCe6uGDUNhAn20FDc/Fkw8GtYTcaxwoB8UKHrLSDSgubttEs2RpPtBgVgxo99mz4v4c", - "eK+k81pcjv03arwPYSoQYqh5xHgfKnQ0TP7Bq1Fx4cPb5ELZxLQxcTGI2XnoIvzsHUups6eDp8VZtip5", - "BMKmss3tvvNM1bxdTr0lBXLb/WsbgGipss7htvRyyHBNlxU8pSVijaWWE5QnhN2Inrs721s3o2fbiRy7", - "KK2KXPIBgByeHBmdKOBMYcqIQDFR2NZ3KwgZcC1pKQOAbpjEkL0y+XG5aGkIZygietwWjfJLXYY3ANOe", - "majeEMWY0YkWyPbNYs9yhrd29/YNfHZIJju7e/1+/6Y4By9yYINWS7FhYuYKkAd9Ofu8dbgDOIM2c/nY", - "OT1484sWZKkU5tDakGPK9gv/zv6ZP4A/zD/HlHlhEFohrtNJDWm9HB6mDX7z+36hwJjTe1pV5fH7hiFY", - "GLBXvKBipaDJu0MPy3jcDOA2KaxLUjq1uvIaakE20ObW4Ol5WRFVAE0vJn+1AFCnH5Zfdzt3F7xj+4RC", - "YTm2fP2i+1bVAeRSAOUavmZCWAaZHEXmr4Czud6uPvzk0hHpnrWA34RjxOJsZl0Wf8x6L/x4WBxI4XcH", - "31n4yQIpv79hhMpShfTvdT10tRRy6uiKzez3PWanQlvQeos65g3NfuCz8DZRXOXeX0//+8//K0+f/LH5", - "56t37/53/vK/j36j//suOn39WVAcywHnHhQ17osBxZlCLEW0uLasdIJV4PHRafOvgcL2ifE4qGAGMNto", - "TPaHrIdeUUUEjvbRsFPJRRx20BoBSwm+0uqubsqmVK/rj0/NBaP++KNTgz9V2wht7rSwC5JBYsh0HPIY", - "U7Y+ZENm20JuIhLsAv1XiAKcKHDnUIa0/btAYwGVZe2NT955F33ESfJpfchsaRwl9AwSLFQGgOp6AKaw", - "ozJRqfZ1kqEjmEuXIctO6wxxzVz79TPlH6I/qqldfqIst9+s5fR04MOmg/QSvZARwKqj7AKNSmD0LO8F", - "PR2s1+25FTZGxkNL2M/ei5v8w4PU5y5uylk8IiENQK64tL2ZTezMMiYNp1knXiL49QLW5ozYUpY4VTMt", - "i0zEUxcFnF9S0oUl7aKJwDEEYsCX5np9xpPeeNGb8STDAsDCBJ9gc0FdNrL/b89OtPeOCDqxPXmLYGkW", - "8SidsGXszEwGYOZdqE3sjV44SZhWfebEvk4lSixwfmgjq1UqmIk507x3cHpsQCcy7iM+nfxHW0ZYuuLB", - "M4BXU7oZH0JaZ5BlsOBxEJJJ9d/lCIOt3T2wYN2/t7daJ5Aa0i3jsjTy2LSxE30tJLYRkzAAox6MnCN8", - "RUyQPgGNJEbgp1Ac/nuOXEP5jssEibmdMilt0sQ6qEgWktnWvVlHdhtYZIsRtrupzXlU24WlJKYWLZhg", - "AfgsaoGp8cIkT755dY4UEbFLZ18L9OrALrFVW6mUqZafFKODw5MX6/0Whd5hqZYmOZUH7YE+sMEDTTER", - "uY8Gx6SLjo8gedUeK7ktBtkGP3OBInMq5ofRPnprod1LTSET7GwYI1rkEQhGbRl21l2LSfV420dnmQmI", - "s6GUisuXgxrywwSatfFoJhWi1nq3VrZZOPPPnseQ+IBVJou1mth8frX3OTqEJH1QVTxkNz6QikEhjf6v", - "wtp/aRDdL6+jb99MR3elOJMZlj7unhVvNeGlJeW6aeWqRjSH8JT6XVmCunAMKaLPudLnvuLBu73NzTeb", - "Ozd3390U/7SM2FRAc8sgUNtjl94FBmjdlXVN1agx1hvpxzay2/lF3p2gGZbsBwUPK96Rze0nrUoW6l7b", - "RkkX46P5xAwpk1IO/imL7jVAWJc0iowCI+mU4Qg9Q2vnxy9/PX71ah310OvXJ9WlWPaFb31aQKE6UfHy", - "9C3cjGM5coGGzbl1OM9PJddUKllHx2oVr/s50Kvm03alZ9wkTRt5BZrl+K2/lABGvXBn63cAvOqCrWvk", - "fABM0odMavv68FCXIph+LgypNcTvCIW0Ueb7EDwrGfy7TeL/9niidzKcEgCKT2wVVQCXcXxrAM9uh3qy", - "LQ+klswkRMeneeGb3Fvvmq/M6dlWf3PvaX9zMOhvDtpVOguW9H1ycNi+88GW0Tj28Xg/CPfBor3tpY5l", - "bKOj4+gKLyQaOitq2DFWYMH8K2xba2m1CkCp46TeDha1quc0AJ+Cvucie6Qtmdyk87RIp6uCfMVpZDK2", - "i2WWy0rkDEskE4NoaUoQZCrukMEAuxZZC04OGhOEg0CkuW/O1bszCnGaWL4fMkFkwplWkrWxg34lC4li", - "CreEWfcQGyhRlrURDtmacBk+WSpPglNJQv0DxMt3XVy2HhpVUGpBfzBkcpYqLcTW++iQM5nGRFi3JRpT", - "uFJZRzI1Nh+MF6ix0AJT0pCIIdOveTAzP2b6+/7eYDAYwJ2EMXW29b8HPm6609vBvsWENTmqgE7HLDos", - "wN6JlKGUhUSgMy6JUhgRww7VGLIb3ix+Jhis+7ydemU/z/Uq/8ZchR3bDhT2c5E4l9WNPi9XjG5tve7+", - "47OKS5O2+qvNjrFfjW5yxU9QwNMo1CbRWJ92xmNFQuunk0TlxbjhgHzLLpneo6Wp29hcxdGfKREL9O7k", - "pBQXIMjE1hpuMXGQEg3rwJMbLcPWCifCytHcEp/1PjBZq5pKQUP84gisxSs4l7RrOLTFVVxuSHozKygz", - "S6P5ZMmcKu7tkMxHaeozRPQjB9Xx9u3xUYk5MN7bfDp4+qz3dLy519sJB5s9vLm919vaxYPJdvBku6Ha", - "f/vMqtsnS3mtUV/5dhcyPXKh276I2qbA+cq5b4OBrygL+VXpaPFGZxZ7t5Gfq7qvx3W3HoI3GwQqCUNL", - "DVLiBA5QEui2TTR0pbh3g5dt781gc4WXbaW8gME1yN83ImXmTs/km2f+67gw4OJilcd5O3EKA3JZF6uo", - "Vey8PdEG+7vP9nc/l2guc2DVGKvsdI+L2xSO5PB4K6kJLj2u4KNx+JUdq28Yl7LNZOh0O1myBfwNB20l", - "kDd73CqDqGnDdv1iZJn8bkikPS7ZAhAmYfDYwn2tBWQ5mONUoSxdW6sXhxFPQ1Twcxl4KrgEOi7YBboZ", - "uJOx7i8DN2kyAbT9APjGgK5PmRbEcPmlG7FZt/voJbwLj3BsTCY7CFPjoXjvg8OFCdbQ+8t1bQyY5UM+", - "t7YLfKMNGaT/BdPWZLDu0OVNGM1nH/3G4ZvMkmK86lc1r4MJU3+96oNds+mwDkUBOrNq3D76OVPdMuXP", - "Kntrktg/R1Zg5eAl66UUcrviHc0t+coV0qG7HUPRTrfjCAVp0/UE6rc519f2X5EVfVFMBEemwHmWoJoq", - "GlnIZpgJlYoG0mYu6MVt0i9s+RkSjoxh0hQQabIerfGSfeTUl3cnaA1Q+f6GrNNY/2s9C54snXVbz3ae", - "7T3ZerbXCnsnH+BqtfMQcnLrg1upgwZJOrJ+h6apH56+NX6FwFjsWeDFu5Mi1EEiuBY9euauwWLnz/rP", - "ipBDIU/HUeFWy+KTGYRTWDAvqlYmixqC8P6k0ZxOJuzPD8Hl1h+CxpvXe3JrvNkApWo68ru0jos32zX/", - "Lxn3TCkZPyoMMJSQjcBJZ0TCDNA5UQj4p4dwAKZDlkprWc7BK1mKexlrZ3t7++mT3a1WfGVHV9g4I3Bw", - "eQ5lO4LCFoM30drZ+TnaKDCcadPhCwDaNbNmpX+fIVtOdVBWSPubg20flzQc3DnX2LbncSPJ31nTzE7K", - "Eh0ygjOzrbbLvdTe3h482dl9uttuG1vX60hcL5cwLl/GkMeCshdXfg20yTcHpwiyUSc4KPtNXHjSjUal", - "bjQqKChggMBvMLCnT/Z2d7a3NtshgPkiHiy2XWnDlmWXZ9N5mMKzGh5S1EVvt+m08KlThsHOSBBhGh8E", - "Lr6/cvoYwO+RMK/li9DmYLDe/trB1eLbVo6jzB1kskOMasAFSllWZqK/+rrzi9xaNkttczysluq+nBCm", - "qWehakyZqVuQMhFkTnkqv0BDXJmEzUnEubjRt00GyxmRaaTM1SKV6N3JDyBTNK8hqUhStqEsNy4B9Lnl", - "5G60n0ss4mfyJmK1Wo02S79swt2GXdtdhuZQkgaNMFqhllwpWx15eIijIIXCKjhbTz0rQMCBfPQkiRYm", - "sDyKOGcomGEGFxDClnFiU4TRjEdh3xuGqZ+MJt6IBX6FIm4AgC8JSWzNETMI/ZlWYeicoLVCPjwyrFSp", - "DbkbGyFjq0qUuXE39he5w9KXKZXlYWt6YsUL2LTmk5LLMeJTCUahgpD5fhUSPcHCRMJjZmrozGNjS3qi", - "fT1DrAhz34lqTlI+sQauVTkgy9lQEgeCS4lIRKdQr+XdSSV5dknCVZZCuzqasDzYFqxrLgs9RxkccbJ1", - "qS3f+ehJJvmcExJ4GBLWlsTpOedkjFkKVUgKjEyuEyoMe7SLxZtxqUYZKNINByvVCEoLpILkyGlZynfm", - "D3LveM9FJ9puQy4b9Hqrr2tc5W+qaYDNMtVLUT+1uhkP+ti4Dgu1FIkqh7aq4hjdBLgsx6KnElqlBcws", - "tAYJFwWxVMBTX28TgOI3WXU/NWvVVlB8tTM4b4spthxC7BSr2TGbcA/wxA1uJZ0n2oZKJkTEFIproJAw", - "SkJnS2bXk9bVBdnKkSQoTImlnNFPBbYEx2Z7A3gEcz4yyqYVWV/tsI172IxheeUB6Ne+2CaUSPqzOd+I", - "FGhlYgElwnleZ6sASypH/uusesOCTNMIC1TFzVsyZLmII8ou27QuF/GYRzRA+oPqnfOERxG/GulH8ieY", - "y3qr2ekPRk1las7N4GxSmlmQSr/5FH7Ss1yvpMSCJ2bDfL8BcCVtIrO8Yco/04hYaLm3jF4XGL0Mzb2z", - "NWhK1W5otJSkXYclvKnktizr3fGp9CS2LdVyXIEdElq8dKP2JKk0tT5anEoOkNNdAd7uRqectfB5uBiH", - "Rl5XUDHQmEDSiZtaXWq0EIttpuItM5DKGfqDj8sO0bYhtZ7iVWssx2cQZOINLocVXeqQNm/UaFJY3ZsA", - "IIBY1ROFj26IK7CqzFceu9QkT85qFa9mxJKMujma6lctqjm42Iosd9722j6JvlqbzBMLDBApUi2gyieU", - "VlkUCu5JNOZCABqx1nA4c7MBzA+t82haO9Al9GZGFkiQGFM2ZJRlTlJA8iKIkTkRhRRNLrSRNSVhH/29", - "YOIBfnOcqIUFBgfn+Q8S8SuWjXHIioPUjadSt3PAjGdRpFBQPX9JNwtWn2YUSJmFSzAlqMOEnQgiZ8W5", - "++onah3viouwsTDNArlXoN4J3LEixS8JK4qyrBmvaWgaGpmv6hFypvgpPLX2JyrVI0XVeqPL+0slEX4l", - "MZtS9kqr0JXCVikYJwaBBOA4oM6c/cuI+AyCowXgRt78/7gm859Os8bLv1VeK4BqOIzbA+O29bpgA5Mi", - "Uwn2Kd+krQxVgRysZVAq9bsEtObCk12xjrImUCia0eqcbJcGVg3Ed6PZkCQo977zdPfJXsuqJZ91WWeg", - "o7701dw8XnIl17BSJ23ufZ7uPn32bHtn99nWjW5YXM5Gw/o05W0U1wetkWulN2v073/+691J5dZnF+Kb", - "BzcalMna8A+pIXOjPKB3J//+57/cqG49IJ+gqcNTN9zbN0bpRMWVdIEC5Su8dpdkS+z7g5KTAGdiBq2R", - "yYSAG3Rk6NbLB1PBoGinBeMEB1QtPIIcX5lI8uyVCsxym+ug8mB9Kq9p20Jyaskl03Ge8bjmOkd/NXfD", - "FV542rr4kUzHTffQr6u9mlvo/NaiGOPQIsQgL9Nfd3Bn87nCshQsrf8OIKfBZW/VM1nMG8shX6tpBhDF", - "Ymt8FUIBfVDhFX3SflRc/spyFu4tS26dKsXfL9mHzVvwRl5fz4nscfoGq9NWK/LBHoC3+2o0LpYlW1r3", - "rVTDLD91b95vi8zcOkh/doLdvL9CMuJNPqwC0gI/2jFYkudtd0ss0cBNhTwTjwONR6SXBerZJBQkU3Mj", - "qPe8xTj3ZEcGl3wyKQOt7jYDcwPmDCRSuV6wUtoy6SJy7XwWVVRnAzAz7OzKYUebAMPOZjzsVK6tvKmJ", - "Mb4e2Q7KwCKDZUjZWUHO6iClm8E44sGlqbgFhZz7aIBigplEKYPNX7lV2xwsvx3qdpLC2mS41MSEONXE", - "FoxpTGZ4TqEcgr1TmZYCMck1VRICRqGdfRRyAzVUKjdqZ6hfM4mD+/mk4dDBbGEb1g3q9zhzEa35u+Dg", - "m0CRU/aBCN61mfJaYr9+fdI1AQwQemgGVopvdBM1I9ACMuuigu2f/+6PHx5HZATjrmLFx3U6FvO94WZV", - "EEmUtODROTtUmAAFPGWqCiIftzPkyilb9SMpZRDsZ8M/ABTM9m5r1YckgB0p63uxzOi3YO5K3oCltC9x", - "YNvHwrAp4C7Jf1d8Zi+EqwMwzoZCpWDTTjGu29wSjqTitrRUtqtH5DogJKyiTfpfaRsrb7/0xsq/whag", - "Jivia9+GeOf67Pp3lzwFY22idjGmn3HWA2gMt6QWxsLg0lmglDKjlXCvC3ARIx+2p++FNtnM5Ho5rX8j", - "1wrAucM0Mohrfta1osoeRqsofuuswaYNzcXqmvd3UMLMxJvfqoiZDVV/iDpm9q07qV1WW51zoty755aN", - "mivYl6qMlK60XMC/e6UcY2NYqYvsAY824/UKC+7M/F4RiwjbMv+R4ZiMEkEm9HoJ85gXjGFchgzJN1KW", - "wWDALddifI12nqBghoWsjJ3R6UxFi3IAzo4HyOezCvwJoghzjsI2K5+vpvuwHu1ml7PYuk85Pi/A7tTq", - "aViVdLQMtPkwv22z3vkEL8CL03hJ+GR7ZzDY3hrcCrXZDesG5DrMP7Hl8MrtNKXUFb6zF/2lKNViC1kC", - "c73G6pWgkAedkUkqQXC8D4k3CQ4IisgEENqy2tarbxarXS8fvFWoLNhLxv9uoey6uTv4cr2WrCsLeO2m", - "0XGXi2V8n+LzFReiDWImqOG5eXLutnuDvTeb2/u7e/ubm3eBtJwRqSnb48mHzasn0Rae7ERPF0/+3Jw9", - "mW7F21477JKasjRtePVX/W5jlE1+SJZxgkoiDa3ZOSREVIvnVotOSxJRRnoyy5Banaa4RBaY+/eV+/9m", - "fn4zg6W6w3l5kkUVAqucOCXOehhsKzuZpXcX1dkcHy2fxa0ykKoD8fNbdSjAXu0GA+URNjufiXqQspbH", - "0NvCi60PoqVZcauOIt8NO+x07yo3UNzH3iU5Wdpwyw7w+iHn8Z1OuaBqFi8/LbLXMgxriJv+IFVYxlLq", - "o+Mpg8rZxZ+zMLmiEaU/7nQ70Yed8p6xv7dH1bLwtxkD2qUuagUtwsigMPtyKsArueEhTCS7ttX1mH/a", - "7G0+gziE6MPOT4Pes3LEQddQq0i+Tfd26ddBGxoW68+5ukWbz24Uce3ouYyDfqW+6mn5uWyBcS2P53WK", - "3dHhEm5LC5w/rq1xBSWnUQH9XE3PHm6jotIUkggvfMDoBUetrFiPRSZDYzKlTLbx224PMsftbjzs9NGB", - "RacGWzavSl9qHuqRF/iExjEJqdYxjenfnMGw1dIXV7UlblYYw33l0db6fnXt2WqIhFUJV6uOyf5n5ON+", - "lvXbzuJdht4BfjVnogI+F7zYRXSCMKtUx6RsjiMa2kR6SIyEeLV9B4KWs6yVATLXA52fpIumXKE8hb6l", - "vy1lzX7BbPzkGvytSzAzDENsfRFAlAyciy4TX8dHKBE8TIM8fzSCQeeIHyKtwJ8tUfJXh+TepX8DErMn", - "XKDV/o0mh0Y7/2TTeld8k5phm5d6c7B6qe/EKdLtpEm4WoaZl9pJsBvBhq9IQfS4aMpkr2iChcm8byHR", - "z4oUrNu8xpccaJUoTdwFi+apOid5rlvgisEX13tEIqKPqXojiEdhniVBZS5FV4vUzb2ns6YrTriRqg/k", - "V0ISbasA/hH0F2O28A7M1bzMzpK1gatnK82FV8/UyrHUKg/uyUpNrHGpii7cJvx+I+UrPm+Dl3JZcH9n", - "+NlF1ayOgOIEfklJO2vGn7dfurC3Rv/xXbjlHlJJe22vHiqQqA6wN0Med/3nscBarSsz747vet7HFm+s", - "ZdwE81rNAi16nQ96/zBeZjTq72/89Lf/t/f+r15vc8VulkT0QjKBQKNLsuiZyjfaRu+XQU4Bdl8r01PL", - "KgTH4EMCAHG7GYvj3R1kQmPxG45rU4AIrULZms2VE/rbX5rjmwpkfAtyciXLfnZViruo3qm4O47WYiKm", - "LpbcJZKt94cMctMuyUKiQjEsq9I4Rv1BZp8UItDRhVED+4TNL9CYQnVBOWTaqsVBQBJtTdj6KtSUyOYg", - "fQTBUbEdW5TLJX7bC0kTT0DQu5MaQu7rt2+ev37729Ho9emL3w6OR7+++F8I8bjqmR7Cnua9nd09Wxi7", - "SMlNzxJ/Bob/Z4HU+tjNQF16+AuSLKHauEdhphIgEly4QeFltAbJCq7SpsvVXL8Z9OZB1qA32O0LF1QZ", - "PPsSRQ/fLq1yOOdRT2vUDcD3XgemoYU3WBuaMkHwnSY/93TsURutN3FKp9jj2va5sL9IcUI3oJVJOrX1", - "b6zS5A+dP6pWBTDSwJCqgmJfsUul6jVH1sdakRrlFdjL8Rops+mStBDOVc6NjJnasEVEfRANIQds6mUJ", - "svkuc4B8Pfhodd7nUlW+MLPCSJrX5sRprBWdegmBTjVprmZEkMJCwAc52voNSWZTQVoAf5gibAkReZik", - "yyPRihDccEq0ljkbHAmyBNe6B3Y5mv4Jvs56AO89lrUrL5hHXu5m8+VzQD4/cyX+6MQ1AcOo2BN+nO8y", - "Fy2jieOq+mIUuao+b/O+d+NZWbVE+jXtrQpz5n2UWNPHj3/HVP3MBVggzTAbdw4XDtZNSATAjlXBwFsh", - "adOYhCOequX731YQtxgbWRnIvIyos7YwMHFQyiBtkgUOCCIfQ53SmhwkSAVVC22eW2V4DJl3rnYnEBI6", - "gp/zjqFe4qdP4KeceHIUXhJGBA2gGqXejzFmoKSjdyeFomSmPl0NIhTUy9eHx9bCdSizYLFQBaznQv0O", - "To873c6cCGPldQb97f4ANnNCGE5oZ7+z3d/sDzqgyM9gihtQQdym7NoU18xWOg6tJvTcvaS/FDgmCr74", - "3ZN8DqFu9nXQevG0YLckmApruCQRJMUbhqH6a0CPdwfqvjmVu4bsrd10kNkKCRckeW0X9z0olbB3YJpb", - "g4HF0lb2+IV0EROjvvGHDVjM+22l1VkSecDUa5aF0y0z0n/qdnYGmzca07KhwN71dfyWYZs3SsAg3L0h", - "IW7V6TEzmWA2r9dG4BR3HDBSca/9/l6vmUzjGIuFI1iRWgmXTYoxkQi7dw3Er5Io0KICSr700WtGbNl8", - "rBA2wbIiZVBK1n2oObS8C0zbbpEzXJznPFx8MRKW+nBm8aeyONPb5VONn78c72RsXF9I+8iBOhuuvQcG", - "eo6zOsgPtlN2Bs/uvtNDziYRDRTqZQxsQ2CphCiTCCCqHdwNF+jPlCuMsgjyR7Slrc46ztitmx9FGx9p", - "+Mls74j4PK+nRMSYmXh8886KTV/bzsYLnm/npaeaY3yoJgEnlcN9MQcVKHLlLVo8tqrKYP042vEk/ds+", - "zfTCB2T8nXvY4XayWSnMh9xyUF8RpZI8pu1kb3XGuRLi1eVeEvW18PzgPo8si1v/De6ix8LAL0mm4eWr", - "VTsUNhKRMmMAezXAszxHzn73Q1n5e5M/KQRmgCtdNw0VFJS5ysPhoo8cTY3RrxaA6iMIzDOsHyunenhf", - "yw7buo8dBjPOLie+H1Pfj6llu9xwi5sCbMzCLm/hg7iRB+Lb8z/c2Pvw3ffQ3vfQyvPAyJX1LvzBx31k", - "gyChDr2c8TQK0ZggA7Hjwh0UFv3pB4RFMKNzAjhqUBcsjRRNsIBghhiFWGFzbdvomFjqlsia29DN9Vzo", - "W07gKnSCJCOAfhs1QR7mQW+UMRIi/YlFi8sR7GrVoc3e9zrYswbzoxFdzbgkGYQcU4XTHDJqpbGOodn+", - "kL2x2KKagBC/62SNJBEgpC7x/3CG8JDZD350IsTFHkkc55ILC4CpowYM0SxLPZtKj3QkA+6Dd3lDGGaq", - "JxMS0AkN7LQuycKGEHobbFXqRw/YjfPdSZYjgLbW/RBhgAjox4M9yp4hy0nl+xsGcbdBlIb5JZdDrcFi", - "jKPIWwtiGvExjkaGPpfEcyf4Et6wRClWkXe3SYyHxFQETxZqxpn5Ox2nTKXm77HgV5KIYWe9P2QQ+29p", - "TcJuriCiK6gdFidc7zPBY9PnhhnixsdLsvjUH7KDMKbMcQR8giPJEbmG76CkEsA0GOnVwA9mN/nvwQ9T", - "qXhcBNt0fGeGyVOVpMomMUiiuj6gySFTHH10cIKfNj7mPX6Cy2KCQ80nhVfMlEC3bhq1HGE9+xG86rlu", - "J0CAYUcfpMOO/nsqMFMGKTLDQ0TT4pKuZYD8epOuVykcYIYSnphiBsBUM6xZrtQGwAPgKEIKtpL7Vivu", - "sJIN87Fob/G4EerNYHNVthFl6OR5YTMNdp7695MkgSC+iJL/Pn/9G4JTWa+BeS2PEDJZBEwrDChM4erU", - "ybQXOJghc1EF9euGHRoOO9l1brgOY02lTZjv9eBO8Sc9tJ9MN10a/tTv66bMdeU++v2jaWVf76UkNtCT", - "w86nLio8mFI1S8fZs/d+gjYhZp2XBAFaM8fcOkgSTAHcpHDimyMSsxBxewpEC4RRLoGKgStjyrBYLMtd", - "85DeUpBPTPBcgRgfhxAsN+zsD1243LDTHXYIm8NvNqZu2Pnkp4C9tWwulgbnWXa5mTHR3mCwvhp82dLX", - "c2fZ4mLgC9uAjVZRVulRr6BF/vy27gf+o+3P7OoHM915joZjDH/nfH+EFxAFjb1oiXquICpqN2YBiZza", - "vdrRc/+XB3qxAhJF982gD8We2fVYBg7/qNgRFivfRkvd9w/McYP7OlRKbvuH4d9H5z/3eM+t75zMXaiz", - "vzQGwJ5YUxqZlxGW6BzG1DvXxvcL+LVv/+tsP4Dxu4j49GLfmO4o4lMUUWZD0AuBylo9sLSEjwzySfad", - "BUJxdcnWjCbx73/+CwZF2fTf//yXhRP/9z//Bdt9wyB6QVnjixnBQo0JVhf76FdCkh6O6Jy4yUDhUTIn", - "YoG2B9bnD49Qobq61dLkkA3ZGVGpYIVQfVMiTNoG7VWBng9lKZEWOUa/SCe2fomJbfT4bdxeNqS81x3d", - "9SDwwQwKE9CnouMBgC+jpraztUQ7fpepmXPJaVoN06wF662WL4pcK8O9PTPAGwoYILFv38EDO2m0dn7+", - "Yr2PwNoyXAE1asB2yJuxZkT/u0xaLZOMRCkLFKCykU0GTmm50//IvtPO629b/Jbc/rZs2Q38/sb5A6CK", - "bgW+3wG0uAPw083dB/ic8kcOL+zuggVNFw8UK+h4r05z86RAsodwBqA1B8QADlUu0OnhMcJhKIiU6//Z", - "rgI9U8Ol+dGBOIOiAA9xa23HwoWFqLKmWplBHos4OLOjRtjNq1oNsni+bZRKRTSedFnViPzIu/vTo9Lp", - "TY6RvGJlzmvfT5KVcXpUBhyKYuXc0gtwAoR06ku2T4tctMohZSIAsyNnqbpkxfPxkduQ9+easl2nrHo2", - "3INQPKoIxAcUhOUszWKN18fEzW+zVXTYqEs8V18Xaw7uTwu6by+Wj80fkxsrrJBNS0GDJ9B4gL4kyqAI", - "dO5woW0PnomfE+F2tSvJDbPOpmU+RQYOASYEV/PLbd9j80o709e09y1ZvkCem2gsluTfVZQWxm5Oq2UG", - "7rEtWHp39i30cCPz9svdeFsG8xAZwm7GzmMtFAnRGpYLFqx/v/T+4hxtQqJyI1a4eZMQJRFWEB0JQCyZ", - "nXVfet0BK1bH1TqdtWspQ5OITmfK3H+EdAJhPqpYbRZGuXUPo8yqugqsiI1ueowpg6eayPYCaU6EQq8P", - "jw39i0fqxkeId1ttKjnhtfR0fXv2qkdYwCHAMQvO8+uk9skXNpgM/5fSAO9/1z3CVDjq1IMmhfEz1t/E", - "oSITOtun/L+2fo7oWGCx+K+tn3GUUEb+a/sgwopItX5nzDK4r5Puvg2YR8x82n6hZaKBaGJTADhcofBn", - "b7XU+d3735TabyZ9I8U/o+t33b+N7l8k11L13y7FnRoApo8HuuHKmM1HbXj0HQ3jHpymliMLaBilW6Qc", - "D2PGpYJHjy810saj0ozjisdGS+9/viGXHh+OdY+PukBIqH8K+Os28+ie7gLcOO5dubX93v9FwEE8ptOU", - "p7KY1BRjFcyItAl/ESkL4MemdufHc6Pi/RVz6eA+j45716u/8/0dafzVBTXC21zordL53VttdX77vtb5", - "DRiiTYq0IPFdV0BkvSFG08EhtmXjEmpkPXbUNy6fLYLeakMlNxcQWBD7Q/Z/tP3xuyI4fv+Ty75KB4Ot", - "PfidsPn7n1wCFjtxrEIYVDaHRNyD347gFnUKOJJQEirP9ayOw1SYBdZzINj/cQZSfpHc3kJyXPjdQmpl", - "IRXItdxCsmtxtyZSGUj/3m0kx28+gls44m/TSvrGr0dKFpxMJxMaUMKgHAHktMpaPKCx5L7fjNwyl5HZ", - "+8hCMFFJE2ltRmZSa4WGnldAvfdAsuO85Mt9W4+u2OrjzMngia1eaO21XFtoNti+Nn4Y3O/pdf+G2mNm", - "MWMR1UmXaKXbU1TElNOJUwVBsDkMEUQZI2HMmqzFPjrMss9lmiRcKGlK8oCFYIp2zrSF4CvfU67I4yvB", - "A2VnKJHdIYOirPqxQdHYuCQLU3CHcpbV1slmauvW+HL9ygWPHnQbfXkl1F/NqZUSes/b2Nbnezgl9MFE", - "x72oe8elsqdr2cYAi3tMsp3Ms2RS+oGy6fqjing2wiqbWwE0zaNqbeBUcVe8f2PGDX6SH0LuNMIBIMjp", - "1wy4kc1ONmhmxaYg5VjwKCLCgFYlqXLVvYYsGxxlherFtpTGhW5+lDJFo4uuiaYB5AGJMFtY5JYhK3WG", - "lSJxogWbxSKCEQqSmBFXyprpQVOeSniriyQvdYlwdIUXcsgEmUQksHODEpCCBAbfLYr66BcO6d4ITzFl", - "NgNZv2mKgv0gh+yChhEZ2WztC0QlkjMuFGEkRDGfE1nul2ARUSJgEodYU06iGC8ANskgyBn68IQYaKJS", - "TjjX/8YspFAyS/ecTXl/yDDaGgxQTDCTiELasMQTor+ybSAYRGlAPyKMdgbP7FeVdQNoT0f+Nb1fhCBz", - "HuBxtEBEczGciGodFjDbX1B9Ui/fhApp1itzL9raRKWFpdJV0Qy7KGUBFHlMhf4XFyhl9njVLQrIhId5", - "2ks4QkVWHM2m7Y9JgDU9GS/3A4BpPAhS4Tsc9VIXyvf9JyqZhemdA6l80knTAcGeCmHNGVcz2NMcttL6", - "jw1clTPVt3HIeDcJFwijAl/nDgWodM2maA0Axi7ywmDM1Za8WP/R7R29fa0gcNvfQHw9lvMJmIhPJqUN", - "uPpoMht4WXpFnYW/1X166CpCFkVcSPGUcalo4IRhtWbxd+OxtfG4nLJebp5wcVnUrcr8+zMXl22tr3NX", - "iP9RGWHFGX6F9wB6eAAR+/DXAeCMNoaKZpp7N9Cq/JXtUlC6qJIuzpijiLOp3kW5U/zevfYViy6IUlDL", - "nSnnnCDaCBnZH01RST0ZW7IPPPyBbfWhZZHu/R7ugn7jCtE4iUhMoOhkzzCbXuxMqzY1oalEs6za381k", - "pd5VxdRhYwtKc/3fdeoQ8JVbsDXQ3uvL5RWqEZ+uhgvLOnfYWB68sCEzNauJK3B9gTIZrBVaA86NrmY0", - "mAF2GNitun0DLYaT5CKDTV3fRy9hIxfRY6HzNQPJrXlN8ogYSLB5HF/s18sqvjs5gY8MbJgpoHixj1wp", - "xez8kPqtIhaYnkWEpUK/WYSztcwYhxW9UFjbm9n81i1KWA5rO2Q+xDBGrmyDdIIuCuBhFw3oYU7evuLT", - "B1PGus1g5GYuiiNrOgJvEhZ2mmIsaOTHDdscDHwYuS0xzMww7hjCrDaYV3yaAaGXWBknSVv2tcMELp7H", - "8RIeRmu5BEFShTxVf5MqJELAx5a7m5gbreHAFsHBl5pRmZFKbmOvA/t5I4kMMrGXVFqodrodwtK4s/+7", - "/dc8jjvdjh1PAdH4Bsr9Ciy4aoP1iBe9MgXAt+9q+U2g3MrCvoDlVjk5rDndrJGfmRe++ZtF57N7QDYE", - "/aDixP2aVNDCeMsOH8aRZDiRM64eF3qUdTVVtLZmV42bZU8PL0xdpY42IRzn9tNz9+VXYP2uiuxwY0Zu", - "uvce4lEfwWPOhJW12Uy4qEIOrYr9+OoZ6cstSW2qbTjkO2/e3M/XijG1pl4XEfaD0FRuwqniMVY0gKoh", - "wYxzWWD7MZnhOeX2qtTdWWWcCc4NY2faEPoLzaoX1hF8YRX5feu0Qrj4yPbRh89t4L3/C/co/+Lngl2e", - "SfyuU74BWRvKGgtKJijBqSRar0pjgoJFoKWiKRNDcDBDAU5UKghUwCIopozGaVxwNWjDScwxgFRcbMYX", - "XTROFYqwmIJdZB6acHpBAh7HhIUEPGRDNiN4TrVRJ1CEFWHBoicJVM6cE3TFxWXEcQhGvo3CMZW3BNEc", - "SDnropgoHGKFQdW40Dt+ZLJ4LrJimsawZuQ654ZwyETKfjRo4LrZCzfQC0SkwuOIyllWdC3AIWGBF2r7", - "/OsWY1/eG3xOVHWiDxSXcytZ+pCBOkWvpxvO1xHD88iCkbmwy9hGzC9RemWzEVlOf3Bs9J+5pc1c3Rwf", - "6IonI/GyXfx13O1kTPfV3O88/AUOFyhMTXeFXQls/q3eymQCpRjuBKmVZhlvezWTVZfKyHwjmbfx0f15", - "fAtv2lciCbuNhn1THZN80l+DyLVUvZXMfSA3ovUlFbxiDyiCXUzVg6lPXBSk3GNxd1qBbbZmJreL0kkJ", - "DNYXZ9/FdlVs25CD24pt55utXaoXBDllPYjS9Etw68ZtFNXWdfAfmgtSmV1BZD64iMzvDu5NLB5ngtCI", - "xgQvIo7DbyFMd8kNTsCFMPgPgCjxmPBHC17DYoA++Oa6mYToutzKdycn601SQqilMkKoRywhCkkx+rPY", - "V+p/ToSgoSthfnhyZANmqUQiZX30OqZQV/ySkCTPKQEgj76en0PCqBdjLkFedDuEKbFIOGVq5SjyV+9m", - "MJ9uVcL5nuWkBbT+fiHd+kIaPPuPT5yBlIGsCTOB5ZapwqoxFNCFxlFmKrRrvQyPeapb1zJIk0mv5xRO", - "wQmNiFxIRWITFzhJI9huUBzB1s6035lV7kJUrN45JmEtISKmUlLO5JDZbI2ECN23/ly3Xwhx8l4IKJzJ", - "11MjJL+O8Dk9GBMxhlUT1QCzCCrXd/Y7GzhJNkKscEOIlh3eZwzpZ4iHQ3IRj3lEAxRRdinRWkQvjXmC", - "5hJF+o/1pQF1I/juS1cGvf3O0pQ+ZhPuLZ5meDZj5m8qr8qKNXcx+ejE2ktS3CxO/sBC+8WaXCnXBMFR", - "T9GYZMg1KFU0oh+MqNONUKloYJJ+csiCdyc5asGQnRAl9DsYksuiiATKOWw2EsGDjWE6GGwHCQX4s20C", - "gwOB1/w4hh4PT9+aRFASc7HoDpn+BzT85uDU3O5OsPUmFAbKiLri4hIdb7xeEWJ8DmT6D47RMxNcih3g", - "XfDvV4I3RwRp3EOyYYvyZJmpxJNvPojUanDf/QqP068AkEzZbNamAgegFMtZqkJ+xfw+hDmP0lj/w/xx", - "vArYS+Fg9g5e/Wq0XTOcld24CT6KTWnnFBJT3PFBLj0MwR5rzKomnJsCKDGlaEDvKXCgvkXu/vLu+yId", - "v8LrTktRVzj1q9lb933y2TE4jIsiPR7LNjec5mai+HLv0xWmzd6n5xEPLqUFQym6DbXdBgDj+sccENpe", - "EYKaALmZyIIIIXKdUAHIbxUHpMHckQgjRURMGY42YM6mEYC2dl4sPOcUUqSDiEKSGg0BtSgCdLqrGWFI", - "zwYcVa6Bwo2utKWliu8ULyMVR2MS8Jg4uO91n+n2d0zVz1yUsbu/Frn4pkB/PR89VT3PFXDlzT1+Fnz5", - "Cb6GUOkwtRfKbkRrL3n+o3EFdRGszbCzPZDDThcNO1vxsKNX4BCDCxUrtItiylJFZB8dGf8WJMHuDZAk", - "AWehdKjjzoO3PZBNKbGGLRvyK/fgu/tUeyxXASnPbCc+8aDfQ/p7SNpBa8UNZ/dk2IVNFyKeKuPut/vK", - "vhUSBe6R9Xu/qy3ske+2fRtJ/ne7fUsyClZZi8vC0hvJnqRyRppdbq9MJZ9UjQHN2lXXlDP0Bx/LLmLk", - "ynjDhVT9mtzTX5+aDu4DaV93dROUfTv37xD7LSD2c1r54RJNgKU+kh13GMxEcm0QYbFLe7c8BJYEYDfw", - "AEfo9eHxkAVaFBlwP0FiDtLJAoKbU/jg7+foxeFZFx1BpUf0Szpe76PXLFq4etvmjmbIjCZmhFeAGRob", - "riWh73g2Ywfuuctgcd3BA5VONjvDc7Pi1soFiXc7M4JD0Eg+dl5x05kHI/jsld5Aev3sl9myd5YqH50z", - "osSidzBRRNSbPbF5UixDrbCHtAOBs4qbgZ7UHUqHfZb3aXQDA06xvdXxYFV8+l714O4rhN7PLZmJEzH1", - "5sYpYH0ySDLA4eJxxTLJGcqEo08EFo/rrG5AU5awlWVLDQzosiny+ytyuS+VXSUk+P/U3QUzfbQXTUlp", - "nTQTZ/VGVt70uuTgmQEkthdVAU5wQNWii3AU2TPKngRZREovU3/HguDLkF+x/pCdZZVObEIvOjx923UX", - "tSik8tK0YO9i++j1nAiZjrPBIdho5tYYaE7CIVMcBTgK0kirG2QyIQHk4kIBE9lwl5sNpXOHeyfvxFtt", - "pRDVnj66Im9+noDVy9miynEbZqk3BAkiTONm+G+rqEHAIYQajHWjnCHKJpENqQoElxLZpnokolM6jmyA", - "kOyjNzOCJI7JkCURZowIlEoTFa+H3ksEkTI1Cd66AYDJNRzVRTm0XyK4sqEJEedCmmgCzeHvTpBUJFnC", - "Zmem5ROY8x3ptqZx29MDOakrY2h2hdhXkF4QwymG4JqP0sgFMN5rKLoZ0ENriY9l478RdDolQu8KbISs", - "Cccz29qR02z6UsZyY8HH8+ytdgUfs1YLWYmFjL2l0GyjHO067Nws6s/T+SVtRO+zj26WRfyr/qhl3+Vs", - "Vf8g7KPPnOW3Ukf/vJAk2NaBlXP4Y3MnFUZe2qqlRNvVsFqtM2vvMtO1NX7Wg8FmPWa0LFxKn20yeL8+", - "RhjcL8rDfZdEe9y8VUK7KtmmDSn/q/HsvwoOvBsg+wdGObkFkP1XlXcPSOMPh3/i3agPlUdfunt21Wa/", - "eSz6u0qfN4D0AMfWlD5vpJ4NXl1qKL2z77Qzk2yL35IGb+Mdb6C/O7J/t/pbmAwFYq26gtYMT+JELVxA", - "m72rzIPOJP1A+g0XwVnc6t1dBd8ipPPLsYfj08aAzm+zOPyDxIza4n1UouMjT9X1R4YxWNxzpYNlQ586", - "PSyCGZ2TZqd7eQdbEiWC9BKewOVKaAhm6eHOMoVFf/oB2eYt5qr9F1R/BLB8EqKQChKoaGEqcWqJYPr4", - "QSLBtSUAz7lYNEeJmC3ys+DxgZ3NivPQ7inrDMvjDONFL8QK9+ZO2ixxoX1GdKeLp9QCD1GGXj5Ha+Ra", - "CVNjAk205YPoJCOpKbcvgSfXiwPeHDR4NukHMpqO24xySbWQ17YaCwpSqXjs1v74CK1B9bEpYXottKo/", - "AU02EXxOQxKWxtiZ88hQdbOBoDf1u2qlIisd54wLM7gH0WHaHEjTDzQpi4UsJGZMGYbBrazLUd5TJolf", - "94cpcwE4do3cKL4fYdbyW3PGjuZEqIRpiag4NxDP69+Pucd8zBWTodyZVjrtXHjOcud1u/yolmlLd1H4", - "Icudu1+39buvJ6WHykeZzWNd5/PMIG1ym39dLDi4v/Phvt3l7x5xCuhL4ozvgqscGtAt+hjmFcR0h2RO", - "Ip7EUJEc3u10O6mIOvudmVLJ/sYGxH7PuFT7O8+ebHc+vf/0/wcAAP//eC/k9kLQAQA=", + "qol5WEFytO+3O4henr49bRpThq2IiqOrzSmLg1oGhu2iXSYpM+iOBZ8KtqjLxrwxt3K2sz56TqTqkcmE", + "C7VvA1ngQqqA7I4XSJAw1fxghkIjquAOQ9JxBHs061Ur/PoXPaGJwDEZp5MJEeWk9mc+Xbrw9ij2OPd+", + "1s+ReSHLqj95Xtantd7e1mo/LXEDmO0THFA2XW+93B6HYGUa3cLyvffzx5kFB2zCVdCkzAAEDaRCH/2W", + "wWdqUkuUB8t5PIUV6IKmPKXT2ULSAEemRYN9RVnRwQe7obWGfpp/aF2hHj3dj7rqdh5am0+TFPb9+Vnv", + "+PW7jTgk825pTBDgNuMR0eNeL4inuUuWz5OqSlJp3uRpMYwh2+7YAq0ykdGaSAUB4aGO4gpHIxlxX8zQ", + "G/0QwUO09u5nk+2qR9BFSWkp9e8FKpT4e8+7YwCnpqHbc+iw6rItbXCv7Vou12HcO4XplTr1bRWTYlTX", + "seqQx/yyvND8cjXMrmmkud9DlwVVcapbRRaZZCkEyVIuDAKZT43X3LpmJEmwwIpEC6PZZGK2DJtC6jfa", + "5JoEN0jDeqFf/2SgslJBRmomiJzxqBwHsd2tw61LiMWdE4swaeZUcPwrjmIsLuEkdoo8SpmhQDmUe3sV", + "fMlMqeQGk/rlzZtTY90rIuY4qiYDyNoN/xGJ8AKNiboihLmpYIkweskzKM9qTqJsQPkTapQQQXmZhp1t", + "T7/nJj4YTQUOCDJfuWINdkkkxBe1JaXtxQOhHQREyob13Vy2vvbTSRq1W2PfsDZX1jYJbrLAbw5PHVpd", + "Vg3AkXmrTuVTInpmy7myAMuXdksux010XTHOSL0zwccEgBRtBkYxQdLFNwFQpP68lJRYEA6ymEZn+4Fd", + "YEjVNfv8fSvtsrrdfRf5MWahr6qCCbw3md/TNNYLoocsUlD9aGiCn0wyurEDivkDguCQMiJlJW02SEXU", + "6XZ6Ezur/Y2NiAc4mnGp9ne2N59uLA8jXRo/bMOlRiFdZl+6oCoTduOyS011A5h0mSU2cJK08MAZOq44", + "H0A81eMVoZKCPtsKGp4LYB8MahnA1zhQDvsVXHKlK1dc3LaJZsnSfKDBrPrQ7rNnxf058N6B58W/HPtv", + "1Hgf4mIgplHziHF3VOhomPyDV6PiwgfwyYWymXBj4oIes/PQhRTaS51SZ08HT4uzbFVjCYRNZZvbfeeZ", + "qnm7nOtLCuS2+9c2AOFZZZ3DbenlGOWaLit4SkvEGkstJyhPCLsRPXd3trduRs+2Ezl2YWEVueRDHDk8", + "OTI6UcCZwpQRgWKisC0oVxAy4MvSUgYQ5DCJIV1m8uNy0dIQP1GEELkt/OWXun1vQMI9M2HEIYoxoxMt", + "kO2bxZ7lDG/t7u0bvO6QTHZ29/r9/k2BFV7kSAqtlmLDBOkVMBb6cvZ563AH+Alt5vKxc3rw5hctyFIp", + "zKG1IceU7Rf+nf0zfwB/mH+OKfPiLrSCeKeTGrR7OR5NG/zm9/1CRTOn97QqA+R3RkN0MoC9eFHMSlGa", + "dwdXlvG4GcBtcmaX5JBqdeU1FJ9soM2t0drzOiaqgNJezDZrgdhOPyy/X3fuLnjH9gmVyXIw+/rN+q3K", + "EciliM01QM+EsAyjOYrMXwFnc71dfYDNpSPSPWuB9wnHiAX2zLos/pj1XvjxsDiQwu8OL7Twk0Vufn/D", + "kJilCunf63roaink1NEVm9nve8xOhbYo+RbmzBsL/sBn4W3Cxsq9v57+95//V54++WPzz1fv3v3v/OV/", + "H/1G//dddPr6s7A/liPcPShM3RdDpjOVX4rwdG1Z6QSrwOOj0+ZfA4XtE+NxUMEMcL3RmOwPWQ+9oooI", + "HO2jYaeS/DjsoDUClhJ8pdVd3ZTN4V7XH5+aG0398UenBn+qthHaZG1hFyTD4JDpOOQxpmx9yIbMtoXc", + "RCTYBfqvEAU4UeDOoQxp+3eBxgJK2dorprzzLvqIk+TT+pDZWjxK6BkkWKgMcdX1AExhR2XCYO3rJINj", + "MBmJQ5ad1hnEm7ln7GfKP4SbVHPJ/ERZbr9Zy+npwAeGB/kseiEjwHFH2Y0dlcDoWaINejpYr9tzK2yM", + "jIeWsJ+9iDcJjwepz13clCR5REIagFxxeYIzm0mapWgaTrNOvETw6wWszRmxtTNxqmZaFpkQqy4KOL+k", + "pAtL2oXrMIj8gC/Nff6MJ73xojfjSQY+gIWJdsHmRrxsZP/fnp1o7x0RdGJ78lbd0iziUTphy9iZmZTD", + "zLtQm9gbvXCSMK36zIl9nUqUWKT+0IZyq1QwE+Smee/g9NigXGTcR3w6+Y+2brF01YpngOemdDM+SLbO", + "IEuZweMgJJPqv8shDVu7e2DBun9vb7XOWDWkW8ZlaeSxaWMn+lpIbCMmYQBGPRg5R/iKICR9Atp7XPBT", + "KA7/PUeuoXzHZYLE3E6ZHDppgitUJAvZc+veNCe7DSyUxgjb3dTmPKrtwlLWVIsWTHQCfBa1APF4YbI1", + "37w6R4qI2OXPrwV6dWCX2DKxVMpUy0+K0cHhyYv1fovK8rBUS7OqyoP2YC3YaIWmIIzcR4Nj0kXHR5At", + "a4+V3BaD9IafuUCRORXzw2gfvbVY8qWmkImuNowRLfKQB6O2DDvrrsWkerzto7PMBMTZUErV7MtRFPlh", + "As3aADiTe1FrvVurEy2c+WfPY8i0wCqTxVpNbD6/2vscHSSTPqgqHrIbH0jFKJRG/1dh7b80au+X19G3", + "b6aju9qfyQxLH3fPirea8NKS+uC0clUjmmOGSv2urHldOIYU0edc6XNfteLd3ubmm82dm7vvbgq4WoaI", + "KsDHZZir7cFS7wJ0tO7KuqZq1BhcjvRjG0ru/CLvTtAMS/aDgocV78jm9pNWNRJ1r23DsosB2XxihpRJ", + "KYc3lYUTG+StSxpFRoGRdMpwhJ6htfPjl78ev3q1jnro9euT6lIs+8K3Pi2wV52oeHn6Fm7GsRy5yMbm", + "ZD6cJ8SSayqVrMNxtQoQ/hysV/Npu1o3bpKmjbzkzXLA2F9KiKZefLX1O0B6ddHdNXI+AAjqQ2bRfX0A", + "rEshUz8X99Qa4ncEe9oo832QoRXIgN0m8X97ANM7GU4JccUntooqgEtxvjViaLdDPemdB1JLZhKi49O8", + "0k7urXfNV+b0bKu/ufe0vzkY9DcH7UqrBUv6Pjk4bN/5YMtoHPt4vB+E+2DR3vZSxzK20dFxdIUXEg2d", + "FTXsGCuwYP4Vtq21tFoFoNSBWW+Hw1rVcxqQVkHfc5E90tZobtJ5WuTvVVHF4jQyKeLFus5lJXKGJZKJ", + "gdA0NQ8yFXfIYIBdC+UFJweNCcJBINLcN+cK7BmFOE0s3w+ZIDLhTCvJ2thBv5KFRDGFW8Kse4gNlChL", + "EwmHbE24lKIsdyjBqSSh/gEC9Ls29DvUQ6MKajvoD4ZMzlKlhdh6Hx1yJtOYCOu2RGMKVyrrSKbG5oPx", + "AjUWWmBKGhIxZPo1D0jnx0x/398bDAYDuJMwps62/vfAx013ejvYtyC0JikW4PCYhaMFnD2RMpSykAh0", + "xiVRCiNi2KEaQ3bDm8XPRJ91n7dTr+znuV7l35irwGrbodB+LvTnskLV5+US1a2t191/fFY1a9JWf7Xp", + "OPar0U2u+AkKeBqF2iQa69POeKxIaP10kqi8+jcckG/ZJdN7tDR1G5urOPozJWKB3p2clOICBJnY4sYt", + "Jg5SomEdeHKjZdha4URYOZpbAsLeBwhsVVMpaIhfHPK1eAXnsoQNh7a4issNSW9mBWVmaTSfLJlTxb0d", + "kvkoTX2GiH7ksEHevj0+KjEHxnubTwdPn/Wejjf3ejvhYLOHN7f3elu7eDDZDp5sb25tL8nCaJHKdfvs", + "LK816qsX70KmRy502xdR2xQ4Xzn3bTDwFWUhvyodLd7ozGLvNvJzVff1uO7WQ/Bmg0DpYmipQUqcwAFK", + "At22iYauVBNv8LLtvRlsrvCyrZQXMLgG+ftGpMzc6ZkE98x/HRcGXFys8jhvJ05hQC7rYhW1ip23J9pg", + "f/fZ/u7nEs1lDqwaY5Wd7nFxm8KRHABwJTXBpccVfDQOMLNj9Q3jUraZDJ1uJ0u2gL/hoK0E8maPW2UQ", + "NW3Yrl+MLJPfDZm7xyVbAMIkDABcuK+1AKfrA8x+lh+u1YvDiKchKvi5DB4WXAIdF+wC3QzcyVj3l8G3", + "NJkA2n4AQGWA86dMC2K4/NKN2DTfffQS3oVHODYmkx2EKSpRvPfB4cIEa+j95bo2BszyIZ9b2wW+0YYM", + "0v+CaWsyWHfo8iaM5rOPfuPwTWZJMV71q5rXwYSpv171wa5ZrGgH2wCdWTVuH/2cqW6Z8meVvTVJ7J8j", + "K7BytJT1Us66XfGO5pZ85Qr5192OoWin23GEgjztesb225zra/uvyIq+KCaCI1NRPUtQTRWNLEY0zIRK", + "RQNpMxf04jbpF7beDQlHxjBpCog0WY/WeMk+curLuxO0BjCAf0PWaaz/tZ4FT5bOuq1nO8/2nmw922sF", + "9pMPcLXaeQg5ufXBrdRBgyQdWb9D09QPT98av0JgLPYs8OLdSRFbIRFcix49c9dgsfNn/WdFjKOQp+Oo", + "cKtlAdEMpCosmBfGK5NFDUF4f9JoTicT9ueH4HLrD0Hjzes9uTXebMBuNR35XVrHxZvtmv+XjHumdo0f", + "hgYYSshGpKYzImEG6JwoBPzTQzgA0yFLpbUs5/CcLMW9jLWzvb399MnuViu+sqMrbJwROLg8h7IdQWGL", + "wZto7ez8HG0UGM606QANAF6bWbPSv8+Qrd86KCuk/c3Bto9LGg7unGts2/O4keTvrGlmJ2WJDhnBmdlW", + "2+Veam9vD57s7D7dbbeNret1JK6XSxiXL2PIY1Hgiyu/Btrkm4NTBNmoExyU/SYuPOlGo1I3GhVUMDDI", + "4zcY2NMne7s721ub7SDHfBEPFkyvtGHLssuz6TxM4VkNDynqorfbdFr41CnDYGckiDCNDwIX3185fQzC", + "+EiY1/JFaHMwWG9/7eBq8W0rx1HmDjLZIUY14AKlLKtr0V993flFbi2bpbY5HlZLdV9OCNPUs9g4pq7V", + "LUiZCDKnPJVfoCGuTMLmJOJc3OjbJoPljMg0UuZqkUr07uQHkCma15BUJCnbUJYblyAI3XJyN9rPJRbx", + "M3kTsVqtRpulXzbhbsOu7S5DcyhJg0bcrlBLrpStjjw8xFGQQiUXnK2nnhUAUEE+epJECxNYHkWcMxTM", + "MIMLCFGA20EzHoV9bximfjKaeCMW+BWKuEEcviQksUVOzCD0Z1qFoXOC1gr58MiwUqUY5W5shIwtY1Hm", + "xt3YX1UPS1+mVJaHremJFS+A4ZpPSi7HiE8lGIUKQub7VQz2BAsTCY+ZKdozj40t6Yn29QyxIsx9J6o5", + "SfnEGrhW5YAsZ0NJHAguJSIRnUKBmHcnleTZJQlXWQrt6mjC8mBbsK65LPQcZQaKqXVtL9/56Ekm+ZwT", + "EngYEtaWxOk552SMWQplTwqMTK4TKgx7tIvFm3GpRhko0g0HK9UIahmkguRQbVnKd+YPcu94z0Un2m5D", + "Lhv0equva1zlb6ppgM0y1UtRP7W6GQ/62LgOC7UUiSqHtqriGN0EKS0Hv6cSWqUFzCy0BgkXBbFUwD9b", + "bxOA4jdZdT81a9WWbHy1Mzhviym2HELsFKvZMZtwD/DEDW4lnSfahkomRMQUqnmgkDBKQmdLZteT1tUF", + "2cqRJChMiaWc0U8FtgTHZnsDeARzPjLKphVZX+2wjXvYjGF5qQPo177YJpRI+rM534gUaGViASXCeV5n", + "qwBLKkf+66x6w4JM0wgLZNEA2wxZLuKIsss2rctFPOYRDZD+oHrnPOFRxK9G+pH8Ceay3mp2+oNRU12c", + "czM4m5RmFqTSbz6Fn/Qs1yspseCJ2TDfbwBcSZvILG+Y8s80IhZa7i2j1wVGL2OB72wNmlK1GxotJWnX", + "YQlvKrkty3p3fCo9iW1LtRxX0YeEFqDdqD1JKk1xkRankkMAdVeAt7vRKWctfB4uxqGR1xVUDDQmkHTi", + "plaXGi3EYpupeOsapHKG/uDjskO0bUitp1rWGsvxGQSZeIPLYUWXOqTNGzWaFFb3JgAIIFb1ROGjG+IK", + "rKorlscuNcmTs1qJrRmxJKNujqbcVovyES62Isudt722T6KvFkPzxAIDRIpUCygrCrVcFoUKfxKNuRAA", + "f6w1HM7cbADzQ+s8mtYOdAm9mZEFEiTGlA0ZZZmTFJC8CGJkTkQhRZMLbWRNSdhHfy+YeAAYHSdqYZHI", + "wXn+g0T8imVjHLLiIHXjqdTtHDDjWRQpVHDPX9LNgtWnGQVSZuESTAko3kfVDE0EkbPi3H0FG7WOd8VF", + "2FgJZ4HcK1BgBe5YkeKXhBVFWdaM1zQ0DY3MV/UIOVNtFZ5a+xOVCqCiaoHT5f2lkgi/kphNKXulVehK", + "YasUjBODQAJwHFDYzv5lRHwGwdECcCNv/n9ck/lPp1nj5d8qrxVANRzG7YFx23pdsIFJkakE+5Rv0laG", + "qkAO1jIolfpdAlpz4cmuOkhZEyhU6Wh1TrZLA6sG4rvRbEgSlHvfebr7ZK9lmZTPuqwz0FFf+mpuHi+5", + "kmtYqZM29z5Pd58+e7a9s/ts60Y3LC5no2F9mvI2iuuD1si10ps1+vc///XupHLrswvxzYMbDcpkbfiH", + "1JC5UR7Qu5N///NfblS3HpBP0NThqRvu7RujdKLiSrpAgfIVXrtLsiX2/UHJSYAzMYPWyGRCwA06MnTr", + "5YOpYFC004JxggOqFh5Bjq9MJHn2SgVmuc11UHmwPpXXtG0hObXkkuk4z3hcc52jv5q74QovPG1dbUmm", + "46Z76NfVXs0tdH5rUYxxaBFiILNC33UHdzafKyxLwdL67wByGlz2Vj2TxbyxHPK1mmYAUSy2qFghFNAH", + "FV7RJ+1HxeWvLGfh3rLk1qlS/P2Sfdi8BW/k9fWcyB6nb7A6bbUiH+wBeLuvRuNiHbSlheZKRdPyU/fm", + "/bbIzK2D9Gcn2M37KyQj3uTDKiAt8KMdgyV53na3xBIN3FTIM/E40HhEelmgnk1CQTI1N4J6z1uMc092", + "ZHDJJ5My0OpuMzA3YM5AIpXrBSulLZMuItfOZ1FFdTYAM8POrhx2tAkw7GzGw07l2sqbmhjj65HtoAws", + "MliGlJ1VAK0OUroZjCMeXJoSX1A5uo8GKCaYSZQy2PyVW7XNwfLboW4nKaxNhktNTIhTTWzBmMZkhucU", + "yiHYO5VpKRCTXFMlIWAU2tlHITdQQ6X6pnaG+jWTOLifTxoOHcwWtmHdoH6PMxfRmr8LDr4JVFVlH4jg", + "XZspryX269cnXRPAAKGHZmCl+EY3UTMCLSCzLirY/vnv/vjhcURGMO4qVnxcp2Mx3xtuVgWRREkLHp2z", + "Q4UJUMBTpqog8nE7Q66cslU/klIGwX42/ANAwWzvtjh+SALYkbK+F8uMfgvmruQNWEr7Ege2fSwMmwLu", + "kvx3xWf2Qrg6AONsKJQmNu0U47rNLeFIKm5rWWW7ekSuA0LCKtqk/5W2sfL2S2+s/CtsAWqyqsH2bYh3", + "rs+uf3fJUzDWJmoXY/oZZz2AxnBLamEsDC6dBUopM1oJ97oAFzHyYXv6XmiTzUyul9P6N3KtAJw7TCOD", + "uOZnXSuq7GG0iuK3zhps2tBcrC6yfwc100y8+a2qptlQ9YconGbfupNiabXVOSfKvXtu2ai5ZH6pykjp", + "SssF/LtXyjE2hpW6yB7waDNer7DgzszvFbGIsC3zHxmOySgRZEKvlzCPecEYxmXIkHwjZRkMBtxyLcbX", + "aOcJCmZYyMrYGZ3OVLQoB+DseIB8PquioCCKMOcobLPy+Wq6D+vRbnY5i637lOPzAuxOrZ6GVUlHy0Cb", + "D/PbNuudT/ACvDiNl4RPtncGg+2twa1Qm92wbkCuw/wTW3+v3E5TSl3hO3vRX4pSLbaQJTDXi7peCQp5", + "0BmZpBIEx/uQeJPggKCITAChLSumvfpmsdr18sFbhcqCvWT87xbKrpu7gy/Xa8m6soDXbhodd7lYxvcp", + "Pl9xIdogZoIanpsn5267N9h7s7m9v7u3v7l5F0jLGZGasj2efNi8ehJt4clO9HTx5M/N2ZPpVrzttcMu", + "qSlL04ZXf9XvNkbZ5IdkGSeoJNLQmp1DQkS1Wm+1yrUkEWWkJ7MMqdVpiktkgbl/X7n/b+bnNzNYqjuc", + "lydZVCGwyolT4qyHwbayk1l6d1GdzfHR8lncKgOpOhA/v1WHAuzVbjBQHmGz85moBylreQy9LbzY+iBa", + "mhW36ijy3bDDTveucgPFfexdkpOlDbfsAK8fch7f6ZQLqmbx8tMiey3DsIa46Q9ShWUspT46njIo1V38", + "OQuTKxpR+uNOtxN92CnvGft7e1QtC3+bMaBd6qJW0CKMDCrBL6cCvJIbHsJEsmtbXY/5p83e5jOIQ4g+", + "7Pw06D0rRxx0DbWK5Nt0b5d+HbShYbH+nKtbtPnsRhHXjp7LOOhX6quelp/LFhjX8nheGNkdHS7htrTA", + "+ePaGldQchoV0M/V9OzhNioqTSGJ8MIHjF5w1MqK9VhkMjQmU8pkG7/t9iBz3O7Gw04fHVh0arBl8zL4", + "peahAHqBT2gck5BqHdOY/s0ZDFstfXFVW+JmhTHcVx5tre9X156thkhYlXC16pjsf0Y+7mdZv+0s3mXo", + "HeBXcyYq4HPBi11EJwizSnVMyuY4oqFNpIfESIhX23cgaDnLWhkgcz3Q+Um6aMoVylPoW/rbUtbsF8zG", + "T67B37oEM8MwxNYXAUTJwLnoMvF1fIQSwcM0yPNHIxh0jvgh0gr82RIlf3VI7l36NyAxe8IFWu3faHJo", + "tPNPNq13xTepGbZ5qTcHq5f6Tpwi3U6ahKtlmHmpnQS7EWz4ihREj4umTPaKJliYzPsWEv2sSMG6zWt8", + "yYFWidLEXbBonqpzkue6Ba4YfHG9RyQi+piqN4J4FOZZElTmUnS1SN3cezpruuKEG6n6QH4lJNG2CuAf", + "QX8xZgvvwFzNy+wsWRu4erbSXHj1TK0cS63y4J6s1MQal6rowm3C7zdSvuLzNngplwX3d4afXVTN6ggo", + "TuCXlLSzZvx5+6ULe2v0H9+FW+4hlbTX9uqhAonqAHsz5HHXfx4LrNW6MvPu+K7nfWzxxlrGTTCv1SzQ", + "otf5oPcP42VGo/7+xk9/+3977//q9TZX7GZJRC8kEwg0uiSLnql8o230fhnkFGD3tTI9taxCcAw+JAAQ", + "t5uxON7dQSY0Fr/huDYFiNAqlK3ZXDmhv/2lOb6pQMa3ICdXsuxnV6W4i+qdirvjaC0mYupiyV0i2Xp/", + "yCA37ZIsJCoUw7IqjWPUH2T2SSECHV0YNbBP2PwCjSlUF5RDpq1aHAQk0daEra9CTYlsDtJHEBwV27FF", + "uVzit72QNPEEBL07qSHkvn775vnrt78djV6fvvjt4Hj064v/hRCPq57pIexp3tvZ3bOFsYuU3PQs8Wdg", + "+H8WSK2P3QzUpYe/IMkSqo17FGYqASLBhRsUXkZrkKzgKm26XM31m0FvHmQNeoPdvnBBlcGzL1H08O3S", + "KodzHvW0Rt0AfO91YBpaeIO1oSkTBN9p8nNPxx610XoTp3SKPa5tnwv7ixQndANamaRTW//GKk3+0Pmj", + "alUAIw0MqSoo9hW7VKpec2R9rBWpUV6BvRyvkTKbLkkL4Vzl3MiYqQ1bRNQH0RBywKZeliCb7zIHyNeD", + "j1bnfS5V5QszK4ykeW1OnMZa0amXEOhUk+ZqRgQpLAR8kKOt35BkNhWkBfCHKcKWEJGHSbo8Eq0IwQ2n", + "RGuZs8GRIEtwrXtgl6Ppn+DrrAfw3mNZu/KCeeTlbjZfPgfk8zNX4o9OXBMwjIo94cf5LnPRMpo4rqov", + "RpGr6vM273s3npVVS6Rf096qMGfeR4k1ffz4d0zVz1yABdIMs3HncOFg3YREAOxYFQy8FZI2jUk44qla", + "vv9tBXGLsZGVgczLiDprCwMTB6UM0iZZ4IAg8jHUKa3JQYJUULXQ5rlVhseQeedqdwIhoSP4Oe8Y6iV+", + "+gR+yoknR+ElYUTQAKpR6v0YYwZKOnp3UihKZurT1SBCQb18fXhsLVyHMgsWC1XAei7U7+D0uNPtzIkw", + "Vl5n0N/uD2AzJ4ThhHb2O9v9zf6gA4r8DKa4ARXEbcquTXHNbKXj0GpCz91L+kuBY6Lgi989yecQ6mZf", + "B60XTwt2S4KpsIZLEkFSvGEYqr8G9Hh3oO6bU7lryN7aTQeZrZBwQZLXdnHfg1IJewemuTUYWCxtZY9f", + "SBcxMeobf9iAxbzfVlqdJZEHTL1mWTjdMiP9p25nZ7B5ozEtGwrsXV/Hbxm2eaMEDMLdGxLiVp0eM5MJ", + "ZvN6bQROcccBIxX32u/v9ZrJNI6xWDiCFamVcNmkGBOJsHvXQPwqiQItKqDkSx+9ZsSWzccKYRMsK1IG", + "pWTdh5pDy7vAtO0WOcPFec7DxRcjYakPZxZ/KoszvV0+1fj5y/FOxsb1hbSPHKiz4dp7YKDnOKuD/GA7", + "ZWfw7O47PeRsEtFAoV7GwDYElkqIMokAotrB3XCB/ky5wiiLIH9EW9rqrOOM3br5UbTxkYafzPaOiM/z", + "ekpEjJmJxzfvrNj0te1svOD5dl56qjnGh2oScFI53BdzUIEiV96ixWOrqgzWj6MdT9K/7dNML3xAxt+5", + "hx1uJ5uVwnzILQf1FVEqyWPaTvZWZ5wrIV5d7iVRXwvPD+7zyLK49d/gLnosDPySZBpevlq1Q2EjESkz", + "BrBXAzzLc+Tsdz+Ulb83+ZNCYAa40nXTUEFBmas8HC76yNHUGP1qAag+gsA8w/qxcqqH97XssK372GEw", + "4+xy4vsx9f2YWrbLDbe4KcDGLOzyFj6IG3kgvj3/w429D999D+19D608D4xcWe/CH3zcRzYIEurQyxlP", + "oxCNCTIQOy7cQWHRn35AWAQzOieAowZ1wdJI0QQLCGaIUYgVNte2jY6JpW6JrLkN3VzPhb7lBK5CJ0gy", + "Aui3URPkYR70RhkjIdKfWLS4HMGuVh3a7H2vgz1rMD8a0dWMS5JByDFVOM0ho1Ya6xia7Q/ZG4stqgkI", + "8btO1kgSAULqEv8PZwgPmf3gRydCXOyRxHEuubAAmDpqwBDNstSzqfRIRzLgPniXN4RhpnoyIQGd0MBO", + "65IsbAiht8FWpX70gN04351kOQJoa90PEQaIgH482KPsGbKcVL6/YRB3G0RpmF9yOdQaLMY4iry1IKYR", + "H+NoZOhzSTx3gi/hDUuUYhV5d5vEeEhMRfBkoWacmb/TccpUav4eC34liRh21vtDBrH/ltYk7OYKIrqC", + "2mFxwvU+Ezw2fW6YIW58vCSLT/0hOwhjyhxHwCc4khyRa/gOSioBTIORXg38YHaT/x78MJWKx0WwTcd3", + "Zpg8VUmqbBKDJKrrA5ocMsXRRwcn+GnjY97jJ7gsJjjUfFJ4xUwJdOumUcsR1rMfwaue63YCBBh29EE6", + "7Oi/pwIzZZAiMzxENC0u6VoGyK836XqVwgFmKOGJKWYATDXDmuVKbQA8AI4ipGAruW+14g4r2TAfi/YW", + "jxuh3gw2V2UbUYZOnhc202DnqX8/SRII4oso+e/z178hOJX1GpjX8gghk0XAtMKAwhSuTp1Me4GDGTIX", + "VVC/btih4bCTXeeG6zDWVNqE+V4P7hR/0kP7yXTTpeFP/b5uylxX7qPfP5pW9vVeSmIDPTnsfOqiwoMp", + "VbN0nD177ydoE2LWeUkQoDVzzK2DJMEUwE0KJ745IjELEbenQLRAGOUSqBi4MqYMi8Wy3DUP6S0F+cQE", + "zxWI8XEIwXLDzv7QhcsNO91hh7A5/GZj6oadT34K2FvL5mJpcJ5ll5sZE+0NBuurwZctfT13li0uBr6w", + "DdhoFWWVHvUKWuTPb+t+4D/a/syufjDTnedoOMbwd873R3gBUdDYi5ao5wqionZjFpDIqd2rHT33f3mg", + "FysgUXTfDPpQ7Jldj2Xg8I+KHWGx8m201H3/wBw3uK9DpeS2fxj+fXT+c4/33PrOydyFOvtLYwDsiTWl", + "kXkZYYnOYUy9c218v4Bf+/a/zvYDGL+LiE8v9o3pjiI+RRFlNgS9EKis1QNLS/jIIJ9k31kgFFeXbM1o", + "Ev/+579gUJRN//3Pf1k48X//81+w3TcMoheUNb6YESzUmGB1sY9+JSTp4YjOiZsMFB4lcyIWaHtgff7w", + "CBWqq1stTQ7ZkJ0RlQpWCNU3JcKkbdBeFej5UJYSaZFj9It0YuuXmNhGj9/G7WVDynvd0V0PAh/MoDAB", + "fSo6HgD4MmpqO1tLtON3mZo5l5ym1TDNWrDeavmiyLUy3NszA7yhgAES+/YdPLCTRmvn5y/W+wisLcMV", + "UKMGbIe8GWtG9L/LpNUyyUiUskABKhvZZOCUljv9j+w77bz+tsVvye1vy5bdwO9vnD8AquhW4PsdQIs7", + "AD/d3H2Azyl/5PDC7i5Y0HTxQLGCjvfqNDdPCiR7CGcAWnNADOBQ5QKdHh4jHIaCSLn+n+0q0DM1XJof", + "HYgzKArwELfWdixcWIgqa6qVGeSxiIMzO2qE3byq1SCL59tGqVRE40mXVY3Ij7y7Pz0qnd7kGMkrVua8", + "9v0kWRmnR2XAoShWzi29ACdASKe+ZPu0yEWrHFImAjA7cpaqS1Y8Hx+5DXl/rinbdcqqZ8M9CMWjikB8", + "QEFYztIs1nh9TNz8NltFh426xHP1dbHm4P60oPv2YvnY/DG5scIK2bQUNHgCjQfoS6IMikDnDhfa9uCZ", + "+DkRble7ktww62xa5lNk4BBgQnA1v9z2PTavtDN9TXvfkuUL5LmJxmJJ/l1FaWHs5rRaZuAe24Kld2ff", + "Qg83Mm+/3I23ZTAPkSHsZuw81kKREK1huWDB+vdL7y/O0SYkKjdihZs3CVESYQXRkQDEktlZ96XXHbBi", + "dVyt01m7ljI0ieh0psz9R0gnEOajitVmYZRb9zDKrKqrwIrY6KbHmDJ4qolsL5DmRCj0+vDY0L94pG58", + "hHi31aaSE15LT9e3Z696hAUcAhyz4Dy/TmqffGGDyfB/KQ3w/nfdI0yFo049aFIYP2P9TRwqMqGzfcr/", + "a+vniI4FFov/2voZRwll5L+2DyKsiFTrd8Ysg/s66e7bgHnEzKftF1omGogmNgWAwxUKf/ZWS53fvf9N", + "qf1m0jdS/DO6ftf92+j+RXItVf/tUtypAWD6eKAbrozZfNSGR9/RMO7BaWo5soCGUbpFyvEwZlwqePT4", + "UiNtPCrNOK54bLT0/ucbcunx4Vj3+KgLhIT6p4C/bjOP7ukuwI3j3pVb2+/9XwQcxGM6TXkqi0lNMVbB", + "jEib8BeRsgB+bGp3fjw3Kt5fMZcO7vPouHe9+jvf35HGX11QI7zNhd4qnd+91Vbnt+9rnd+AIdqkSAsS", + "33UFRNYbYjQdHGJbNi6hRtZjR33j8tki6K02VHJzAYEFsT9k/0fbH78rguP3P7nsq3Qw2NqD3wmbv//J", + "JWCxE8cqhEFlc0jEPfjtCG5Rp4AjCSWh8lzP6jhMhVlgPQeC/R9nIOUXye0tJMeF3y2kVhZSgVzLLSS7", + "FndrIpWB9O/dRnL85iO4hSP+Nq2kb/x6pGTByXQyoQElDMoRQE6rrMUDGkvu+83ILXMZmb2PLAQTlTSR", + "1mZkJrVWaOh5BdR7DyQ7zku+3Lf16IqtPs6cDJ7Y6oXWXsu1hWaD7Wvjh8H9nl73b6g9ZhYzFlGddIlW", + "uj1FRUw5nThVEASbwxBBlDESxqzJWuyjwyz7XKZJwoWSpiQPWAimaOdMWwi+8j3lijy+EjxQdoYS2R0y", + "KMqqHxsUjY1LsjAFdyhnWW2dbKa2bo0v169c8OhBt9GXV0L91ZxaKaH3vI1tfb6HU0IfTHTci7p3XCp7", + "upZtDLC4xyTbyTxLJqUfKJuuP6qIZyOssrkVQNM8qtYGThV3xfs3ZtzgJ/kh5E4jHACCnH7NgBvZ7GSD", + "ZlZsClKOBY8iIgxoVZIqV91ryLLBUVaoXmxLaVzo5kcpUzS66JpoGkAekAizhUVuGbJSZ1gpEidasFks", + "IhihIIkZcaWsmR405amEt7pI8lKXCEdXeCGHTJBJRAI7NygBKUhg8N2iqI9+4ZDujfAUU2YzkPWbpijY", + "D3LILmgYkZHN1r5AVCI540IRRkIU8zmR5X4JFhElAiZxiDXlJIrxAmCTDIKcoQ9PiIEmKuWEc/1vzEIK", + "JbN0z9mU94cMo63BAMUEM4kopA1LPCH6K9sGgkGUBvQjwmhn8Mx+VVk3gPZ05F/T+0UIMucBHkcLRDQX", + "w4mo1mEBs/0F1Sf18k2okGa9MveirU1UWlgqXRXNsItSFkCRx1Tof3GBUmaPV92igEx4mKe9hCNUZMXR", + "bNr+mARY05Pxcj8AmMaDIBW+w1EvdaF833+iklmY3jmQyiedNB0Q7KkQ1pxxNYM9zWErrf/YwFU5U30b", + "h4x3k3CBMCrwde5QgErXbIrWAGDsIi8MxlxtyYv1H93e0dvXCgK3/Q3E12M5n4CJ+GRS2oCrjyazgZel", + "V9RZ+Fvdp4euImRRxIUUTxmXigZOGFZrFn83Hlsbj8sp6+XmCReXRd2qzL8/c3HZ1vo6d4X4H5URVpzh", + "V3gPoIcHELEPfx0AzmhjqGimuXcDrcpf2S4FpYsq6eKMOYo4m+pdlDvF791rX7HogigFtdyZcs4Joo2Q", + "kf3RFJXUk7El+8DDH9hWH1oW6d7v4S7oN64QjZOIxASKTvYMs+nFzrRqUxOaSjTLqv3dTFbqXVVMHTa2", + "oDTX/12nDgFfuQVbA+29vlxeoRrx6Wq4sKxzh43lwQsbMlOzmrgC1xcok8FaoTXg3OhqRoMZYIeB3arb", + "N9BiOEkuMtjU9X30EjZyET0WOl8zkNya1ySPiIEEm8fxxX69rOK7kxP4yMCGmQKKF/vIlVLMzg+p3ypi", + "gelZRFgq9JtFOFvLjHFY0QuFtb2ZzW/dooTlsLZD5kMMY+TKNkgn6KIAHnbRgB7m5O0rPn0wZazbDEZu", + "5qI4sqYj8CZhYacpxoJGftywzcHAh5HbEsPMDOOOIcxqg3nFpxkQeomVcZK0ZV87TODieRwv4WG0lksQ", + "JFXIU/U3qUIiBHxsubuJudEaDmwRHHypGZUZqeQ29jqwnzeSyCATe0mlhWqn2yEsjTv7v9t/zeO40+3Y", + "8RQQjW+g3K/Agqs2WI940StTAHz7rpbfBMqtLOwLWG6Vk8Oa080a+Zl54Zu/WXQ+uwdkQ9APKk7cr0kF", + "LYy37PBhHEmGEznj6nGhR1lXU0Vra3bVuFn29PDC1FXqaBPCcW4/PXdffgXW76rIDjdm5KZ77yEe9RE8", + "5kxYWZvNhIsq5NCq2I+vnpG+3JLUptqGQ77z5s39fK0YU2vqdRFhPwhN5SacKh5jRQOoGhLMOJcFth+T", + "GZ5Tbq9K3Z1Vxpng3DB2pg2hv9CsemEdwRdWkd+3TiuEi49sH3343Abe+79wj/Ivfi7Y5ZnE7zrlG5C1", + "oayxoGSCEpxKovWqNCYoWARaKpoyMQQHMxTgRKWCQAUsgmLKaJzGBVeDNpzEHANIxcVmfNFF41ShCIsp", + "2EXmoQmnFyTgcUxYSMBDNmQzgudUG3UCRVgRFix6kkDlzDlBV1xcRhyHYOTbKBxTeUsQzYGUsy6KicIh", + "VhhUjQu940cmi+ciK6ZpDGtGrnNuCIdMpOxHgwaum71wA71ARCo8jqicZUXXAhwSFnihts+/bjH25b3B", + "50RVJ/pAcTm3kqUPGahT9Hq64XwdMTyPLBiZC7uMbcT8EqVXNhuR5fQHx0b/mVvazNXN8YGueDISL9vF", + "X8fdTsZ0X839zsNf4HCBwtR0V9iVwObf6q1MJlCK4U6QWmmW8bZXM1l1qYzMN5J5Gx/dn8e38KZ9JZKw", + "22jYN9UxySf9NYhcS9VbydwHciNaX1LBK/aAItjFVD2Y+sRFQco9FnenFdhma2ZyuyidlMBgfXH2XWxX", + "xbYNObit2Ha+2dqlekGQU9aDKE2/BLdu3EZRbV0H/6G5IJXZFUTmg4vI/O7g3sTicSYIjWhM8CLiOPwW", + "wnSX3OAEXAiD/wCIEo8Jf7TgNSwG6INvrptJiK7LrXx3crLeJCWEWiojhHrEEqKQFKM/i32l/udECBq6", + "EuaHJ0c2YJZKJFLWR69jCnXFLwlJ8pwSAPLo6/k5JIx6MeYS5EW3Q5gSi4RTplaOIn/1bgbz6VYlnO9Z", + "TlpA6+8X0q0vpMGz//jEGUgZyJowE1humSqsGkMBXWgcZaZCu9bL8JinunUtgzSZ9HpO4RSc0IjIhVQk", + "NnGBkzSC7QbFEWztTPudWeUuRMXqnWMS1hIiYiol5UwOmc3WSIjQfevPdfuFECfvhYDCmXw9NULy6wif", + "04MxEWNYNVENMIugcn1nv7OBk2QjxAo3hGjZ4X3GkH6GeDgkF/GYRzRAEWWXEq1F9NKYJ2guUaT/WF8a", + "UDeC7750ZdDb7yxN6WM24d7iaYZnM2b+pvKqrFhzF5OPTqy9JMXN4uQPLLRfrMmVck0QHPUUjUmGXINS", + "RSP6wYg63QiVigYm6SeHLHh3kqMWDNkJUUK/gyG5LIpIoJzDZiMRPNgYpoPBdpBQgD/bJjA4EHjNj2Po", + "8fD0rUkEJTEXi+6Q6X9Aw28OTs3t7gRbb0JhoIyoKy4u0fHG6xUhxudApv/gGD0zwaXYAd4F/34leHNE", + "kMY9JBu2KE+WmUo8+eaDSK0G992v8Dj9CgDJlM1mbSpwAEqxnKUq5FfM70OY8yiN9T/MH8ergL0UDmbv", + "4NWvRts1w1nZjZvgo9iUdk4hMcUdH+TSwxDsscasasK5KYASU4oG9J4CB+pb5O4v774v0vErvO60FHWF", + "U7+avXXfJ58dg8O4KNLjsWxzw2luJoov9z5dYdrsfXoe8eBSWjCUottQ220AMK5/zAGh7RUhqAmQm4ks", + "iBAi1wkVgPxWcUAazB2JMFJExJThaAPmbBoBaGvnxcJzTiFFOogoJKnREFCLIkCnu5oRhvRswFHlGijc", + "6EpbWqr4TvEyUnE0JgGPiYP7XveZbn/HVP3MRRm7+2uRi28K9Nfz0VPV81wBV97c42fBl5/gawiVDlN7", + "oexGtPaS5z8aV1AXwdoMO9sDOex00bCzFQ87egUOMbhQsUK7KKYsVUT20ZHxb0ES7N4ASRJwFkqHOu48", + "eNsD2ZQSa9iyIb9yD767T7XHchWQ8sx24hMP+j2kv4ekHbRW3HB2T4Zd2HQh4qky7n67r+xbIVHgHlm/", + "97vawh75btu3keR/t9u3JKNglbW4LCy9kexJKmek2eX2ylTySdUY0KxddU05Q3/wsewiRq6MN1xI1a/J", + "Pf31qengPpD2dVc3Qdm3c/8Osd8CYj+nlR8u0QRY6iPZcYfBTCTXBhEWu7R3y0NgSQB2Aw9whF4fHg9Z", + "oEWRAfcTJOYgnSwguDmFD/5+jl4cnnXREVR6RL+k4/U+es2ihau3be5ohsxoYkZ4BZihseFaEvqOZzN2", + "4J67DBbXHTxQ6WSzMzw3K26tXJB4tzMjOASN5GPnFTedeTCCz17pDaTXz36ZLXtnqfLROSNKLHoHE0VE", + "vdkTmyfFMtQKe0g7EDiruBnoSd2hdNhneZ9GNzDgFNtbHQ9WxafvVQ/uvkLo/dySmTgRU29unALWJ4Mk", + "AxwuHlcsk5yhTDj6RGDxuM7qBjRlCVtZttTAgC6bIr+/Ipf7UtlVQoL/T91dMNNHe9GUlNZJM3FWb2Tl", + "Ta9LDp4ZQGJ7URXgBAdULboIR5E9o+xJkEWk9DL1dywIvgz5FesP2VlW6cQm9KLD07ddd1GLQiovTQv2", + "LraPXs+JkOk4GxyCjWZujYHmJBwyxVGAoyCNtLpBJhMSQC4uFDCRDXe52VA6d7h38k681VYKUe3poyvy", + "5ucJWL2cLaoct2GWekOQIMI0bob/tooaBBxCqMFYN8oZomwS2ZCqQHApkW2qRyI6pePIBgjJPnozI0ji", + "mAxZEmHGiECpNFHxeui9RBApU5PgrRsAmFzDUV2UQ/slgisbmhBxLqSJJtAc/u4ESUWSJWx2Zlo+gTnf", + "kW5rGrc9PZCTujKGZleIfQXpBTGcYgiu+SiNXADjvYaimwE9tJb4WDb+G0GnUyL0rsBGyJpwPLOtHTnN", + "pi9lLDcWfDzP3mpX8DFrtZCVWMjYWwrNNsrRrsPOzaL+PJ1f0kb0PvvoZlnEv+qPWvZdzlb1D8I++sxZ", + "fit19M8LSYJtHVg5hz82d1Jh5KWtWkq0XQ2r1Tqz9i4zXVvjZz0YbNZjRsvCpfTZJoP362OEwf2iPNx3", + "SbTHzVsltKuSbdqQ8r8az/6r4MC7AbJ/YJSTWwDZf1V594A0/nD4J96N+lB59KW7Z1dt9pvHor+r9HkD", + "SA9wbE3p80bq2eDVpYbSO/tOOzPJtvgtafA23vEG+rsj+3erv4XJUCDWqitozfAkTtTCBbTZu8o86EzS", + "D6TfcBGcxa3e3VXwLUI6vxx7OD5tDOj8NovDP0jMqC3eRyU6PvJUXX9kGIPFPVc6WDb0qdPDIpjROWl2", + "upd3sCVRIkgv4QlcroSGYJYe7ixTWPSnH5Bt3mKu2n9B9UcAyychCqkggYoWphKnlgimjx8kElxbAvCc", + "i0VzlIjZIj8LHh/Y2aw4D+2ess6wPM4wXvRCrHBv7qTNEhfaZ0R3unhKLfAQZejlc7RGrpUwNSbQRFs+", + "iE4ykppy+xJ4cr044M1Bg2eTfiCj6bjNKJdUC3ltq7GgIJWKx27tj4/QGlQfmxKm10Kr+hPQZBPB5zQk", + "YWmMnTmPDFU3Gwh6U7+rViqy0nHOuDCDexAdps2BNP1Ak7JYyEJixpRhGNzKuhzlPWWS+HV/mDIXgGPX", + "yI3i+xFmLb81Z+xoToRKmJaIinMD8bz+/Zh7zMdcMRnKnWml086F5yx3XrfLj2qZtnQXhR+y3Ln7dVu/", + "+3pSeqh8lNk81nU+zwzSJrf518WCg/s7H+7bXf7uEaeAviTO+C64yqEB3aKPYV5BTHdI5iTiSQwVyeHd", + "TreTiqiz35kplexvbEDs94xLtb/z7Ml259P7T/9/AAAA///ULxEss9ABAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index 32f584193..d69b1e2a4 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -199,7 +199,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuProfileSlots, err := meter.Int64ObservableGauge( "hypeman_resources_gpu_profile_slots", - metric.WithDescription("Available GPU slots by vGPU profile"), + metric.WithDescription("Virtual functions able to create each vGPU profile (best-effort snapshot)"), ) if err != nil { return err diff --git a/openapi.yaml b/openapi.yaml index fcaa132f6..f7d051c54 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1585,7 +1585,7 @@ components: example: 1024 available: type: integer - description: Number of instances that can be created with this profile + description: "Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer." example: 59 PassthroughDevice: From ec367cfcf10e956fba56727439cd2b9f11ac9c8b Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:12:47 +0000 Subject: [PATCH 14/19] Report conservative per-GPU vGPU profile capacity Counting every free VF advertising a type overreports concurrent capacity: sibling VFs share their parent GPU's framebuffer, so one 48Q assignment revokes the type from every other VF on that GPU. Bound each GPU's contribution by both its free VFs and how many times the profile framebuffer fits into the GPU's remaining framebuffer, using the largest still-creatable profile as a lower bound on what remains. --- lib/devices/vendor_vfio_linux.go | 51 ++- lib/devices/vendor_vfio_linux_test.go | 28 +- lib/oapi/oapi.go | 479 +++++++++++++------------- lib/resources/monitoring.go | 2 +- openapi.yaml | 2 +- 5 files changed, 310 insertions(+), 252 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index ceab85ec5..d67ddbc00 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -81,22 +81,37 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// listProfiles counts each free VF advertising a type as one creatable -// instance, matching the driver-reported units that mdev sums through -// available_instances. This is a best-effort snapshot because creating on one -// VF may revoke the type from siblings that share its GPU framebuffer. +// listProfiles reports a conservative estimate of how many instances of each +// profile are concurrently creatable. Free VFs on the same GPU share its +// framebuffer, so counting every advertising VF overreports capacity: one 48Q +// assignment can revoke the type from all sibling VFs. Each GPU instead +// contributes min(free VFs advertising the type, remaining framebuffer / +// profile framebuffer), where the largest profile still creatable on the GPU +// is a lower bound on its remaining framebuffer. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { profilesByType := make(map[string]profileMetadata) - creatableVFs := make(map[string]int) + freeVFsByGPU := make(map[string]map[string]int) + remainingFBByGPU := make(map[string]int) for _, vf := range vfs { creatable, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { return nil, err } + gpu := vf.ParentGPU + if gpu == "" { + gpu = vf.PCIAddress + } for _, profile := range creatable { profilesByType[profile.TypeName] = profile - if !vf.Allocated { - creatableVFs[profile.TypeName]++ + if vf.Allocated { + continue + } + if freeVFsByGPU[gpu] == nil { + freeVFsByGPU[gpu] = make(map[string]int) + } + freeVFsByGPU[gpu][profile.TypeName]++ + if profile.FramebufferMB > remainingFBByGPU[gpu] { + remainingFBByGPU[gpu] = profile.FramebufferMB } } } @@ -109,15 +124,35 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro profiles := make([]GPUProfile, 0, len(metadata)) for _, profile := range metadata { + available := 0 + for gpu, freeVFs := range freeVFsByGPU { + available += gpuProfileCapacity(freeVFs[profile.TypeName], remainingFBByGPU[gpu], profile.FramebufferMB) + } profiles = append(profiles, GPUProfile{ Name: profile.Name, FramebufferMB: profile.FramebufferMB, - Available: creatableVFs[profile.TypeName], + Available: available, }) } return profiles, nil } +// gpuProfileCapacity estimates how many instances of a profile one GPU can +// still create concurrently. When a profile's framebuffer is unknown (0), the +// free VF count is the only signal available. +func gpuProfileCapacity(freeVFs, remainingFB, profileFB int) int { + if freeVFs == 0 { + return 0 + } + if profileFB <= 0 || remainingFB <= 0 { + return freeVFs + } + if byFB := remainingFB / profileFB; byFB < freeVFs { + return byFB + } + return freeVFs +} + func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { vendorVFIOMu.Lock() defer vendorVFIOMu.Unlock() diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 4555f5c2a..b68a9f912 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -184,7 +184,7 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T } } -func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { +func TestVendorVFIOListProfilesReportsPerGPUCapacity(t *testing.T) { t.Parallel() sysfs := newTestVendorVFIOSysfs(t) @@ -196,8 +196,30 @@ func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { require.NoError(t, err) profiles, err := sysfs.listProfiles(vfs) require.NoError(t, err) - assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-48Q"), - "each free VF advertising the type counts as one creatable instance") + assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"), + "one 48Q consumes a whole GPU, so two GPUs mean two creatable instances despite three free VFs") + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-2Q"), + "small profiles stay capped by the free VF count") +} + +func TestVendorVFIOListProfilesCapsCapacityByRemainingFramebuffer(t *testing.T) { + t.Parallel() + + // One 48G GPU with a 24Q already assigned: siblings only advertise up to + // 24Q, so at most one more 24Q fits despite two free VFs. + remaining := "ID : vGPU Name\n1147 : NVIDIA L40S-1Q\n1153 : NVIDIA L40S-24Q\n" + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1153", remaining) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", remaining) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.6", "44", "0", remaining) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-24Q")) + assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-1Q"), + "framebuffer allows more 1Q instances than free VFs, so the VF count caps it") } func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 8f02bcf86..70d8a32c6 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -869,7 +869,7 @@ type GPUConfig struct { // GPUProfile Available vGPU profile type GPUProfile struct { - // Available Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer. + // Available Conservative estimate of instances concurrently creatable with this profile across all GPUs, bounded per GPU by free virtual functions and remaining framebuffer. Best-effort: recomputed from driver state on each query. Available int `json:"available"` // FramebufferMb Frame buffer size in MB @@ -18597,244 +18597,245 @@ var swaggerSpec = []string{ "oz9c6UhpQXdGru6D6JBI1tNs25MMJ3dD8WWRtJmvIX8JTmFBQ9JHsLsg+M2B31R22rniSULCzP/THzKb", "x5L9JM1Fmv7Q0EHNCBWICzql5Y7LftZ7jGW9CWc65ro1dxY/bIpNk9k1oE8GGDyFiQkNNTBfxXRuuyad", "buc8g4a0gqlMmrMMXrNGkTzivDbEl6dvbxqRmgg+oT58YoiQsU+toeZiNV/tDM57m/9josk1+4HGRpmJ", - "qol5WEFytO+3O4henr49bRpThq2IiqOrzSmLg1oGhu2iXSYpM+iOBZ8KtqjLxrwxt3K2sz56TqTqkcmE", - "C7VvA1ngQqqA7I4XSJAw1fxghkIjquAOQ9JxBHs061Ur/PoXPaGJwDEZp5MJEeWk9mc+Xbrw9ij2OPd+", - "1s+ReSHLqj95Xtantd7e1mo/LXEDmO0THFA2XW+93B6HYGUa3cLyvffzx5kFB2zCVdCkzAAEDaRCH/2W", - "wWdqUkuUB8t5PIUV6IKmPKXT2ULSAEemRYN9RVnRwQe7obWGfpp/aF2hHj3dj7rqdh5am0+TFPb9+Vnv", - "+PW7jTgk825pTBDgNuMR0eNeL4inuUuWz5OqSlJp3uRpMYwh2+7YAq0ykdGaSAUB4aGO4gpHIxlxX8zQ", - "G/0QwUO09u5nk+2qR9BFSWkp9e8FKpT4e8+7YwCnpqHbc+iw6rItbXCv7Vou12HcO4XplTr1bRWTYlTX", - "seqQx/yyvND8cjXMrmmkud9DlwVVcapbRRaZZCkEyVIuDAKZT43X3LpmJEmwwIpEC6PZZGK2DJtC6jfa", - "5JoEN0jDeqFf/2SgslJBRmomiJzxqBwHsd2tw61LiMWdE4swaeZUcPwrjmIsLuEkdoo8SpmhQDmUe3sV", - "fMlMqeQGk/rlzZtTY90rIuY4qiYDyNoN/xGJ8AKNiboihLmpYIkweskzKM9qTqJsQPkTapQQQXmZhp1t", - "T7/nJj4YTQUOCDJfuWINdkkkxBe1JaXtxQOhHQREyob13Vy2vvbTSRq1W2PfsDZX1jYJbrLAbw5PHVpd", - "Vg3AkXmrTuVTInpmy7myAMuXdksux010XTHOSL0zwccEgBRtBkYxQdLFNwFQpP68lJRYEA6ymEZn+4Fd", - "YEjVNfv8fSvtsrrdfRf5MWahr6qCCbw3md/TNNYLoocsUlD9aGiCn0wyurEDivkDguCQMiJlJW02SEXU", - "6XZ6Ezur/Y2NiAc4mnGp9ne2N59uLA8jXRo/bMOlRiFdZl+6oCoTduOyS011A5h0mSU2cJK08MAZOq44", - "H0A81eMVoZKCPtsKGp4LYB8MahnA1zhQDvsVXHKlK1dc3LaJZsnSfKDBrPrQ7rNnxf058N6B58W/HPtv", - "1Hgf4mIgplHziHF3VOhomPyDV6PiwgfwyYWymXBj4oIes/PQhRTaS51SZ08HT4uzbFVjCYRNZZvbfeeZ", - "qnm7nOtLCuS2+9c2AOFZZZ3DbenlGOWaLit4SkvEGkstJyhPCLsRPXd3trduRs+2Ezl2YWEVueRDHDk8", - "OTI6UcCZwpQRgWKisC0oVxAy4MvSUgYQ5DCJIV1m8uNy0dIQP1GEELkt/OWXun1vQMI9M2HEIYoxoxMt", - "kO2bxZ7lDG/t7u0bvO6QTHZ29/r9/k2BFV7kSAqtlmLDBOkVMBb6cvZ563AH+Alt5vKxc3rw5hctyFIp", - "zKG1IceU7Rf+nf0zfwB/mH+OKfPiLrSCeKeTGrR7OR5NG/zm9/1CRTOn97QqA+R3RkN0MoC9eFHMSlGa", - "dwdXlvG4GcBtcmaX5JBqdeU1FJ9soM2t0drzOiaqgNJezDZrgdhOPyy/X3fuLnjH9gmVyXIw+/rN+q3K", - "EciliM01QM+EsAyjOYrMXwFnc71dfYDNpSPSPWuB9wnHiAX2zLos/pj1XvjxsDiQwu8OL7Twk0Vufn/D", - "kJilCunf63roaink1NEVm9nve8xOhbYo+RbmzBsL/sBn4W3Cxsq9v57+95//V54++WPzz1fv3v3v/OV/", - "H/1G//dddPr6s7A/liPcPShM3RdDpjOVX4rwdG1Z6QSrwOOj0+ZfA4XtE+NxUMEMcL3RmOwPWQ+9oooI", - "HO2jYaeS/DjsoDUClhJ8pdVd3ZTN4V7XH5+aG0398UenBn+qthHaZG1hFyTD4JDpOOQxpmx9yIbMtoXc", - "RCTYBfqvEAU4UeDOoQxp+3eBxgJK2dorprzzLvqIk+TT+pDZWjxK6BkkWKgMcdX1AExhR2XCYO3rJINj", - "MBmJQ5ad1hnEm7ln7GfKP4SbVHPJ/ERZbr9Zy+npwAeGB/kseiEjwHFH2Y0dlcDoWaINejpYr9tzK2yM", - "jIeWsJ+9iDcJjwepz13clCR5REIagFxxeYIzm0mapWgaTrNOvETw6wWszRmxtTNxqmZaFpkQqy4KOL+k", - "pAtL2oXrMIj8gC/Nff6MJ73xojfjSQY+gIWJdsHmRrxsZP/fnp1o7x0RdGJ78lbd0iziUTphy9iZmZTD", - "zLtQm9gbvXCSMK36zIl9nUqUWKT+0IZyq1QwE+Smee/g9NigXGTcR3w6+Y+2brF01YpngOemdDM+SLbO", - "IEuZweMgJJPqv8shDVu7e2DBun9vb7XOWDWkW8ZlaeSxaWMn+lpIbCMmYQBGPRg5R/iKICR9Atp7XPBT", - "KA7/PUeuoXzHZYLE3E6ZHDppgitUJAvZc+veNCe7DSyUxgjb3dTmPKrtwlLWVIsWTHQCfBa1APF4YbI1", - "37w6R4qI2OXPrwV6dWCX2DKxVMpUy0+K0cHhyYv1fovK8rBUS7OqyoP2YC3YaIWmIIzcR4Nj0kXHR5At", - "a4+V3BaD9IafuUCRORXzw2gfvbVY8qWmkImuNowRLfKQB6O2DDvrrsWkerzto7PMBMTZUErV7MtRFPlh", - "As3aADiTe1FrvVurEy2c+WfPY8i0wCqTxVpNbD6/2vscHSSTPqgqHrIbH0jFKJRG/1dh7b80au+X19G3", - "b6aju9qfyQxLH3fPirea8NKS+uC0clUjmmOGSv2urHldOIYU0edc6XNfteLd3ubmm82dm7vvbgq4WoaI", - "KsDHZZir7cFS7wJ0tO7KuqZq1BhcjvRjG0ru/CLvTtAMS/aDgocV78jm9pNWNRJ1r23DsosB2XxihpRJ", - "KYc3lYUTG+StSxpFRoGRdMpwhJ6htfPjl78ev3q1jnro9euT6lIs+8K3Pi2wV52oeHn6Fm7GsRy5yMbm", - "ZD6cJ8SSayqVrMNxtQoQ/hysV/Npu1o3bpKmjbzkzXLA2F9KiKZefLX1O0B6ddHdNXI+AAjqQ2bRfX0A", - "rEshUz8X99Qa4ncEe9oo832QoRXIgN0m8X97ANM7GU4JccUntooqgEtxvjViaLdDPemdB1JLZhKi49O8", - "0k7urXfNV+b0bKu/ufe0vzkY9DcH7UqrBUv6Pjk4bN/5YMtoHPt4vB+E+2DR3vZSxzK20dFxdIUXEg2d", - "FTXsGCuwYP4Vtq21tFoFoNSBWW+Hw1rVcxqQVkHfc5E90tZobtJ5WuTvVVHF4jQyKeLFus5lJXKGJZKJ", - "gdA0NQ8yFXfIYIBdC+UFJweNCcJBINLcN+cK7BmFOE0s3w+ZIDLhTCvJ2thBv5KFRDGFW8Kse4gNlChL", - "EwmHbE24lKIsdyjBqSSh/gEC9Ls29DvUQ6MKajvoD4ZMzlKlhdh6Hx1yJtOYCOu2RGMKVyrrSKbG5oPx", - "AjUWWmBKGhIxZPo1D0jnx0x/398bDAYDuJMwps62/vfAx013ejvYtyC0JikW4PCYhaMFnD2RMpSykAh0", - "xiVRCiNi2KEaQ3bDm8XPRJ91n7dTr+znuV7l35irwGrbodB+LvTnskLV5+US1a2t191/fFY1a9JWf7Xp", - "OPar0U2u+AkKeBqF2iQa69POeKxIaP10kqi8+jcckG/ZJdN7tDR1G5urOPozJWKB3p2clOICBJnY4sYt", - "Jg5SomEdeHKjZdha4URYOZpbAsLeBwhsVVMpaIhfHPK1eAXnsoQNh7a4issNSW9mBWVmaTSfLJlTxb0d", - "kvkoTX2GiH7ksEHevj0+KjEHxnubTwdPn/Wejjf3ejvhYLOHN7f3elu7eDDZDp5sb25tL8nCaJHKdfvs", - "LK816qsX70KmRy502xdR2xQ4Xzn3bTDwFWUhvyodLd7ozGLvNvJzVff1uO7WQ/Bmg0DpYmipQUqcwAFK", - "At22iYauVBNv8LLtvRlsrvCyrZQXMLgG+ftGpMzc6ZkE98x/HRcGXFys8jhvJ05hQC7rYhW1ip23J9pg", - "f/fZ/u7nEs1lDqwaY5Wd7nFxm8KRHABwJTXBpccVfDQOMLNj9Q3jUraZDJ1uJ0u2gL/hoK0E8maPW2UQ", - "NW3Yrl+MLJPfDZm7xyVbAMIkDABcuK+1AKfrA8x+lh+u1YvDiKchKvi5DB4WXAIdF+wC3QzcyVj3l8G3", - "NJkA2n4AQGWA86dMC2K4/NKN2DTfffQS3oVHODYmkx2EKSpRvPfB4cIEa+j95bo2BszyIZ9b2wW+0YYM", - "0v+CaWsyWHfo8iaM5rOPfuPwTWZJMV71q5rXwYSpv171wa5ZrGgH2wCdWTVuH/2cqW6Z8meVvTVJ7J8j", - "K7BytJT1Us66XfGO5pZ85Qr5192OoWin23GEgjztesb225zra/uvyIq+KCaCI1NRPUtQTRWNLEY0zIRK", - "RQNpMxf04jbpF7beDQlHxjBpCog0WY/WeMk+curLuxO0BjCAf0PWaaz/tZ4FT5bOuq1nO8/2nmw922sF", - "9pMPcLXaeQg5ufXBrdRBgyQdWb9D09QPT98av0JgLPYs8OLdSRFbIRFcix49c9dgsfNn/WdFjKOQp+Oo", - "cKtlAdEMpCosmBfGK5NFDUF4f9JoTicT9ueH4HLrD0Hjzes9uTXebMBuNR35XVrHxZvtmv+XjHumdo0f", - "hgYYSshGpKYzImEG6JwoBPzTQzgA0yFLpbUs5/CcLMW9jLWzvb399MnuViu+sqMrbJwROLg8h7IdQWGL", - "wZto7ez8HG0UGM606QANAF6bWbPSv8+Qrd86KCuk/c3Bto9LGg7unGts2/O4keTvrGlmJ2WJDhnBmdlW", - "2+Veam9vD57s7D7dbbeNret1JK6XSxiXL2PIY1Hgiyu/Btrkm4NTBNmoExyU/SYuPOlGo1I3GhVUMDDI", - "4zcY2NMne7s721ub7SDHfBEPFkyvtGHLssuz6TxM4VkNDynqorfbdFr41CnDYGckiDCNDwIX3185fQzC", - "+EiY1/JFaHMwWG9/7eBq8W0rx1HmDjLZIUY14AKlLKtr0V993flFbi2bpbY5HlZLdV9OCNPUs9g4pq7V", - "LUiZCDKnPJVfoCGuTMLmJOJc3OjbJoPljMg0UuZqkUr07uQHkCma15BUJCnbUJYblyAI3XJyN9rPJRbx", - "M3kTsVqtRpulXzbhbsOu7S5DcyhJg0bcrlBLrpStjjw8xFGQQiUXnK2nnhUAUEE+epJECxNYHkWcMxTM", - "MIMLCFGA20EzHoV9bximfjKaeCMW+BWKuEEcviQksUVOzCD0Z1qFoXOC1gr58MiwUqUY5W5shIwtY1Hm", - "xt3YX1UPS1+mVJaHremJFS+A4ZpPSi7HiE8lGIUKQub7VQz2BAsTCY+ZKdozj40t6Yn29QyxIsx9J6o5", - "SfnEGrhW5YAsZ0NJHAguJSIRnUKBmHcnleTZJQlXWQrt6mjC8mBbsK65LPQcZQaKqXVtL9/56Ekm+ZwT", - "EngYEtaWxOk552SMWQplTwqMTK4TKgx7tIvFm3GpRhko0g0HK9UIahmkguRQbVnKd+YPcu94z0Un2m5D", - "Lhv0equva1zlb6ppgM0y1UtRP7W6GQ/62LgOC7UUiSqHtqriGN0EKS0Hv6cSWqUFzCy0BgkXBbFUwD9b", - "bxOA4jdZdT81a9WWbHy1Mzhviym2HELsFKvZMZtwD/DEDW4lnSfahkomRMQUqnmgkDBKQmdLZteT1tUF", - "2cqRJChMiaWc0U8FtgTHZnsDeARzPjLKphVZX+2wjXvYjGF5qQPo177YJpRI+rM534gUaGViASXCeV5n", - "qwBLKkf+66x6w4JM0wgLZNEA2wxZLuKIsss2rctFPOYRDZD+oHrnPOFRxK9G+pH8Ceay3mp2+oNRU12c", - "czM4m5RmFqTSbz6Fn/Qs1yspseCJ2TDfbwBcSZvILG+Y8s80IhZa7i2j1wVGL2OB72wNmlK1GxotJWnX", - "YQlvKrkty3p3fCo9iW1LtRxX0YeEFqDdqD1JKk1xkRankkMAdVeAt7vRKWctfB4uxqGR1xVUDDQmkHTi", - "plaXGi3EYpupeOsapHKG/uDjskO0bUitp1rWGsvxGQSZeIPLYUWXOqTNGzWaFFb3JgAIIFb1ROGjG+IK", - "rKorlscuNcmTs1qJrRmxJKNujqbcVovyES62Isudt722T6KvFkPzxAIDRIpUCygrCrVcFoUKfxKNuRAA", - "f6w1HM7cbADzQ+s8mtYOdAm9mZEFEiTGlA0ZZZmTFJC8CGJkTkQhRZMLbWRNSdhHfy+YeAAYHSdqYZHI", - "wXn+g0T8imVjHLLiIHXjqdTtHDDjWRQpVHDPX9LNgtWnGQVSZuESTAko3kfVDE0EkbPi3H0FG7WOd8VF", - "2FgJZ4HcK1BgBe5YkeKXhBVFWdaM1zQ0DY3MV/UIOVNtFZ5a+xOVCqCiaoHT5f2lkgi/kphNKXulVehK", - "YasUjBODQAJwHFDYzv5lRHwGwdECcCNv/n9ck/lPp1nj5d8qrxVANRzG7YFx23pdsIFJkakE+5Rv0laG", - "qkAO1jIolfpdAlpz4cmuOkhZEyhU6Wh1TrZLA6sG4rvRbEgSlHvfebr7ZK9lmZTPuqwz0FFf+mpuHi+5", - "kmtYqZM29z5Pd58+e7a9s/ts60Y3LC5no2F9mvI2iuuD1si10ps1+vc///XupHLrswvxzYMbDcpkbfiH", - "1JC5UR7Qu5N///NfblS3HpBP0NThqRvu7RujdKLiSrpAgfIVXrtLsiX2/UHJSYAzMYPWyGRCwA06MnTr", - "5YOpYFC004JxggOqFh5Bjq9MJHn2SgVmuc11UHmwPpXXtG0hObXkkuk4z3hcc52jv5q74QovPG1dbUmm", - "46Z76NfVXs0tdH5rUYxxaBFiILNC33UHdzafKyxLwdL67wByGlz2Vj2TxbyxHPK1mmYAUSy2qFghFNAH", - "FV7RJ+1HxeWvLGfh3rLk1qlS/P2Sfdi8BW/k9fWcyB6nb7A6bbUiH+wBeLuvRuNiHbSlheZKRdPyU/fm", - "/bbIzK2D9Gcn2M37KyQj3uTDKiAt8KMdgyV53na3xBIN3FTIM/E40HhEelmgnk1CQTI1N4J6z1uMc092", - "ZHDJJ5My0OpuMzA3YM5AIpXrBSulLZMuItfOZ1FFdTYAM8POrhx2tAkw7GzGw07l2sqbmhjj65HtoAws", - "MliGlJ1VAK0OUroZjCMeXJoSX1A5uo8GKCaYSZQy2PyVW7XNwfLboW4nKaxNhktNTIhTTWzBmMZkhucU", - "yiHYO5VpKRCTXFMlIWAU2tlHITdQQ6X6pnaG+jWTOLifTxoOHcwWtmHdoH6PMxfRmr8LDr4JVFVlH4jg", - "XZspryX269cnXRPAAKGHZmCl+EY3UTMCLSCzLirY/vnv/vjhcURGMO4qVnxcp2Mx3xtuVgWRREkLHp2z", - "Q4UJUMBTpqog8nE7Q66cslU/klIGwX42/ANAwWzvtjh+SALYkbK+F8uMfgvmruQNWEr7Ege2fSwMmwLu", - "kvx3xWf2Qrg6AONsKJQmNu0U47rNLeFIKm5rWWW7ekSuA0LCKtqk/5W2sfL2S2+s/CtsAWqyqsH2bYh3", - "rs+uf3fJUzDWJmoXY/oZZz2AxnBLamEsDC6dBUopM1oJ97oAFzHyYXv6XmiTzUyul9P6N3KtAJw7TCOD", - "uOZnXSuq7GG0iuK3zhps2tBcrC6yfwc100y8+a2qptlQ9YconGbfupNiabXVOSfKvXtu2ai5ZH6pykjp", - "SssF/LtXyjE2hpW6yB7waDNer7DgzszvFbGIsC3zHxmOySgRZEKvlzCPecEYxmXIkHwjZRkMBtxyLcbX", - "aOcJCmZYyMrYGZ3OVLQoB+DseIB8PquioCCKMOcobLPy+Wq6D+vRbnY5i637lOPzAuxOrZ6GVUlHy0Cb", - "D/PbNuudT/ACvDiNl4RPtncGg+2twa1Qm92wbkCuw/wTW3+v3E5TSl3hO3vRX4pSLbaQJTDXi7peCQp5", - "0BmZpBIEx/uQeJPggKCITAChLSumvfpmsdr18sFbhcqCvWT87xbKrpu7gy/Xa8m6soDXbhodd7lYxvcp", - "Pl9xIdogZoIanpsn5267N9h7s7m9v7u3v7l5F0jLGZGasj2efNi8ehJt4clO9HTx5M/N2ZPpVrzttcMu", - "qSlL04ZXf9XvNkbZ5IdkGSeoJNLQmp1DQkS1Wm+1yrUkEWWkJ7MMqdVpiktkgbl/X7n/b+bnNzNYqjuc", - "lydZVCGwyolT4qyHwbayk1l6d1GdzfHR8lncKgOpOhA/v1WHAuzVbjBQHmGz85moBylreQy9LbzY+iBa", - "mhW36ijy3bDDTveucgPFfexdkpOlDbfsAK8fch7f6ZQLqmbx8tMiey3DsIa46Q9ShWUspT46njIo1V38", - "OQuTKxpR+uNOtxN92CnvGft7e1QtC3+bMaBd6qJW0CKMDCrBL6cCvJIbHsJEsmtbXY/5p83e5jOIQ4g+", - "7Pw06D0rRxx0DbWK5Nt0b5d+HbShYbH+nKtbtPnsRhHXjp7LOOhX6quelp/LFhjX8nheGNkdHS7htrTA", - "+ePaGldQchoV0M/V9OzhNioqTSGJ8MIHjF5w1MqK9VhkMjQmU8pkG7/t9iBz3O7Gw04fHVh0arBl8zL4", - "peahAHqBT2gck5BqHdOY/s0ZDFstfXFVW+JmhTHcVx5tre9X156thkhYlXC16pjsf0Y+7mdZv+0s3mXo", - "HeBXcyYq4HPBi11EJwizSnVMyuY4oqFNpIfESIhX23cgaDnLWhkgcz3Q+Um6aMoVylPoW/rbUtbsF8zG", - "T67B37oEM8MwxNYXAUTJwLnoMvF1fIQSwcM0yPNHIxh0jvgh0gr82RIlf3VI7l36NyAxe8IFWu3faHJo", - "tPNPNq13xTepGbZ5qTcHq5f6Tpwi3U6ahKtlmHmpnQS7EWz4ihREj4umTPaKJliYzPsWEv2sSMG6zWt8", - "yYFWidLEXbBonqpzkue6Ba4YfHG9RyQi+piqN4J4FOZZElTmUnS1SN3cezpruuKEG6n6QH4lJNG2CuAf", - "QX8xZgvvwFzNy+wsWRu4erbSXHj1TK0cS63y4J6s1MQal6rowm3C7zdSvuLzNngplwX3d4afXVTN6ggo", - "TuCXlLSzZvx5+6ULe2v0H9+FW+4hlbTX9uqhAonqAHsz5HHXfx4LrNW6MvPu+K7nfWzxxlrGTTCv1SzQ", - "otf5oPcP42VGo/7+xk9/+3977//q9TZX7GZJRC8kEwg0uiSLnql8o230fhnkFGD3tTI9taxCcAw+JAAQ", - "t5uxON7dQSY0Fr/huDYFiNAqlK3ZXDmhv/2lOb6pQMa3ICdXsuxnV6W4i+qdirvjaC0mYupiyV0i2Xp/", - "yCA37ZIsJCoUw7IqjWPUH2T2SSECHV0YNbBP2PwCjSlUF5RDpq1aHAQk0daEra9CTYlsDtJHEBwV27FF", - "uVzit72QNPEEBL07qSHkvn775vnrt78djV6fvvjt4Hj064v/hRCPq57pIexp3tvZ3bOFsYuU3PQs8Wdg", - "+H8WSK2P3QzUpYe/IMkSqo17FGYqASLBhRsUXkZrkKzgKm26XM31m0FvHmQNeoPdvnBBlcGzL1H08O3S", - "KodzHvW0Rt0AfO91YBpaeIO1oSkTBN9p8nNPxx610XoTp3SKPa5tnwv7ixQndANamaRTW//GKk3+0Pmj", - "alUAIw0MqSoo9hW7VKpec2R9rBWpUV6BvRyvkTKbLkkL4Vzl3MiYqQ1bRNQH0RBywKZeliCb7zIHyNeD", - "j1bnfS5V5QszK4ykeW1OnMZa0amXEOhUk+ZqRgQpLAR8kKOt35BkNhWkBfCHKcKWEJGHSbo8Eq0IwQ2n", - "RGuZs8GRIEtwrXtgl6Ppn+DrrAfw3mNZu/KCeeTlbjZfPgfk8zNX4o9OXBMwjIo94cf5LnPRMpo4rqov", - "RpGr6vM273s3npVVS6Rf096qMGfeR4k1ffz4d0zVz1yABdIMs3HncOFg3YREAOxYFQy8FZI2jUk44qla", - "vv9tBXGLsZGVgczLiDprCwMTB6UM0iZZ4IAg8jHUKa3JQYJUULXQ5rlVhseQeedqdwIhoSP4Oe8Y6iV+", - "+gR+yoknR+ElYUTQAKpR6v0YYwZKOnp3UihKZurT1SBCQb18fXhsLVyHMgsWC1XAei7U7+D0uNPtzIkw", - "Vl5n0N/uD2AzJ4ThhHb2O9v9zf6gA4r8DKa4ARXEbcquTXHNbKXj0GpCz91L+kuBY6Lgi989yecQ6mZf", - "B60XTwt2S4KpsIZLEkFSvGEYqr8G9Hh3oO6bU7lryN7aTQeZrZBwQZLXdnHfg1IJewemuTUYWCxtZY9f", - "SBcxMeobf9iAxbzfVlqdJZEHTL1mWTjdMiP9p25nZ7B5ozEtGwrsXV/Hbxm2eaMEDMLdGxLiVp0eM5MJ", - "ZvN6bQROcccBIxX32u/v9ZrJNI6xWDiCFamVcNmkGBOJsHvXQPwqiQItKqDkSx+9ZsSWzccKYRMsK1IG", - "pWTdh5pDy7vAtO0WOcPFec7DxRcjYakPZxZ/KoszvV0+1fj5y/FOxsb1hbSPHKiz4dp7YKDnOKuD/GA7", - "ZWfw7O47PeRsEtFAoV7GwDYElkqIMokAotrB3XCB/ky5wiiLIH9EW9rqrOOM3br5UbTxkYafzPaOiM/z", - "ekpEjJmJxzfvrNj0te1svOD5dl56qjnGh2oScFI53BdzUIEiV96ixWOrqgzWj6MdT9K/7dNML3xAxt+5", - "hx1uJ5uVwnzILQf1FVEqyWPaTvZWZ5wrIV5d7iVRXwvPD+7zyLK49d/gLnosDPySZBpevlq1Q2EjESkz", - "BrBXAzzLc+Tsdz+Ulb83+ZNCYAa40nXTUEFBmas8HC76yNHUGP1qAag+gsA8w/qxcqqH97XssK372GEw", - "4+xy4vsx9f2YWrbLDbe4KcDGLOzyFj6IG3kgvj3/w429D999D+19D608D4xcWe/CH3zcRzYIEurQyxlP", - "oxCNCTIQOy7cQWHRn35AWAQzOieAowZ1wdJI0QQLCGaIUYgVNte2jY6JpW6JrLkN3VzPhb7lBK5CJ0gy", - "Aui3URPkYR70RhkjIdKfWLS4HMGuVh3a7H2vgz1rMD8a0dWMS5JByDFVOM0ho1Ya6xia7Q/ZG4stqgkI", - "8btO1kgSAULqEv8PZwgPmf3gRydCXOyRxHEuubAAmDpqwBDNstSzqfRIRzLgPniXN4RhpnoyIQGd0MBO", - "65IsbAiht8FWpX70gN04351kOQJoa90PEQaIgH482KPsGbKcVL6/YRB3G0RpmF9yOdQaLMY4iry1IKYR", - "H+NoZOhzSTx3gi/hDUuUYhV5d5vEeEhMRfBkoWacmb/TccpUav4eC34liRh21vtDBrH/ltYk7OYKIrqC", - "2mFxwvU+Ezw2fW6YIW58vCSLT/0hOwhjyhxHwCc4khyRa/gOSioBTIORXg38YHaT/x78MJWKx0WwTcd3", - "Zpg8VUmqbBKDJKrrA5ocMsXRRwcn+GnjY97jJ7gsJjjUfFJ4xUwJdOumUcsR1rMfwaue63YCBBh29EE6", - "7Oi/pwIzZZAiMzxENC0u6VoGyK836XqVwgFmKOGJKWYATDXDmuVKbQA8AI4ipGAruW+14g4r2TAfi/YW", - "jxuh3gw2V2UbUYZOnhc202DnqX8/SRII4oso+e/z178hOJX1GpjX8gghk0XAtMKAwhSuTp1Me4GDGTIX", - "VVC/btih4bCTXeeG6zDWVNqE+V4P7hR/0kP7yXTTpeFP/b5uylxX7qPfP5pW9vVeSmIDPTnsfOqiwoMp", - "VbN0nD177ydoE2LWeUkQoDVzzK2DJMEUwE0KJ745IjELEbenQLRAGOUSqBi4MqYMi8Wy3DUP6S0F+cQE", - "zxWI8XEIwXLDzv7QhcsNO91hh7A5/GZj6oadT34K2FvL5mJpcJ5ll5sZE+0NBuurwZctfT13li0uBr6w", - "DdhoFWWVHvUKWuTPb+t+4D/a/syufjDTnedoOMbwd873R3gBUdDYi5ao5wqionZjFpDIqd2rHT33f3mg", - "FysgUXTfDPpQ7Jldj2Xg8I+KHWGx8m201H3/wBw3uK9DpeS2fxj+fXT+c4/33PrOydyFOvtLYwDsiTWl", - "kXkZYYnOYUy9c218v4Bf+/a/zvYDGL+LiE8v9o3pjiI+RRFlNgS9EKis1QNLS/jIIJ9k31kgFFeXbM1o", - "Ev/+579gUJRN//3Pf1k48X//81+w3TcMoheUNb6YESzUmGB1sY9+JSTp4YjOiZsMFB4lcyIWaHtgff7w", - "CBWqq1stTQ7ZkJ0RlQpWCNU3JcKkbdBeFej5UJYSaZFj9It0YuuXmNhGj9/G7WVDynvd0V0PAh/MoDAB", - "fSo6HgD4MmpqO1tLtON3mZo5l5ym1TDNWrDeavmiyLUy3NszA7yhgAES+/YdPLCTRmvn5y/W+wisLcMV", - "UKMGbIe8GWtG9L/LpNUyyUiUskABKhvZZOCUljv9j+w77bz+tsVvye1vy5bdwO9vnD8AquhW4PsdQIs7", - "AD/d3H2Azyl/5PDC7i5Y0HTxQLGCjvfqNDdPCiR7CGcAWnNADOBQ5QKdHh4jHIaCSLn+n+0q0DM1XJof", - "HYgzKArwELfWdixcWIgqa6qVGeSxiIMzO2qE3byq1SCL59tGqVRE40mXVY3Ij7y7Pz0qnd7kGMkrVua8", - "9v0kWRmnR2XAoShWzi29ACdASKe+ZPu0yEWrHFImAjA7cpaqS1Y8Hx+5DXl/rinbdcqqZ8M9CMWjikB8", - "QEFYztIs1nh9TNz8NltFh426xHP1dbHm4P60oPv2YvnY/DG5scIK2bQUNHgCjQfoS6IMikDnDhfa9uCZ", - "+DkRble7ktww62xa5lNk4BBgQnA1v9z2PTavtDN9TXvfkuUL5LmJxmJJ/l1FaWHs5rRaZuAe24Kld2ff", - "Qg83Mm+/3I23ZTAPkSHsZuw81kKREK1huWDB+vdL7y/O0SYkKjdihZs3CVESYQXRkQDEktlZ96XXHbBi", - "dVyt01m7ljI0ieh0psz9R0gnEOajitVmYZRb9zDKrKqrwIrY6KbHmDJ4qolsL5DmRCj0+vDY0L94pG58", - "hHi31aaSE15LT9e3Z696hAUcAhyz4Dy/TmqffGGDyfB/KQ3w/nfdI0yFo049aFIYP2P9TRwqMqGzfcr/", - "a+vniI4FFov/2voZRwll5L+2DyKsiFTrd8Ysg/s66e7bgHnEzKftF1omGogmNgWAwxUKf/ZWS53fvf9N", - "qf1m0jdS/DO6ftf92+j+RXItVf/tUtypAWD6eKAbrozZfNSGR9/RMO7BaWo5soCGUbpFyvEwZlwqePT4", - "UiNtPCrNOK54bLT0/ucbcunx4Vj3+KgLhIT6p4C/bjOP7ukuwI3j3pVb2+/9XwQcxGM6TXkqi0lNMVbB", - "jEib8BeRsgB+bGp3fjw3Kt5fMZcO7vPouHe9+jvf35HGX11QI7zNhd4qnd+91Vbnt+9rnd+AIdqkSAsS", - "33UFRNYbYjQdHGJbNi6hRtZjR33j8tki6K02VHJzAYEFsT9k/0fbH78rguP3P7nsq3Qw2NqD3wmbv//J", - "JWCxE8cqhEFlc0jEPfjtCG5Rp4AjCSWh8lzP6jhMhVlgPQeC/R9nIOUXye0tJMeF3y2kVhZSgVzLLSS7", - "FndrIpWB9O/dRnL85iO4hSP+Nq2kb/x6pGTByXQyoQElDMoRQE6rrMUDGkvu+83ILXMZmb2PLAQTlTSR", - "1mZkJrVWaOh5BdR7DyQ7zku+3Lf16IqtPs6cDJ7Y6oXWXsu1hWaD7Wvjh8H9nl73b6g9ZhYzFlGddIlW", - "uj1FRUw5nThVEASbwxBBlDESxqzJWuyjwyz7XKZJwoWSpiQPWAimaOdMWwi+8j3lijy+EjxQdoYS2R0y", - "KMqqHxsUjY1LsjAFdyhnWW2dbKa2bo0v169c8OhBt9GXV0L91ZxaKaH3vI1tfb6HU0IfTHTci7p3XCp7", - "upZtDLC4xyTbyTxLJqUfKJuuP6qIZyOssrkVQNM8qtYGThV3xfs3ZtzgJ/kh5E4jHACCnH7NgBvZ7GSD", - "ZlZsClKOBY8iIgxoVZIqV91ryLLBUVaoXmxLaVzo5kcpUzS66JpoGkAekAizhUVuGbJSZ1gpEidasFks", - "IhihIIkZcaWsmR405amEt7pI8lKXCEdXeCGHTJBJRAI7NygBKUhg8N2iqI9+4ZDujfAUU2YzkPWbpijY", - "D3LILmgYkZHN1r5AVCI540IRRkIU8zmR5X4JFhElAiZxiDXlJIrxAmCTDIKcoQ9PiIEmKuWEc/1vzEIK", - "JbN0z9mU94cMo63BAMUEM4kopA1LPCH6K9sGgkGUBvQjwmhn8Mx+VVk3gPZ05F/T+0UIMucBHkcLRDQX", - "w4mo1mEBs/0F1Sf18k2okGa9MveirU1UWlgqXRXNsItSFkCRx1Tof3GBUmaPV92igEx4mKe9hCNUZMXR", - "bNr+mARY05Pxcj8AmMaDIBW+w1EvdaF833+iklmY3jmQyiedNB0Q7KkQ1pxxNYM9zWErrf/YwFU5U30b", - "h4x3k3CBMCrwde5QgErXbIrWAGDsIi8MxlxtyYv1H93e0dvXCgK3/Q3E12M5n4CJ+GRS2oCrjyazgZel", - "V9RZ+Fvdp4euImRRxIUUTxmXigZOGFZrFn83Hlsbj8sp6+XmCReXRd2qzL8/c3HZ1vo6d4X4H5URVpzh", - "V3gPoIcHELEPfx0AzmhjqGimuXcDrcpf2S4FpYsq6eKMOYo4m+pdlDvF791rX7HogigFtdyZcs4Joo2Q", - "kf3RFJXUk7El+8DDH9hWH1oW6d7v4S7oN64QjZOIxASKTvYMs+nFzrRqUxOaSjTLqv3dTFbqXVVMHTa2", - "oDTX/12nDgFfuQVbA+29vlxeoRrx6Wq4sKxzh43lwQsbMlOzmrgC1xcok8FaoTXg3OhqRoMZYIeB3arb", - "N9BiOEkuMtjU9X30EjZyET0WOl8zkNya1ySPiIEEm8fxxX69rOK7kxP4yMCGmQKKF/vIlVLMzg+p3ypi", - "gelZRFgq9JtFOFvLjHFY0QuFtb2ZzW/dooTlsLZD5kMMY+TKNkgn6KIAHnbRgB7m5O0rPn0wZazbDEZu", - "5qI4sqYj8CZhYacpxoJGftywzcHAh5HbEsPMDOOOIcxqg3nFpxkQeomVcZK0ZV87TODieRwv4WG0lksQ", - "JFXIU/U3qUIiBHxsubuJudEaDmwRHHypGZUZqeQ29jqwnzeSyCATe0mlhWqn2yEsjTv7v9t/zeO40+3Y", - "8RQQjW+g3K/Agqs2WI940StTAHz7rpbfBMqtLOwLWG6Vk8Oa080a+Zl54Zu/WXQ+uwdkQ9APKk7cr0kF", - "LYy37PBhHEmGEznj6nGhR1lXU0Vra3bVuFn29PDC1FXqaBPCcW4/PXdffgXW76rIDjdm5KZ77yEe9RE8", - "5kxYWZvNhIsq5NCq2I+vnpG+3JLUptqGQ77z5s39fK0YU2vqdRFhPwhN5SacKh5jRQOoGhLMOJcFth+T", - "GZ5Tbq9K3Z1Vxpng3DB2pg2hv9CsemEdwRdWkd+3TiuEi49sH3343Abe+79wj/Ivfi7Y5ZnE7zrlG5C1", - "oayxoGSCEpxKovWqNCYoWARaKpoyMQQHMxTgRKWCQAUsgmLKaJzGBVeDNpzEHANIxcVmfNFF41ShCIsp", - "2EXmoQmnFyTgcUxYSMBDNmQzgudUG3UCRVgRFix6kkDlzDlBV1xcRhyHYOTbKBxTeUsQzYGUsy6KicIh", - "VhhUjQu940cmi+ciK6ZpDGtGrnNuCIdMpOxHgwaum71wA71ARCo8jqicZUXXAhwSFnihts+/bjH25b3B", - "50RVJ/pAcTm3kqUPGahT9Hq64XwdMTyPLBiZC7uMbcT8EqVXNhuR5fQHx0b/mVvazNXN8YGueDISL9vF", - "X8fdTsZ0X839zsNf4HCBwtR0V9iVwObf6q1MJlCK4U6QWmmW8bZXM1l1qYzMN5J5Gx/dn8e38KZ9JZKw", - "22jYN9UxySf9NYhcS9VbydwHciNaX1LBK/aAItjFVD2Y+sRFQco9FnenFdhma2ZyuyidlMBgfXH2XWxX", - "xbYNObit2Ha+2dqlekGQU9aDKE2/BLdu3EZRbV0H/6G5IJXZFUTmg4vI/O7g3sTicSYIjWhM8CLiOPwW", - "wnSX3OAEXAiD/wCIEo8Jf7TgNSwG6INvrptJiK7LrXx3crLeJCWEWiojhHrEEqKQFKM/i32l/udECBq6", - "EuaHJ0c2YJZKJFLWR69jCnXFLwlJ8pwSAPLo6/k5JIx6MeYS5EW3Q5gSi4RTplaOIn/1bgbz6VYlnO9Z", - "TlpA6+8X0q0vpMGz//jEGUgZyJowE1humSqsGkMBXWgcZaZCu9bL8JinunUtgzSZ9HpO4RSc0IjIhVQk", - "NnGBkzSC7QbFEWztTPudWeUuRMXqnWMS1hIiYiol5UwOmc3WSIjQfevPdfuFECfvhYDCmXw9NULy6wif", - "04MxEWNYNVENMIugcn1nv7OBk2QjxAo3hGjZ4X3GkH6GeDgkF/GYRzRAEWWXEq1F9NKYJ2guUaT/WF8a", - "UDeC7750ZdDb7yxN6WM24d7iaYZnM2b+pvKqrFhzF5OPTqy9JMXN4uQPLLRfrMmVck0QHPUUjUmGXINS", - "RSP6wYg63QiVigYm6SeHLHh3kqMWDNkJUUK/gyG5LIpIoJzDZiMRPNgYpoPBdpBQgD/bJjA4EHjNj2Po", - "8fD0rUkEJTEXi+6Q6X9Aw28OTs3t7gRbb0JhoIyoKy4u0fHG6xUhxudApv/gGD0zwaXYAd4F/34leHNE", - "kMY9JBu2KE+WmUo8+eaDSK0G992v8Dj9CgDJlM1mbSpwAEqxnKUq5FfM70OY8yiN9T/MH8ergL0UDmbv", - "4NWvRts1w1nZjZvgo9iUdk4hMcUdH+TSwxDsscasasK5KYASU4oG9J4CB+pb5O4v774v0vErvO60FHWF", - "U7+avXXfJ58dg8O4KNLjsWxzw2luJoov9z5dYdrsfXoe8eBSWjCUottQ220AMK5/zAGh7RUhqAmQm4ks", - "iBAi1wkVgPxWcUAazB2JMFJExJThaAPmbBoBaGvnxcJzTiFFOogoJKnREFCLIkCnu5oRhvRswFHlGijc", - "6EpbWqr4TvEyUnE0JgGPiYP7XveZbn/HVP3MRRm7+2uRi28K9Nfz0VPV81wBV97c42fBl5/gawiVDlN7", - "oexGtPaS5z8aV1AXwdoMO9sDOex00bCzFQ87egUOMbhQsUK7KKYsVUT20ZHxb0ES7N4ASRJwFkqHOu48", - "eNsD2ZQSa9iyIb9yD767T7XHchWQ8sx24hMP+j2kv4ekHbRW3HB2T4Zd2HQh4qky7n67r+xbIVHgHlm/", - "97vawh75btu3keR/t9u3JKNglbW4LCy9kexJKmek2eX2ylTySdUY0KxddU05Q3/wsewiRq6MN1xI1a/J", - "Pf31qengPpD2dVc3Qdm3c/8Osd8CYj+nlR8u0QRY6iPZcYfBTCTXBhEWu7R3y0NgSQB2Aw9whF4fHg9Z", - "oEWRAfcTJOYgnSwguDmFD/5+jl4cnnXREVR6RL+k4/U+es2ihau3be5ohsxoYkZ4BZihseFaEvqOZzN2", - "4J67DBbXHTxQ6WSzMzw3K26tXJB4tzMjOASN5GPnFTedeTCCz17pDaTXz36ZLXtnqfLROSNKLHoHE0VE", - "vdkTmyfFMtQKe0g7EDiruBnoSd2hdNhneZ9GNzDgFNtbHQ9WxafvVQ/uvkLo/dySmTgRU29unALWJ4Mk", - "AxwuHlcsk5yhTDj6RGDxuM7qBjRlCVtZttTAgC6bIr+/Ipf7UtlVQoL/T91dMNNHe9GUlNZJM3FWb2Tl", - "Ta9LDp4ZQGJ7URXgBAdULboIR5E9o+xJkEWk9DL1dywIvgz5FesP2VlW6cQm9KLD07ddd1GLQiovTQv2", - "LraPXs+JkOk4GxyCjWZujYHmJBwyxVGAoyCNtLpBJhMSQC4uFDCRDXe52VA6d7h38k681VYKUe3poyvy", - "5ucJWL2cLaoct2GWekOQIMI0bob/tooaBBxCqMFYN8oZomwS2ZCqQHApkW2qRyI6pePIBgjJPnozI0ji", - "mAxZEmHGiECpNFHxeui9RBApU5PgrRsAmFzDUV2UQ/slgisbmhBxLqSJJtAc/u4ESUWSJWx2Zlo+gTnf", - "kW5rGrc9PZCTujKGZleIfQXpBTGcYgiu+SiNXADjvYaimwE9tJb4WDb+G0GnUyL0rsBGyJpwPLOtHTnN", - "pi9lLDcWfDzP3mpX8DFrtZCVWMjYWwrNNsrRrsPOzaL+PJ1f0kb0PvvoZlnEv+qPWvZdzlb1D8I++sxZ", - "fit19M8LSYJtHVg5hz82d1Jh5KWtWkq0XQ2r1Tqz9i4zXVvjZz0YbNZjRsvCpfTZJoP362OEwf2iPNx3", - "SbTHzVsltKuSbdqQ8r8az/6r4MC7AbJ/YJSTWwDZf1V594A0/nD4J96N+lB59KW7Z1dt9pvHor+r9HkD", - "SA9wbE3p80bq2eDVpYbSO/tOOzPJtvgtafA23vEG+rsj+3erv4XJUCDWqitozfAkTtTCBbTZu8o86EzS", - "D6TfcBGcxa3e3VXwLUI6vxx7OD5tDOj8NovDP0jMqC3eRyU6PvJUXX9kGIPFPVc6WDb0qdPDIpjROWl2", - "upd3sCVRIkgv4QlcroSGYJYe7ixTWPSnH5Bt3mKu2n9B9UcAyychCqkggYoWphKnlgimjx8kElxbAvCc", - "i0VzlIjZIj8LHh/Y2aw4D+2ess6wPM4wXvRCrHBv7qTNEhfaZ0R3unhKLfAQZejlc7RGrpUwNSbQRFs+", - "iE4ykppy+xJ4cr044M1Bg2eTfiCj6bjNKJdUC3ltq7GgIJWKx27tj4/QGlQfmxKm10Kr+hPQZBPB5zQk", - "YWmMnTmPDFU3Gwh6U7+rViqy0nHOuDCDexAdps2BNP1Ak7JYyEJixpRhGNzKuhzlPWWS+HV/mDIXgGPX", - "yI3i+xFmLb81Z+xoToRKmJaIinMD8bz+/Zh7zMdcMRnKnWml086F5yx3XrfLj2qZtnQXhR+y3Ln7dVu/", - "+3pSeqh8lNk81nU+zwzSJrf518WCg/s7H+7bXf7uEaeAviTO+C64yqEB3aKPYV5BTHdI5iTiSQwVyeHd", - "TreTiqiz35kplexvbEDs94xLtb/z7Ml259P7T/9/AAAA///ULxEss9ABAA==", + "qol5WEFytO+3O4henr49bRpThq2IiqOrzSmLg/Ji40oi5satQaSisV5NPskIJAuIDtHCBKpAj3Z/6JPH", + "EgUHgksJMQIvT9/KrnHckBAlBBJUtA47EYRkgTWTlBkgSevQcbDUE4FjMk4nEyL66DmRqkcmEy7UPhJE", + "c1GaWcHWEWN4kDOTQvRnSipx+7vPfLp3oZtR7HEG/qyfI/NCloV/8rysf2s9v62Vf1riHjDzJzigbLre", + "mj08DsTKNLqF5X7v56czCybYhMOg1yoDHDQQDH30Wwa3CauL8uA6j2exAnXQlNd0OltIGmDDLxYri7Ki", + "QxB2T2uN/jT/0LpOPXq9H6XV7VS0Np8mKciJ87Pe8et3G3FI5t3SmCAgbsYjose9XhBnc5dcnydhlaTY", + "vMkzYxhDtt3hBVplIqY1kQoCxUMdxRWORjLivhijN/ohgodo7d3PJjtWj6CLktJS6t8LVCjx9553xwCu", + "TUO359Bh1cVb2uBeW7dc3sO4gwrTK3Xq2yomJamuk9UhkvlleaH55WpYXtNIc7+HLmuq4oS3ii8yyVUI", + "kqtc2AQynxovu3XlSJJggRWJFkZmZmdzGWaF1G/AyTUJbpC29UK//slAa6WCjNRMEDnjUTluYrvrO4KC", + "FE4gg8pl5lS4KFAcxVhcwsntFH+UMkOBsqjfXgV3MlMqucGkfnnz5tR4A5Q+J6Nq8oCsRQQckQgv0Jio", + "K0KYmwqWCKOXPIP+rOYwygZUQKFGCRGUl2nY2fb0e27iidFU4IAg85Ur7mCXREI8UltS2l48kNtBQKRs", + "WN/NZetrP52kUbs19g1rc2UtlOAmC/zm8NSh22XVAxyZt+pUPiWiZ7acKyOwfGm35HKcRdcV44zUOxN8", + "TAB40WZsFBMqXTwUAEvqz0tJjAXhIItpd7Yf2AWGVF2zz9+30kar29138R9jFvo0TROobzLFp2msF0QP", + "WaRw10VDEyxlkteN3VDMNxAEh5QRKStptkEqok6305vYWe1vbEQ8wNGMS7W/s735dGN52OnSeGMbXjUK", + "6TJ71AVhmTAdl41qqiHApMsssYGTpIXHztBxxfkA4qke3wiVF/TZVtDwXMD7YFDLGL7GgXJYseDCK13R", + "4uK2TTRLluYDDWbVinafPSvuz4H3zjwvFubYf6PG+xBHAzGQmkeMe6RCR8PkH7waFRc+QFAulM2cGxMX", + "JJmdhy4E0V4ClTp7OnhanGWrmkwgbCrb3O47z1TN2+XcYFIgt92/tgEI5yrrHG5LL8c013RZwVNaItZY", + "ajlBeULYjei5u7O9dTN6tp3IsQsjq8glH0LJ4cmR0YkCzhSmjAgUE4VtAbqCkAHfl5YygDiHSQzpNZMf", + "l4uWhniLIuTIbeEyv9RtfQNy7pkJOw5RjBmdaIFs3yz2LGd4a3dv3+B7h2Sys7vX7/dvCsTwIkdeaLUU", + "Gyaor4DJ0Jezz1uHO8BbaDOXj53Tgze/aEGWSmEOrQ05pmy/8O/sn/kD+MP8c0yZF6ehFSQ8ndSg4Mvx", + "a9rgN7/vFyqgOb2nVdkgv/MaopkBHMaLelaK6rw7eLOMx80AbpNjuyTnVKsrr6FYZQNtbo3untc9UQVU", + "92J2WguEd/ph+X28c3fBO7ZPqGSWg9/Xb+JvVb5ALkV4rgGAJoRlmM5RZP4KOJvr7eoDeC4dke5ZC3xQ", + "OEYsEGjWZfHHrPfCj4fFgRR+d/iihZ8s0vP7G4bQLFVI/17XQ1dLIaeOrtjMft9jdiq0RdW3sGje2PEH", + "PgtvE2ZW7v319L///L/y9Mkfm3++evfuf+cv//voN/q/76LT15+FFbIcEe9BYe2+GJKdqRRThLNry0on", + "WAUeH502/xoobJ8Yj4MKZoADjsZkf8h66BVVROBoHw07lWTJYQetEbCU4Cut7uqmbM73uv741NyA6o8/", + "OjX4U7WN0CZ3C7sgGWaHTMchjzFl60M2ZLYt5CYiwS7Qf4UowIkCdw5lSNu/CzQWUPrWXknlnXfRR5wk", + "n9aHzNbuUULPIMFCZQitrgdgCjsqEzZrXycZfIPJYByy7LTOIOHMvWQ/U/4hPKWae+YnynL7zVpOTwc+", + "8DzIf9ELGQHuO8pu+KgERs8Sc9DTwXrdnlthY2Q8tIT97MW9SZA8SH3u4qakyiMS0gDkissrnNnM0yyl", + "03CadeIlgl8vYG3OiK21iVM107LIhGR1UcD5JSVdWNIu3KNBpAh8ae7/ZzzpjRe9GU8ysAIsTHQMNjfo", + "ZSP7//bsRHvviKAT25O3SpdmEY/SCVvGzsykKGbehdrE3uiFk4Rp1WdO7OtUosQi+4f2plClgpmgOM17", + "B6fHBhUj4z7i08l/tHWOpatuPAP8N6Wb8UG4dQZZig0eByGZVP9dDoHY2t0DC9b9e3urdYarId0yLksj", + "j00bO9HXQmIbMQkDMOrByDnCVwQt6RPQSGIEfgrF4b/nyDWU77hMkJjbKZNzJ00whopkIdtu3ZsWZbeB", + "hd4YYbub2pxHtV1YyrJq0YKJZoDPohagHy9MduebV+dIERG7fPu1QK8O7BJbVpZKmWr5STE6ODx5sd5v", + "UYkelmppFlZ50B5sBhvd0BS0kftocEy66PgIsmvtsZLbYpAO8TMXKDKnYn4Y7aO3Fnu+1BQy0diGMaJF", + "HiJh1JZhZ921mFSPt310lpmAOBtKqfp9OeoiP0ygWRswZ3I1aq13a3WlhTP/7HkMmRlYZbJYq4nN51d7", + "n6ODcNIHVcVDduMDqRi10uj/Kqz9l0b5/fI6+vbNdHRXKzSZYenj7lnxVhNeWlJPnFauakRzjFGp35U1", + "sgvHkCL6nCt97qtuvNvb3HyzuXNz991NAVrLkFIFuLkMo7U9uOpdgJTWXVnXVI0ag9GRfmxDz51f5N0J", + "mmHJflDwsOId2dx+0qqmou61bRh3MYCbT8yQMinl8Kmy8GOD1HVJo8goMJJOGY7QM7R2fvzy1+NXr9ZR", + "D71+fVJdimVf+NanBVarExUvT9/CzTiWIxcJ2Zz8h/MEWnJNpZJ1+K5WAcWfgw1rPm1XG8dN0rSRl8hZ", + "DjD7SwkB1YvHtn4HyLAuGrxGzgcATX3IrLuvD7B1KcTq5+KkWkP8jmBSG2W+D2K0AjGw2yT+bw94eifD", + "KSG0+MRWUQVwKdG3RhjtdqgnHfRAaslMQnR8mlfmyb31rvnKnJ5t9Tf3nvY3B4P+5qBdKbZgSd8nB4ft", + "Ox9sGY1jH4/3g3AfLNrbXupYxjY6Oo6u8EKiobOihh1jBRbMv8K2tZZWqwCUOpDr7XBbq3pOAzIr6Hsu", + "skfams5NOk+LfL8qClmcRib2ulgHuqxEzrBEMjGQmybAOVNxhwwG2LXQX3By0JggHAQizX1zriCfUYjT", + "xPL9kAkiE860kqyNHfQrWUgUU7glzLqH2ECJsrSScMjWhEtBynKNEpxKEuofIKC/i6ytoIdGFdSC0B8M", + "mZylSgux9T465EymMRHWbYnGFK5U1pFMjc0H4wVqLLTAlDQkYsj0ax5Qz4+Z/r6/NxgMBnAnYUydbf3v", + "gY+b7vR2sG9Ba00SLcDnMQtfC7h8ImUoZSER6IxLohRGxLBDNYbshjeLn4lW6z5vp17Zz3O9yr8xV4Hb", + "tkOt/Vyo0GWFrc/LJa1bW6+7//is6tekrf5q03fsV6ObXPETFPA0CrVJNNannfFYkdD66SRRebVwOCDf", + "skum92hp6jY2V3GTYIHenZyU4gIEmdhiyC0mDlKiYR14cqNl2FrhRFg5mlsCyN4HaGxVUyloiF8cIrZ4", + "Beeyig2HtriKyw1Jb2YFZWZpNJ8smVPFvR2S+ShNfYaIfuSwRN6+PT4qMQfGe5tPB0+f9Z6ON/d6O+Fg", + "s4c3t/d6W7t4MNkOnmxvbm0vycJokfp1+2wurzXqqy/vQqZHLnS7IXfLGzhfOfdtMPAVZSG/Kh0t3ujM", + "Yu828nNV9/W47tZD8GaDQKljaKlBSpzAAUoC3baJhq5UH2/wsu29GWyu8LKtlBcwuAb5+0akzNzpmYT4", + "zH8dFwZcXKzyOG8nTmFALutiFbWKnbcn2mB/99n+7ucSzWUOrBpjlZ3ucXGbwpEcYHAlNcGlxxV8NA5g", + "s2P1DeNStpkMnW4nS7aAv+GgrQTyZo9bZRA1bdiuX4wsk98Nmb7HJVsAwiQMYFy4r7UAp+sDLH+WT67V", + "i8OIpyEq+LkMfhZcAh0X7ALdDNzJWPeXwcM0mQDafgAAZoD/p0wLYrj80o3YtOB99BLehUc4NiaTHYQp", + "QlG898HhwgRr6P3lujYGzPIhn1vbBb7RhgzS/4JpazJYd+jyJozms49+4/BNZkkxXvWrmtfBhKm/XvXB", + "rllsaQfzAJ1ZNW4f/ZypbpnyZ5W9NUnsnyMrsHJ0lfVSjrtd8Y7mlnzlCvna3Y6haKfbcYSCvO56hvfb", + "nOtr+6/Iir4oJoIjU4E9S1BNFY0spjTMhEpFA2kzF/TiNukXtj4OCUfGMGkKiDRZj9Z4yT5y6su7E7QG", + "sIF/Q9ZprP+1ngVPls66rWc7z/aebD3bawUOlA9wtdp5CDm59cGt1EGDJB1Zv0PT1A9P3xq/QmAs9izw", + "4t1JEYshEVyLHj1z12Cx82f9Z0VMpJCn46hwq2UB1AwEKyyYF/Yrk0UNQXh/0mhOJxP254fgcusPQePN", + "6z25Nd5swHo1HfldWsfFm+2a/5eMe6bWjR+2BhhKyEZkpzMiYQbonCgE/NNDOADTIUultSzn8J8sxb2M", + "tbO9vf30ye5WK76yoytsnBE4uDyHsh1BYYvBm2jt7PwcbRQYzrTpABAAjptZs9K/z5Ct9zooK6T9zcG2", + "j0saDu6ca2zb87iR5O+saWYnZYkOGcGZ2Vbb5V5qb28PnuzsPt1tt42t63UkrpdLGJcvY8hjUeOLK78G", + "2uSbg1ME2agTHJT9Ji486UajUjcaFVQ8MEjlNxjY0yd7uzvbW5vtIMp8EQ8WfK+0Ycuyy7PpPEzhWQ0P", + "Keqit9t0WvjUKcNgZySIMI0PAhffXzl9DCL5SJjX8kVoczBYb3/t4GrxbSvHUeYOMtkhRjXgAqUsq4PR", + "X33d+UVuLZultjkeVkt1X04I09SzWDqmDtYtSJkIMqc8lV+gIa5MwuYk4lzc6Nsmg+WMyDRS5mqRSvTu", + "5AeQKZrXkFQkKdtQlhuXIA7dcnI32s8lFvEzeROxWq1Gm6VfNuFuw67tLkNzKEmDRpyvUEuulK2OPDzE", + "UZBC5RecraeeFQDyQD56kkQLE1geRZwzFMwwgwsIYetMsSnCaMajsO8Nw9RPRhNvxAK/QhE3CMWXhCS2", + "KIoZhP5MqzB0TtBaIR8eGVaqFK/cjY2QsWUvyty4G/ur8GHpy5TK8rA1PbHiBfBc80nJ5RjxqcEbUhAy", + "369itidYmEh4zEyRn3lsbElPtK9niBVh7jtRzUnKJ9bAtSoHZDkbSlogJRLRKRSUeXdSSZ5dknCVpdCu", + "jiYsD7YF65rLQs9RZlCcWtcC852PnmSSzzkhgYchYW1JnJ5zTsaYpVAmpcDI5DqhwrBHu1i8GZdqlIEi", + "3XCwUo2g9kEqSA7tlqV8Z/4g9473XHSi7TbkskGvt/q6xlX+ppoG2CxTvRT1U6ub8aCPjeuwUEuRqHJo", + "qyqO0U2Q1XKwfCqhVVrAzEJrkHBREEsFwPf1NgEofpNV91OzVm2Jx1c7g/O2mGLLIcROsZodswn3AE/c", + "4FbSeaJtqGRCREyh+gcKCaMkdLZkdj1pXV2QrRxJgsKUWMoZ/VRgS3BstjeARzDnI6NsWpH11Q7buIfN", + "GJaXRoB+7YttQomkP5vzjUiBViYWUCKc53W2CrCkcuS/zqo3LMg0jbBAFj2wzZDlIo4ou2zTulzEYx7R", + "AOkPqnfOEx5F/GqkH8mfYC7rrWanPxg11dE5N4OzSWlmQSr95lP4Sc9yvZISC56YDfP9BsCVtInM8oYp", + "/0wjYqHl3jJ6XWD0Mnb4ztagKVW7odFSknYdlvCmktuyrHfHp9KT2LZUy3EVgEhoAd2N2pOk0hQjaXEq", + "OcRQdwV4uxudctbC5+FiHBp5XUHFQGMCSSduanWp0UIstpmKtw5CKmfoDz4uO0TbhtR6qmutsRyfQZCJ", + "N7gcVnSpQ9q8UaNJYXVvAoAAYlVPFD66Ia7AqjpkeexSkzw5q5XkmhFLMurmaMpztSg34WIrstx522v7", + "JPpq8TRPLDBApEi1gDKkUPtlUagIKNGYCwFwyVrD4czNBjA/tM6jae1Al9CbGVlYeNYhoyxzkgKSF0GM", + "zIkopGhyoY2sKQn76O8FEw8ApuNELSxyOTjPf5CIX7FsjENWHKRuPJW6nQNmPIsihYrv+Uu6WbD6NKNA", + "yixcgikBxf6omqGJIHJWnLuvwKPW8a64CBsr5yyQewUKssAdK1L8krCiKMua8ZqGpqGR+aoeIWeqs8JT", + "a3+iUsFUVC2Iury/VBLhVxKzKWWvtApdKWyVgnFiEEgAjgMK4dm/jIjPIDhaAG7kzf+PazL/6TRrvPxb", + "5bUCqIbDuD0wbluvCzYwKTKVYJ/yTdrKUBXIwVoGpVK/S0BrLjzZVRMpawKFqh6tzsl2aWDVQHw3mg1J", + "gnLvO093n+y1LKvyWZd1BjrqS1/NzeMlV3INK3XS5t7n6e7TZ8+2d3afbd3ohsXlbDSsT1PeRnF90Bq5", + "VnqzRv/+57/enVRufXYhvnlwo0GZrA3/kBoyN8oDenfy73/+y43q1gPyCZo6PHXDvX1jlE5UXEkXKFC+", + "wmt3SbbEvj8oOQlwJmbQGplMCLhBR4ZuvXwwFQyKdlowTnBA1cIjyPGViSTPXqnALLe5DioP1qfymrYt", + "JKeWXDId5xmPa65z9FdzN1zhhaetqzPJdNx0D/262qu5hc5vLYoxDi1CDGRWGLzu4M7mc4VlKVha/x1A", + "ToPL3qpnspg3lkO+VtMMIIrFFiErhAL6oMIr+qT9qLj8leUs3FuW3DpVir9fsg+bt+CNvL6eE9nj9A1W", + "p61W5IM9AG/31WhcrJu2tDBdqchafurevN8Wmbl1kP7sBLt5f4VkxJt8WAWkBX60Y7Akz9vulliigZsK", + "eSYeBxqPSC8L1LNJKEim5kZQ73mLce7Jjgwu+WRSBlrdbQbmBswZSKRyvWCltGXSReTa+SyqqM4GYGbY", + "2ZXDjjYBhp3NeNipXFt5UxNjfD2yHZSBRQbLkLKziqHVQUo3g3HEg0tTEgwqTffRAMUEM4lSBpu/cqu2", + "OVh+O9TtJIW1yXCpiQlxqoktGNOYzPCcQjkEe6cyLQVikmuqJASMQjv7KOQGaqhUD9XOUL9mEgf380nD", + "oYPZwjasG9TvceYiWvN3wcE3gSqs7AMRvGsz5bXEfv36pGsCGCD00AysFN/oJmpGoAVk1kUF2z//3R8/", + "PI7ICMZdxYqP63Qs5nvDzaogkihpwaNzdqgwAQp4ylQVRD5uZ8iVU7bqR1LKINjPhn8AKJjt3RbTD0kA", + "O1LW92KZ0W/B3JW8AUtpX+LAto+FYVPAXZL/rvjMXghXB2CcDYVSxqadYly3uSUcScVt7atsV4/IdUBI", + "WEWb9L/SNlbefumNlX+FLUBNVmXYvg3xzvXZ9e8ueQrG2kTtYkw/46wH0BhuSS2MhcGls0ApZUYr4V4X", + "4CJGPmxP3wttspnJ9XJa/0auFYBzh2lkENf8rGtFlT2MVlH81lmDTRuai9VF+e+gxpqJN79VlTUbqv4Q", + "hdbsW3dSXK22OudEuXfPLRs1l9gvVRkpXWm5gH/3SjnGxrBSF9kDHm3G6xUW3Jn5vSIWEbZl/iPDMRkl", + "gkzo9RLmMS8Yw7gMGZJvpCyDwYBbrsX4Gu08QcEMC1kZO6PTmYoW5QCcHQ+Qz2dVIBREEeYchW1WPl9N", + "92E92s0uZ7F1n3J8XoDdqdXTsCrpaBlo82F+22a98wlegBen8ZLwyfbOYLC9NbgVarMb1g3IdZh/Yuv1", + "ldtpSqkrfGcv+ktRqsUWsgTmehHYK0EhDzojk1SC4HgfEm8SHBAUkQkgtGXFt1ffLFa7Xj54q1BZsJeM", + "/91C2XVzd/Dlei1ZVxbw2k2j4y4Xy/g+xecrLkQbxExQw3Pz5Nxt9wZ7bza393f39jc37wJpOSNSU7bH", + "kw+bV0+iLTzZiZ4unvy5OXsy3Yq3vXbYJTVladrw6q/63cYom/yQLOMElUQaWrNzSIioVvetVsWWJKKM", + "9GSWIbU6TXGJLDD37yv3/838/GYGS3WH8/IkiyoEVjlxSpz1MNhWdjJL7y6qszk+Wj6LW2UgVQfi57fq", + "UIC92g0GyiNsdj4T9SBlLY+ht4UXWx9ES7PiVh1Fvht22OneVW6guI+9S3KytOGWHeD1Q87jO51yQdUs", + "Xn5aZK9lGNYQN/1BqrCMpdRHx1MGpb2LP2dhckUjSn/c6XaiDzvlPWN/b4+qZeFvMwa0S13UClqEkUHl", + "+OVUgFdyw0OYSHZtq+sx/7TZ23wGcQjRh52fBr1n5YiDrqFWkXyb7u3Sr4M2NCzWn3N1izaf3Sji2tFz", + "GQf9Sn3V0/Jz2QLjWh7PCym7o8Ml3JYWOH9cW+MKSk6jAvq5mp493EZFpSkkEV74gNELjlpZsR6LTIbG", + "ZEqZbOO33R5kjtvdeNjpowOLTg22bF42v9Q8FEwv8AmNYxJSrWMa0785g2GrpS+uakvcrDCG+8qjrfX9", + "6tqz1RAJqxKuVh2T/c/Ix/0s67edxbsMvQP8as5EBXwueLGL6ARhVqmOSdkcRzS0ifSQGAnxavsOBC1n", + "WSsDZK4HOj9JF025QnkKfUt/W8qa/YLZ+Mk1+FuXYGYYhtj6IoAoGTgXXSa+jo9QIniYBnn+aASDzhE/", + "RFqBP1ui5K8Oyb1L/wYkZk+4QKv9G00OjXb+yab1rvgmNcM2L/XmYPVS34lTpNtJk3C1DDMvtZNgN4IN", + "X5GC6HHRlMle0QQLk3nfQqKfFSlYt3mNLznQKlGauAsWzVN1TvJct8AVgy+u94hERB9T9UYQj8I8S4LK", + "XIquFqmbe09nTVeccCNVH8ivhCTaVgH8I+gvxmzhHZireZmdJWsDV89WmguvnqmVY6lVHtyTlZpY41IV", + "XbhN+P1Gyld83gYv5bLg/s7ws4uqWR0BxQn8kpJ21ow/b790YW+N/uO7cMs9pJL22l49VCBRHWBvhjzu", + "+s9jgbVaV2beHd/1vI8t3ljLuAnmtZoFWvQ6H/T+YbzMaNTf3/jpb/9v7/1fvd7mit0sieiFZAKBRpdk", + "0TOVb7SN3i+DnALsvlamp5ZVCI7BhwQA4nYzFse7O8iExuI3HNemABFahbI1mysn9Le/NMc3Fcj4FuTk", + "Spb97KoUd1G9U3F3HK3FRExdLLlLJFvvDxnkpl2ShUSFYlhWpXGM+oPMPilEoKMLowb2CZtfoDGF6oJy", + "yLRVi4OAJNqasPVVqCmRzUH6CIKjYju2KJdL/LYXkiaegKB3JzWE3Ndv3zx//fa3o9Hr0xe/HRyPfn3x", + "vxDicdUzPYQ9zXs7u3u2MHaRkpueJf4MDP/PAqn1sZuBuvTwFyRZQrVxj8JMJUAkuHCDwstoDZIVXKVN", + "l6u5fjPozYOsQW+w2xcuqDJ49iWKHr5dWuVwzqOe1qgbgO+9DkxDC2+wNjRlguA7TX7u6dijNlpv4pRO", + "sce17XNhf5HihG5AK5N0auvfWKXJHzp/VK0KYKSBIVUFxb5il0rVa46sj7UiNcorsJfjNVJm0yVpIZyr", + "nBsZM7Vhi4j6IBpCDtjUyxJk813mAPl68NHqvM+lqnxhZoWRNK/NidNYKzr1EgKdatJczYgghYWAD3K0", + "9RuSzKaCtAD+MEXYEiLyMEmXR6IVIbjhlGgtczY4EmQJrnUP7HI0/RN8nfUA3nssa1deMI+83M3my+eA", + "fH7mSvzRiWsChlGxJ/w432UuWkYTx1X1xShyVX3e5n3vxrOyaon0a9pbFebM+yixpo8f/46p+pkLsECa", + "YTbuHC4crJuQCIAdq4KBt0LSpjEJRzxVy/e/rSBuMTayMpB5GVFnbWFg4qCUQdokCxwQRD6GOqU1OUiQ", + "CqoW2jy3yvAYMu9c7U4gJHQEP+cdQ73ET5/ATznx5Ci8JIwIGkA1Sr0fY8xASUfvTgpFyUx9uhpEKKiX", + "rw+PrYXrUGbBYqEKWM+F+h2cHne6nTkRxsrrDPrb/QFs5oQwnNDOfme7v9kfdECRn8EUN6CCuE3ZtSmu", + "ma10HFpN6Ll7SX8pcEwUfPG7J/kcQt3s66D14mnBbkkwFdZwSSJIijcMQ/XXgB7vDtR9cyp3Ddlbu+kg", + "sxUSLkjy2i7ue1AqYe/ANLcGA4ulrezxC+kiJkZ94w8bsJj320qrsyTygKnXLAunW2ak/9Tt7Aw2bzSm", + "ZUOBvevr+C3DNm+UgEG4e0NC3KrTY2YywWxer43AKe44YKTiXvv9vV4zmcYxFgtHsCK1Ei6bFGMiEXbv", + "GohfJVGgRQWUfOmj14zYsvlYIWyCZUXKoJSs+1BzaHkXmLbdIme4OM95uPhiJCz14cziT2VxprfLpxo/", + "fzneydi4vpD2kQN1Nlx7Dwz0HGd1kB9sp+wMnt19p4ecTSIaKNTLGNiGwFIJUSYRQFQ7uBsu0J8pVxhl", + "EeSPaEtbnXWcsVs3P4o2PtLwk9neEfF5Xk+JiDEz8fjmnRWbvradjRc8385LTzXH+FBNAk4qh/tiDipQ", + "5MpbtHhsVZXB+nG040n6t32a6YUPyPg797DD7WSzUpgPueWgviJKJXlM28ne6oxzJcSry70k6mvh+cF9", + "HlkWt/4b3EWPhYFfkkzDy1erdihsJCJlxgD2aoBneY6c/e6HsvL3Jn9SCMwAV7puGiooKHOVh8NFHzma", + "GqNfLQDVRxCYZ1g/Vk718L6WHbZ1HzsMZpxdTnw/pr4fU8t2ueEWNwXYmIVd3sIHcSMPxLfnf7ix9+G7", + "76G976GV54GRK+td+IOP+8gGQUIdejnjaRSiMUEGYseFOygs+tMPCItgRucEcNSgLlgaKZpgAcEMMQqx", + "wubattExsdQtkTW3oZvrudC3nMBV6ARJRgD9NmqCPMyD3ihjJET6E4sWlyPY1apDm73vdbBnDeZHI7qa", + "cUkyCDmmCqc5ZNRKYx1Ds/0he2OxRTUBIX7XyRpJIkBIXeL/4QzhIbMf/OhEiIs9kjjOJRcWAFNHDRii", + "WZZ6NpUe6UgG3Afv8oYwzFRPJiSgExrYaV2ShQ0h9DbYqtSPHrAb57uTLEcAba37IcIAEdCPB3uUPUOW", + "k8r3NwziboMoDfNLLodag8UYR5G3FsQ04mMcjQx9LonnTvAlvGGJUqwi726TGA+JqQieLNSMM/N3Ok6Z", + "Ss3fY8GvJBHDznp/yCD239KahN1cQURXUDssTrjeZ4LHps8NM8SNj5dk8ak/ZAdhTJnjCPgER5Ijcg3f", + "QUklgGkw0quBH8xu8t+DH6ZS8bgItun4zgyTpypJlU1ikER1fUCTQ6Y4+ujgBD9tfMx7/ASXxQSHmk8K", + "r5gpgW7dNGo5wnr2I3jVc91OgADDjj5Ihx3991RgpgxSZIaHiKbFJV3LAPn1Jl2vUjjADCU8McUMgKlm", + "WLNcqQ2AB8BRhBRsJfetVtxhJRvmY9He4nEj1JvB5qpsI8rQyfPCZhrsPPXvJ0kCQXwRJf99/vo3BKey", + "XgPzWh4hZLIImFYYUJjC1amTaS9wMEPmogrq1w07NBx2suvccB3GmkqbMN/rwZ3iT3poP5luujT8qd/X", + "TZnryn30+0fTyr7eS0lsoCeHnU9dVHgwpWqWjrNn7/0EbULMOi8JArRmjrl1kCSYArhJ4cQ3RyRmIeL2", + "FIgWCKNcAhUDV8aUYbFYlrvmIb2lIJ+Y4LkCMT4OIVhu2NkfunC5Yac77BA2h99sTN2w88lPAXtr2Vws", + "Dc6z7HIzY6K9wWB9Nfiypa/nzrLFxcAXtgEbraKs0qNeQYv8+W3dD/xH25/Z1Q9muvMcDccY/s75/ggv", + "IAoae9ES9VxBVNRuzAISObV7taPn/i8P9GIFJIrum0Efij2z67EMHP5RsSMsVr6NlrrvH5jjBvd1qJTc", + "9g/Dv4/Of+7xnlvfOZm7UGd/aQyAPbGmNDIvIyzROYypd66N7xfwa9/+19l+AON3EfHpxb4x3VHEpyii", + "zIagFwKVtXpgaQkfGeST7DsLhOLqkq0ZTeLf//wXDIqy6b//+S8LJ/7vf/4LtvuGQfSCssYXM4KFGhOs", + "LvbRr4QkPRzROXGTgcKjZE7EAm0PrM8fHqFCdXWrpckhG7IzolLBCqH6pkSYtA3aqwI9H8pSIi1yjH6R", + "Tmz9EhPb6PHbuL1sSHmvO7rrQeCDGRQmoE9FxwMAX0ZNbWdriXb8LlMz55LTtBqmWQvWWy1fFLlWhnt7", + "ZoA3FDBAYt++gwd20mjt/PzFeh+BtWW4AmrUgO2QN2PNiP53mbRaJhmJUhYoQGUjmwyc0nKn/5F9p53X", + "37b4Lbn9bdmyG/j9jfMHQBXdCny/A2hxB+Cnm7sP8Dnljxxe2N0FC5ouHihW0PFenebmSYFkD+EMQGsO", + "iAEcqlyg08NjhMNQECnX/7NdBXqmhkvzowNxBkUBHuLW2o6FCwtRZU21MoM8FnFwZkeNsJtXtRpk8Xzb", + "KJWKaDzpsqoR+ZF396dHpdObHCN5xcqc176fJCvj9KgMOBTFyrmlF+AECOnUl2yfFrlolUPKRABmR85S", + "dcmK5+MjtyHvzzVlu05Z9Wy4B6F4VBGIDygIy1maxRqvj4mb32ar6LBRl3iuvi7WHNyfFnTfXiwfmz8m", + "N1ZYIZuWggZPoPEAfUmUQRHo3OFC2x48Ez8nwu1qV5IbZp1Ny3yKDBwCTAiu5pfbvsfmlXamr2nvW7J8", + "gTw30Vgsyb+rKC2M3ZxWywzcY1uw9O7sW+jhRubtl7vxtgzmITKE3Yydx1ooEqI1LBcsWP9+6f3FOdqE", + "ROVGrHDzJiFKIqwgOhKAWDI76770ugNWrI6rdTpr11KGJhGdzpS5/wjpBMJ8VLHaLIxy6x5GmVV1FVgR", + "G930GFMGTzWR7QXSnAiFXh8eG/oXj9SNjxDvttpUcsJr6en69uxVj7CAQ4BjFpzn10ntky9sMBn+L6UB", + "3v+ue4SpcNSpB00K42esv4lDRSZ0tk/5f239HNGxwGLxX1s/4yihjPzX9kGEFZFq/c6YZXBfJ919GzCP", + "mPm0/ULLRAPRxKYAcLhC4c/eaqnzu/e/KbXfTPpGin9G1++6fxvdv0iupeq/XYo7NQBMHw90w5Uxm4/a", + "8Og7GsY9OE0tRxbQMEq3SDkexoxLBY8eX2qkjUelGccVj42W3v98Qy49PhzrHh91gZBQ/xTw123m0T3d", + "Bbhx3Ltya/u9/4uAg3hMpylPZTGpKcYqmBFpE/4iUhbAj03tzo/nRsX7K+bSwX0eHfeuV3/n+zvS+KsL", + "aoS3udBbpfO7t9rq/PZ9rfMbMESbFGlB4ruugMh6Q4ymg0Nsy8Yl1Mh67KhvXD5bBL3VhkpuLiCwIPaH", + "7P9o++N3RXD8/ieXfZUOBlt78Dth8/c/uQQsduJYhTCobA6JuAe/HcEt6hRwJKEkVJ7rWR2HqTALrOdA", + "sP/jDKT8Irm9heS48LuF1MpCKpBruYVk1+JuTaQykP6920iO33wEt3DE36aV9I1fj5QsOJlOJjSghEE5", + "AshplbV4QGPJfb8ZuWUuI7P3kYVgopIm0tqMzKTWCg09r4B674Fkx3nJl/u2Hl2x1ceZk8ETW73Q2mu5", + "ttBssH1t/DC439Pr/g21x8xixiKqky7RSrenqIgppxOnCoJgcxgiiDJGwpg1WYt9dJhln8s0SbhQ0pTk", + "AQvBFO2caQvBV76nXJHHV4IHys5QIrtDBkVZ9WODorFxSRam4A7lLKutk83U1q3x5fqVCx496Db68kqo", + "v5pTKyX0nrexrc/3cErog4mOe1H3jktlT9eyjQEW95hkO5lnyaT0A2XT9UcV8WyEVTa3AmiaR9XawKni", + "rnj/xowb/CQ/hNxphANAkNOvGXAjm51s0MyKTUHKseBRRIQBrUpS5ap7DVk2OMoK1YttKY0L3fwoZYpG", + "F10TTQPIAxJhtrDILUNW6gwrReJECzaLRQQjFCQxI66UNdODpjyV8FYXSV7qEuHoCi/kkAkyiUhg5wYl", + "IAUJDL5bFPXRLxzSvRGeYspsBrJ+0xQF+0EO2QUNIzKy2doXiEokZ1wowkiIYj4nstwvwSKiRMAkDrGm", + "nEQxXgBskkGQM/ThCTHQRKWccK7/jVlIoWSW7jmb8v6QYbQ1GKCYYCYRhbRhiSdEf2XbQDCI0oB+RBjt", + "DJ7ZryrrBtCejvxrer8IQeY8wONogYjmYjgR1TosYLa/oPqkXr4JFdKsV+ZetLWJSgtLpauiGXZRygIo", + "8pgK/S8uUMrs8apbFJAJD/O0l3CEiqw4mk3bH5MAa3oyXu4HANN4EKTCdzjqpS6U7/tPVDIL0zsHUvmk", + "k6YDgj0VwpozrmawpzlspfUfG7gqZ6pv45DxbhIuEEYFvs4dClDpmk3RGgCMXeSFwZirLXmx/qPbO3r7", + "WkHgtr+B+Hos5xMwEZ9MShtw9dFkNvCy9Io6C3+r+/TQVYQsiriQ4injUtHACcNqzeLvxmNr43E5Zb3c", + "POHisqhblfn3Zy4u21pf564Q/6Mywooz/ArvAfTwACL24a8DwBltDBXNNPduoFX5K9uloHRRJV2cMUcR", + "Z1O9i3Kn+L177SsWXRCloJY7U845QbQRMrI/mqKSejK2ZB94+APb6kPLIt37PdwF/cYVonESkZhA0cme", + "YTa92JlWbWpCU4lmWbW/m8lKvauKqcPGFpTm+r/r1CHgK7dga6C915fLK1QjPl0NF5Z17rCxPHhhQ2Zq", + "VhNX4PoCZTJYK7QGnBtdzWgwA+wwsFt1+wZaDCfJRQabur6PXsJGLqLHQudrBpJb85rkETGQYPM4vtiv", + "l1V8d3ICHxnYMFNA8WIfuVKK2fkh9VtFLDA9iwhLhX6zCGdrmTEOK3qhsLY3s/mtW5SwHNZ2yHyIYYxc", + "2QbpBF0UwMMuGtDDnLx9xacPpox1m8HIzVwUR9Z0BN4kLOw0xVjQyI8btjkY+DByW2KYmWHcMYRZbTCv", + "+DQDQi+xMk6StuxrhwlcPI/jJTyM1nIJgqQKear+JlVIhICPLXc3MTdaw4EtgoMvNaMyI5Xcxl4H9vNG", + "EhlkYi+ptFDtdDuEpXFn/3f7r3kcd7odO54CovENlPsVWHDVBusRL3plCoBv39Xym0C5lYV9AcutcnJY", + "c7pZIz8zL3zzN4vOZ/eAbAj6QcWJ+zWpoIXxlh0+jCPJcCJnXD0u9Cjraqpobc2uGjfLnh5emLpKHW1C", + "OM7tp+fuy6/A+l0V2eHGjNx07z3Eoz6Cx5wJK2uzmXBRhRxaFfvx1TPSl1uS2lTbcMh33ry5n68VY2pN", + "vS4i7AehqdyEU8VjrGgAVUOCGeeywPZjMsNzyu1VqbuzyjgTnBvGzrQh9BeaVS+sI/jCKvL71mmFcPGR", + "7aMPn9vAe/8X7lH+xc8FuzyT+F2nfAOyNpQ1FpRMUIJTSbRelcYEBYtAS0VTJobgYIYCnKhUEKiARVBM", + "GY3TuOBq0IaTmGMAqbjYjC+6aJwqFGExBbvIPDTh9IIEPI4JCwl4yIZsRvCcaqNOoAgrwoJFTxKonDkn", + "6IqLy4jjEIx8G4VjKm8JojmQctZFMVE4xAqDqnGhd/zIZPFcZMU0jWHNyHXODeGQiZT9aNDAdbMXbqAX", + "iEiFxxGVs6zoWoBDwgIv1Pb51y3Gvrw3+Jyo6kQfKC7nVrL0IQN1il5PN5yvI4bnkQUjc2GXsY2YX6L0", + "ymYjspz+4NjoP3NLm7m6OT7QFU9G4mW7+Ou428mY7qu533n4CxwuUJia7gq7Etj8W72VyQRKMdwJUivN", + "Mt72aiarLpWR+UYyb+Oj+/P4Ft60r0QSdhsN+6Y6JvmkvwaRa6l6K5n7QG5E60sqeMUeUAS7mKoHU5+4", + "KEi5x+LutALbbM1MbhelkxIYrC/Ovovtqti2IQe3FdvON1u7VC8Icsp6EKXpl+DWjdsoqq3r4D80F6Qy", + "u4LIfHARmd8d3JtYPM4EoRGNCV5EHIffQpjukhucgAth8B8AUeIx4Y8WvIbFAH3wzXUzCdF1uZXvTk7W", + "m6SEUEtlhFCPWEIUkmL0Z7Gv1P+cCEFDV8L88OTIBsxSiUTK+uh1TKGu+CUhSZ5TAkAefT0/h4RRL8Zc", + "grzodghTYpFwytTKUeSv3s1gPt2qhPM9y0kLaP39Qrr1hTR49h+fOAMpA1kTZgLLLVOFVWMooAuNo8xU", + "aNd6GR7zVLeuZZAmk17PKZyCExoRuZCKxCYucJJGsN2gOIKtnWm/M6vchahYvXNMwlpCREylpJzJIbPZ", + "GgkRum/9uW6/EOLkvRBQOJOvp0ZIfh3hc3owJmIMqyaqAWYRVK7v7Hc2cJJshFjhhhAtO7zPGNLPEA+H", + "5CIe84gGKKLsUqK1iF4a8wTNJYr0H+tLA+pG8N2Xrgx6+52lKX3MJtxbPM3wbMbM31RelRVr7mLy0Ym1", + "l6S4WZz8gYX2izW5Uq4JgqOeojHJkGtQqmhEPxhRpxuhUtHAJP3kkAXvTnLUgiE7IUrodzAkl0URCZRz", + "2Gwkggcbw3Qw2A4SCvBn2wQGBwKv+XEMPR6evjWJoCTmYtEdMv0PaPjNwam53Z1g600oDJQRdcXFJTre", + "eL0ixPgcyPQfHKNnJrgUO8C74N+vBG+OCNK4h2TDFuXJMlOJJ998EKnV4L77FR6nXwEgmbLZrE0FDkAp", + "lrNUhfyK+X0Icx6lsf6H+eN4FbCXwsHsHbz61Wi7Zjgru3ETfBSb0s4pJKa444NcehiCPdaYVU04NwVQ", + "YkrRgN5T4EB9i9z95d33RTp+hdedlqKucOpXs7fu++SzY3AYF0V6PJZtbjjNzUTx5d6nK0ybvU/PIx5c", + "SguGUnQbarsNAMb1jzkgtL0iBDUBcjORBRFC5DqhApDfKg5Ig7kjEUaKiJgyHG3AnE0jAG3tvFh4zimk", + "SAcRhSQ1GgJqUQTodFczwpCeDTiqXAOFG11pS0sV3yleRiqOxiTgMXFw3+s+0+3vmKqfuShjd38tcvFN", + "gf56Pnqqep4r4Mqbe/ws+PITfA2h0mFqL5TdiNZe8vxH4wrqIlibYWd7IIedLhp2tuJhR6/AIQYXKlZo", + "F8WUpYrIPjoy/i1Igt0bIEkCzkLpUMedB297IJtSYg1bNuRX7sF396n2WK4CUp7ZTnziQb+H9PeQtIPW", + "ihvO7smwC5suRDxVxt1v95V9KyQK3CPr935XW9gj3237NpL873b7lmQUrLIWl4WlN5I9SeWMNLvcXplK", + "PqkaA5q1q64pZ+gPPpZdxMiV8YYLqfo1uae/PjUd3AfSvu7qJij7du7fIfZbQOzntPLDJZoAS30kO+4w", + "mInk2iDCYpf2bnkILAnAbuABjtDrw+MhC7QoMuB+gsQcpJMFBDen8MHfz9GLw7MuOoJKj+iXdLzeR69Z", + "tHD1ts0dzZAZTcwIrwAzNDZcS0Lf8WzGDtxzl8HiuoMHKp1sdobnZsWtlQsS73ZmBIegkXzsvOKmMw9G", + "8NkrvYH0+tkvs2XvLFU+OmdEiUXvYKKIqDd7YvOkWIZaYQ9pBwJnFTcDPak7lA77LO/T6AYGnGJ7q+PB", + "qvj0verB3VcIvZ9bMhMnYurNjVPA+mSQZIDDxeOKZZIzlAlHnwgsHtdZ3YCmLGEry5YaGNBlU+T3V+Ry", + "Xyq7Skjw/6m7C2b6aC+aktI6aSbO6o2svOl1ycEzA0hsL6oCnOCAqkUX4SiyZ5Q9CbKIlF6m/o4FwZch", + "v2L9ITvLKp3YhF50ePq26y5qUUjlpWnB3sX20es5ETIdZ4NDsNHMrTHQnIRDpjgKcBSkkVY3yGRCAsjF", + "hQImsuEuNxtK5w73Tt6Jt9pKIao9fXRF3vw8AauXs0WV4zbMUm8IEkSYxs3w31ZRg4BDCDUY60Y5Q5RN", + "IhtSFQguJbJN9UhEp3Qc2QAh2UdvZgRJHJMhSyLMGBEolSYqXg+9lwgiZWoSvHUDAJNrOKqLcmi/RHBl", + "QxMizoU00QSaw9+dIKlIsoTNzkzLJzDnO9JtTeO2pwdyUlfG0OwKsa8gvSCGUwzBNR+lkQtgvNdQdDOg", + "h9YSH8vGfyPodEqE3hXYCFkTjme2tSOn2fSljOXGgo/n2VvtCj5mrRayEgsZe0uh2UY52nXYuVnUn6fz", + "S9qI3mcf3SyL+Ff9Ucu+y9mq/kHYR585y2+ljv55IUmwrQMr5/DH5k4qjLy0VUuJtqthtVpn1t5lpmtr", + "/KwHg816zGhZuJQ+22Twfn2MMLhflIf7Lon2uHmrhHZVsk0bUv5X49l/FRx4N0D2D4xycgsg+68q7x6Q", + "xh8O/8S7UR8qj7509+yqzX7zWPR3lT5vAOkBjq0pfd5IPRu8utRQemffaWcm2Ra/JQ3exjveQH93ZP9u", + "9bcwGQrEWnUFrRmexIlauIA2e1eZB51J+oH0Gy6Cs7jVu7sKvkVI55djD8enjQGd32Zx+AeJGbXF+6hE", + "x0eequuPDGOwuOdKB8uGPnV6WAQzOifNTvfyDrYkSgTpJTyBy5XQEMzSw51lCov+9AOyzVvMVfsvqP4I", + "YPkkRCEVJFDRwlTi1BLB9PGDRIJrSwCec7FojhIxW+RnweMDO5sV56HdU9YZlscZxoteiBXuzZ20WeJC", + "+4zoThdPqQUeogy9fI7WyLUSpsYEmmjLB9FJRlJTbl8CT64XB7w5aPBs0g9kNB23GeWSaiGvbTUWFKRS", + "8dit/fERWoPqY1PC9FpoVX8Cmmwi+JyGJCyNsTPnkaHqZgNBb+p31UpFVjrOGRdmcA+iw7Q5kKYfaFIW", + "C1lIzJgyDINbWZejvKdMEr/uD1PmAnDsGrlRfD/CrOW35owdzYlQCdMSUXFuIJ7Xvx9zj/mYKyZDuTOt", + "dNq58Jzlzut2+VEt05buovBDljt3v27rd19PSg+VjzKbx7rO55lB2uQ2/7pYcHB/58N9u8vfPeIU0JfE", + "Gd8FVzk0oFv0McwriOkOyZxEPImhIjm82+l2UhF19jszpZL9jQ2I/Z5xqfZ3nj3Z7nx6/+n/DwAA//8n", + "ZpPE49ABAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index d69b1e2a4..2e7eeea4e 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -199,7 +199,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuProfileSlots, err := meter.Int64ObservableGauge( "hypeman_resources_gpu_profile_slots", - metric.WithDescription("Virtual functions able to create each vGPU profile (best-effort snapshot)"), + metric.WithDescription("Estimated concurrently creatable instances per vGPU profile (best-effort snapshot)"), ) if err != nil { return err diff --git a/openapi.yaml b/openapi.yaml index f7d051c54..cab8de2d3 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1585,7 +1585,7 @@ components: example: 1024 available: type: integer - description: "Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer." + description: "Conservative estimate of instances concurrently creatable with this profile across all GPUs, bounded per GPU by free virtual functions and remaining framebuffer. Best-effort: recomputed from driver state on each query." example: 59 PassthroughDevice: From 288078685056aa590b1eca6665e7a5fd872de1fe Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:29:25 +0000 Subject: [PATCH 15/19] Revert "Report conservative per-GPU vGPU profile capacity" This reverts commit d6adc4c41df2aff189c966df5dfdd73619f6b5d9. --- lib/devices/vendor_vfio_linux.go | 51 +-- lib/devices/vendor_vfio_linux_test.go | 28 +- lib/oapi/oapi.go | 479 +++++++++++++------------- lib/resources/monitoring.go | 2 +- openapi.yaml | 2 +- 5 files changed, 252 insertions(+), 310 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index d67ddbc00..ceab85ec5 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -81,37 +81,22 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// listProfiles reports a conservative estimate of how many instances of each -// profile are concurrently creatable. Free VFs on the same GPU share its -// framebuffer, so counting every advertising VF overreports capacity: one 48Q -// assignment can revoke the type from all sibling VFs. Each GPU instead -// contributes min(free VFs advertising the type, remaining framebuffer / -// profile framebuffer), where the largest profile still creatable on the GPU -// is a lower bound on its remaining framebuffer. +// listProfiles counts each free VF advertising a type as one creatable +// instance, matching the driver-reported units that mdev sums through +// available_instances. This is a best-effort snapshot because creating on one +// VF may revoke the type from siblings that share its GPU framebuffer. func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { profilesByType := make(map[string]profileMetadata) - freeVFsByGPU := make(map[string]map[string]int) - remainingFBByGPU := make(map[string]int) + creatableVFs := make(map[string]int) for _, vf := range vfs { creatable, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { return nil, err } - gpu := vf.ParentGPU - if gpu == "" { - gpu = vf.PCIAddress - } for _, profile := range creatable { profilesByType[profile.TypeName] = profile - if vf.Allocated { - continue - } - if freeVFsByGPU[gpu] == nil { - freeVFsByGPU[gpu] = make(map[string]int) - } - freeVFsByGPU[gpu][profile.TypeName]++ - if profile.FramebufferMB > remainingFBByGPU[gpu] { - remainingFBByGPU[gpu] = profile.FramebufferMB + if !vf.Allocated { + creatableVFs[profile.TypeName]++ } } } @@ -124,35 +109,15 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro profiles := make([]GPUProfile, 0, len(metadata)) for _, profile := range metadata { - available := 0 - for gpu, freeVFs := range freeVFsByGPU { - available += gpuProfileCapacity(freeVFs[profile.TypeName], remainingFBByGPU[gpu], profile.FramebufferMB) - } profiles = append(profiles, GPUProfile{ Name: profile.Name, FramebufferMB: profile.FramebufferMB, - Available: available, + Available: creatableVFs[profile.TypeName], }) } return profiles, nil } -// gpuProfileCapacity estimates how many instances of a profile one GPU can -// still create concurrently. When a profile's framebuffer is unknown (0), the -// free VF count is the only signal available. -func gpuProfileCapacity(freeVFs, remainingFB, profileFB int) int { - if freeVFs == 0 { - return 0 - } - if profileFB <= 0 || remainingFB <= 0 { - return freeVFs - } - if byFB := remainingFB / profileFB; byFB < freeVFs { - return byFB - } - return freeVFs -} - func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { vendorVFIOMu.Lock() defer vendorVFIOMu.Unlock() diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index b68a9f912..4555f5c2a 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -184,7 +184,7 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T } } -func TestVendorVFIOListProfilesReportsPerGPUCapacity(t *testing.T) { +func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { t.Parallel() sysfs := newTestVendorVFIOSysfs(t) @@ -196,30 +196,8 @@ func TestVendorVFIOListProfilesReportsPerGPUCapacity(t *testing.T) { require.NoError(t, err) profiles, err := sysfs.listProfiles(vfs) require.NoError(t, err) - assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"), - "one 48Q consumes a whole GPU, so two GPUs mean two creatable instances despite three free VFs") - assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-2Q"), - "small profiles stay capped by the free VF count") -} - -func TestVendorVFIOListProfilesCapsCapacityByRemainingFramebuffer(t *testing.T) { - t.Parallel() - - // One 48G GPU with a 24Q already assigned: siblings only advertise up to - // 24Q, so at most one more 24Q fits despite two free VFs. - remaining := "ID : vGPU Name\n1147 : NVIDIA L40S-1Q\n1153 : NVIDIA L40S-24Q\n" - sysfs := newTestVendorVFIOSysfs(t) - sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1153", remaining) - sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", remaining) - sysfs.addVF(t, "0000:82:00.0", "0000:82:00.6", "44", "0", remaining) - - vfs, err := sysfs.discoverVFs() - require.NoError(t, err) - profiles, err := sysfs.listProfiles(vfs) - require.NoError(t, err) - assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-24Q")) - assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-1Q"), - "framebuffer allows more 1Q instances than free VFs, so the VF count caps it") + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-48Q"), + "each free VF advertising the type counts as one creatable instance") } func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 70d8a32c6..8f02bcf86 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -869,7 +869,7 @@ type GPUConfig struct { // GPUProfile Available vGPU profile type GPUProfile struct { - // Available Conservative estimate of instances concurrently creatable with this profile across all GPUs, bounded per GPU by free virtual functions and remaining framebuffer. Best-effort: recomputed from driver state on each query. + // Available Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer. Available int `json:"available"` // FramebufferMb Frame buffer size in MB @@ -18597,245 +18597,244 @@ var swaggerSpec = []string{ "oz9c6UhpQXdGru6D6JBI1tNs25MMJ3dD8WWRtJmvIX8JTmFBQ9JHsLsg+M2B31R22rniSULCzP/THzKb", "x5L9JM1Fmv7Q0EHNCBWICzql5Y7LftZ7jGW9CWc65ro1dxY/bIpNk9k1oE8GGDyFiQkNNTBfxXRuuyad", "buc8g4a0gqlMmrMMXrNGkTzivDbEl6dvbxqRmgg+oT58YoiQsU+toeZiNV/tDM57m/9josk1+4HGRpmJ", - "qol5WEFytO+3O4henr49bRpThq2IiqOrzSmLg/Ji40oi5satQaSisV5NPskIJAuIDtHCBKpAj3Z/6JPH", - "EgUHgksJMQIvT9/KrnHckBAlBBJUtA47EYRkgTWTlBkgSevQcbDUE4FjMk4nEyL66DmRqkcmEy7UPhJE", - "c1GaWcHWEWN4kDOTQvRnSipx+7vPfLp3oZtR7HEG/qyfI/NCloV/8rysf2s9v62Vf1riHjDzJzigbLre", - "mj08DsTKNLqF5X7v56czCybYhMOg1yoDHDQQDH30Wwa3CauL8uA6j2exAnXQlNd0OltIGmDDLxYri7Ki", - "QxB2T2uN/jT/0LpOPXq9H6XV7VS0Np8mKciJ87Pe8et3G3FI5t3SmCAgbsYjose9XhBnc5dcnydhlaTY", - "vMkzYxhDtt3hBVplIqY1kQoCxUMdxRWORjLivhijN/ohgodo7d3PJjtWj6CLktJS6t8LVCjx9553xwCu", - "TUO359Bh1cVb2uBeW7dc3sO4gwrTK3Xq2yomJamuk9UhkvlleaH55WpYXtNIc7+HLmuq4oS3ii8yyVUI", - "kqtc2AQynxovu3XlSJJggRWJFkZmZmdzGWaF1G/AyTUJbpC29UK//slAa6WCjNRMEDnjUTluYrvrO4KC", - "FE4gg8pl5lS4KFAcxVhcwsntFH+UMkOBsqjfXgV3MlMqucGkfnnz5tR4A5Q+J6Nq8oCsRQQckQgv0Jio", - "K0KYmwqWCKOXPIP+rOYwygZUQKFGCRGUl2nY2fb0e27iidFU4IAg85Ur7mCXREI8UltS2l48kNtBQKRs", - "WN/NZetrP52kUbs19g1rc2UtlOAmC/zm8NSh22XVAxyZt+pUPiWiZ7acKyOwfGm35HKcRdcV44zUOxN8", - "TAB40WZsFBMqXTwUAEvqz0tJjAXhIItpd7Yf2AWGVF2zz9+30kar29138R9jFvo0TROobzLFp2msF0QP", - "WaRw10VDEyxlkteN3VDMNxAEh5QRKStptkEqok6305vYWe1vbEQ8wNGMS7W/s735dGN52OnSeGMbXjUK", - "6TJ71AVhmTAdl41qqiHApMsssYGTpIXHztBxxfkA4qke3wiVF/TZVtDwXMD7YFDLGL7GgXJYseDCK13R", - "4uK2TTRLluYDDWbVinafPSvuz4H3zjwvFubYf6PG+xBHAzGQmkeMe6RCR8PkH7waFRc+QFAulM2cGxMX", - "JJmdhy4E0V4ClTp7OnhanGWrmkwgbCrb3O47z1TN2+XcYFIgt92/tgEI5yrrHG5LL8c013RZwVNaItZY", - "ajlBeULYjei5u7O9dTN6tp3IsQsjq8glH0LJ4cmR0YkCzhSmjAgUE4VtAbqCkAHfl5YygDiHSQzpNZMf", - "l4uWhniLIuTIbeEyv9RtfQNy7pkJOw5RjBmdaIFs3yz2LGd4a3dv3+B7h2Sys7vX7/dvCsTwIkdeaLUU", - "Gyaor4DJ0Jezz1uHO8BbaDOXj53Tgze/aEGWSmEOrQ05pmy/8O/sn/kD+MP8c0yZF6ehFSQ8ndSg4Mvx", - "a9rgN7/vFyqgOb2nVdkgv/MaopkBHMaLelaK6rw7eLOMx80AbpNjuyTnVKsrr6FYZQNtbo3untc9UQVU", - "92J2WguEd/ph+X28c3fBO7ZPqGSWg9/Xb+JvVb5ALkV4rgGAJoRlmM5RZP4KOJvr7eoDeC4dke5ZC3xQ", - "OEYsEGjWZfHHrPfCj4fFgRR+d/iihZ8s0vP7G4bQLFVI/17XQ1dLIaeOrtjMft9jdiq0RdW3sGje2PEH", - "PgtvE2ZW7v319L///L/y9Mkfm3++evfuf+cv//voN/q/76LT15+FFbIcEe9BYe2+GJKdqRRThLNry0on", - "WAUeH502/xoobJ8Yj4MKZoADjsZkf8h66BVVROBoHw07lWTJYQetEbCU4Cut7uqmbM73uv741NyA6o8/", - "OjX4U7WN0CZ3C7sgGWaHTMchjzFl60M2ZLYt5CYiwS7Qf4UowIkCdw5lSNu/CzQWUPrWXknlnXfRR5wk", - "n9aHzNbuUULPIMFCZQitrgdgCjsqEzZrXycZfIPJYByy7LTOIOHMvWQ/U/4hPKWae+YnynL7zVpOTwc+", - "8DzIf9ELGQHuO8pu+KgERs8Sc9DTwXrdnlthY2Q8tIT97MW9SZA8SH3u4qakyiMS0gDkissrnNnM0yyl", - "03CadeIlgl8vYG3OiK21iVM107LIhGR1UcD5JSVdWNIu3KNBpAh8ae7/ZzzpjRe9GU8ysAIsTHQMNjfo", - "ZSP7//bsRHvviKAT25O3SpdmEY/SCVvGzsykKGbehdrE3uiFk4Rp1WdO7OtUosQi+4f2plClgpmgOM17", - "B6fHBhUj4z7i08l/tHWOpatuPAP8N6Wb8UG4dQZZig0eByGZVP9dDoHY2t0DC9b9e3urdYarId0yLksj", - "j00bO9HXQmIbMQkDMOrByDnCVwQt6RPQSGIEfgrF4b/nyDWU77hMkJjbKZNzJ00whopkIdtu3ZsWZbeB", - "hd4YYbub2pxHtV1YyrJq0YKJZoDPohagHy9MduebV+dIERG7fPu1QK8O7BJbVpZKmWr5STE6ODx5sd5v", - "UYkelmppFlZ50B5sBhvd0BS0kftocEy66PgIsmvtsZLbYpAO8TMXKDKnYn4Y7aO3Fnu+1BQy0diGMaJF", - "HiJh1JZhZ921mFSPt310lpmAOBtKqfp9OeoiP0ygWRswZ3I1aq13a3WlhTP/7HkMmRlYZbJYq4nN51d7", - "n6ODcNIHVcVDduMDqRi10uj/Kqz9l0b5/fI6+vbNdHRXKzSZYenj7lnxVhNeWlJPnFauakRzjFGp35U1", - "sgvHkCL6nCt97qtuvNvb3HyzuXNz991NAVrLkFIFuLkMo7U9uOpdgJTWXVnXVI0ag9GRfmxDz51f5N0J", - "mmHJflDwsOId2dx+0qqmou61bRh3MYCbT8yQMinl8Kmy8GOD1HVJo8goMJJOGY7QM7R2fvzy1+NXr9ZR", - "D71+fVJdimVf+NanBVarExUvT9/CzTiWIxcJ2Zz8h/MEWnJNpZJ1+K5WAcWfgw1rPm1XG8dN0rSRl8hZ", - "DjD7SwkB1YvHtn4HyLAuGrxGzgcATX3IrLuvD7B1KcTq5+KkWkP8jmBSG2W+D2K0AjGw2yT+bw94eifD", - "KSG0+MRWUQVwKdG3RhjtdqgnHfRAaslMQnR8mlfmyb31rvnKnJ5t9Tf3nvY3B4P+5qBdKbZgSd8nB4ft", - "Ox9sGY1jH4/3g3AfLNrbXupYxjY6Oo6u8EKiobOihh1jBRbMv8K2tZZWqwCUOpDr7XBbq3pOAzIr6Hsu", - "skfams5NOk+LfL8qClmcRib2ulgHuqxEzrBEMjGQmybAOVNxhwwG2LXQX3By0JggHAQizX1zriCfUYjT", - "xPL9kAkiE860kqyNHfQrWUgUU7glzLqH2ECJsrSScMjWhEtBynKNEpxKEuofIKC/i6ytoIdGFdSC0B8M", - "mZylSgux9T465EymMRHWbYnGFK5U1pFMjc0H4wVqLLTAlDQkYsj0ax5Qz4+Z/r6/NxgMBnAnYUydbf3v", - "gY+b7vR2sG9Ba00SLcDnMQtfC7h8ImUoZSER6IxLohRGxLBDNYbshjeLn4lW6z5vp17Zz3O9yr8xV4Hb", - "tkOt/Vyo0GWFrc/LJa1bW6+7//is6tekrf5q03fsV6ObXPETFPA0CrVJNNannfFYkdD66SRRebVwOCDf", - "skum92hp6jY2V3GTYIHenZyU4gIEmdhiyC0mDlKiYR14cqNl2FrhRFg5mlsCyN4HaGxVUyloiF8cIrZ4", - "Beeyig2HtriKyw1Jb2YFZWZpNJ8smVPFvR2S+ShNfYaIfuSwRN6+PT4qMQfGe5tPB0+f9Z6ON/d6O+Fg", - "s4c3t/d6W7t4MNkOnmxvbm0vycJokfp1+2wurzXqqy/vQqZHLnS7IXfLGzhfOfdtMPAVZSG/Kh0t3ujM", - "Yu828nNV9/W47tZD8GaDQKljaKlBSpzAAUoC3baJhq5UH2/wsu29GWyu8LKtlBcwuAb5+0akzNzpmYT4", - "zH8dFwZcXKzyOG8nTmFALutiFbWKnbcn2mB/99n+7ucSzWUOrBpjlZ3ucXGbwpEcYHAlNcGlxxV8NA5g", - "s2P1DeNStpkMnW4nS7aAv+GgrQTyZo9bZRA1bdiuX4wsk98Nmb7HJVsAwiQMYFy4r7UAp+sDLH+WT67V", - "i8OIpyEq+LkMfhZcAh0X7ALdDNzJWPeXwcM0mQDafgAAZoD/p0wLYrj80o3YtOB99BLehUc4NiaTHYQp", - "QlG898HhwgRr6P3lujYGzPIhn1vbBb7RhgzS/4JpazJYd+jyJozms49+4/BNZkkxXvWrmtfBhKm/XvXB", - "rllsaQfzAJ1ZNW4f/ZypbpnyZ5W9NUnsnyMrsHJ0lfVSjrtd8Y7mlnzlCvna3Y6haKfbcYSCvO56hvfb", - "nOtr+6/Iir4oJoIjU4E9S1BNFY0spjTMhEpFA2kzF/TiNukXtj4OCUfGMGkKiDRZj9Z4yT5y6su7E7QG", - "sIF/Q9ZprP+1ngVPls66rWc7z/aebD3bawUOlA9wtdp5CDm59cGt1EGDJB1Zv0PT1A9P3xq/QmAs9izw", - "4t1JEYshEVyLHj1z12Cx82f9Z0VMpJCn46hwq2UB1AwEKyyYF/Yrk0UNQXh/0mhOJxP254fgcusPQePN", - "6z25Nd5swHo1HfldWsfFm+2a/5eMe6bWjR+2BhhKyEZkpzMiYQbonCgE/NNDOADTIUultSzn8J8sxb2M", - "tbO9vf30ye5WK76yoytsnBE4uDyHsh1BYYvBm2jt7PwcbRQYzrTpABAAjptZs9K/z5Ct9zooK6T9zcG2", - "j0saDu6ca2zb87iR5O+saWYnZYkOGcGZ2Vbb5V5qb28PnuzsPt1tt42t63UkrpdLGJcvY8hjUeOLK78G", - "2uSbg1ME2agTHJT9Ji486UajUjcaFVQ8MEjlNxjY0yd7uzvbW5vtIMp8EQ8WfK+0Ycuyy7PpPEzhWQ0P", - "Keqit9t0WvjUKcNgZySIMI0PAhffXzl9DCL5SJjX8kVoczBYb3/t4GrxbSvHUeYOMtkhRjXgAqUsq4PR", - "X33d+UVuLZultjkeVkt1X04I09SzWDqmDtYtSJkIMqc8lV+gIa5MwuYk4lzc6Nsmg+WMyDRS5mqRSvTu", - "5AeQKZrXkFQkKdtQlhuXIA7dcnI32s8lFvEzeROxWq1Gm6VfNuFuw67tLkNzKEmDRpyvUEuulK2OPDzE", - "UZBC5RecraeeFQDyQD56kkQLE1geRZwzFMwwgwsIYetMsSnCaMajsO8Nw9RPRhNvxAK/QhE3CMWXhCS2", - "KIoZhP5MqzB0TtBaIR8eGVaqFK/cjY2QsWUvyty4G/ur8GHpy5TK8rA1PbHiBfBc80nJ5RjxqcEbUhAy", - "369itidYmEh4zEyRn3lsbElPtK9niBVh7jtRzUnKJ9bAtSoHZDkbSlogJRLRKRSUeXdSSZ5dknCVpdCu", - "jiYsD7YF65rLQs9RZlCcWtcC852PnmSSzzkhgYchYW1JnJ5zTsaYpVAmpcDI5DqhwrBHu1i8GZdqlIEi", - "3XCwUo2g9kEqSA7tlqV8Z/4g9473XHSi7TbkskGvt/q6xlX+ppoG2CxTvRT1U6ub8aCPjeuwUEuRqHJo", - "qyqO0U2Q1XKwfCqhVVrAzEJrkHBREEsFwPf1NgEofpNV91OzVm2Jx1c7g/O2mGLLIcROsZodswn3AE/c", - "4FbSeaJtqGRCREyh+gcKCaMkdLZkdj1pXV2QrRxJgsKUWMoZ/VRgS3BstjeARzDnI6NsWpH11Q7buIfN", - "GJaXRoB+7YttQomkP5vzjUiBViYWUCKc53W2CrCkcuS/zqo3LMg0jbBAFj2wzZDlIo4ou2zTulzEYx7R", - "AOkPqnfOEx5F/GqkH8mfYC7rrWanPxg11dE5N4OzSWlmQSr95lP4Sc9yvZISC56YDfP9BsCVtInM8oYp", - "/0wjYqHl3jJ6XWD0Mnb4ztagKVW7odFSknYdlvCmktuyrHfHp9KT2LZUy3EVgEhoAd2N2pOk0hQjaXEq", - "OcRQdwV4uxudctbC5+FiHBp5XUHFQGMCSSduanWp0UIstpmKtw5CKmfoDz4uO0TbhtR6qmutsRyfQZCJ", - "N7gcVnSpQ9q8UaNJYXVvAoAAYlVPFD66Ia7AqjpkeexSkzw5q5XkmhFLMurmaMpztSg34WIrstx522v7", - "JPpq8TRPLDBApEi1gDKkUPtlUagIKNGYCwFwyVrD4czNBjA/tM6jae1Al9CbGVlYeNYhoyxzkgKSF0GM", - "zIkopGhyoY2sKQn76O8FEw8ApuNELSxyOTjPf5CIX7FsjENWHKRuPJW6nQNmPIsihYrv+Uu6WbD6NKNA", - "yixcgikBxf6omqGJIHJWnLuvwKPW8a64CBsr5yyQewUKssAdK1L8krCiKMua8ZqGpqGR+aoeIWeqs8JT", - "a3+iUsFUVC2Iury/VBLhVxKzKWWvtApdKWyVgnFiEEgAjgMK4dm/jIjPIDhaAG7kzf+PazL/6TRrvPxb", - "5bUCqIbDuD0wbluvCzYwKTKVYJ/yTdrKUBXIwVoGpVK/S0BrLjzZVRMpawKFqh6tzsl2aWDVQHw3mg1J", - "gnLvO093n+y1LKvyWZd1BjrqS1/NzeMlV3INK3XS5t7n6e7TZ8+2d3afbd3ohsXlbDSsT1PeRnF90Bq5", - "VnqzRv/+57/enVRufXYhvnlwo0GZrA3/kBoyN8oDenfy73/+y43q1gPyCZo6PHXDvX1jlE5UXEkXKFC+", - "wmt3SbbEvj8oOQlwJmbQGplMCLhBR4ZuvXwwFQyKdlowTnBA1cIjyPGViSTPXqnALLe5DioP1qfymrYt", - "JKeWXDId5xmPa65z9FdzN1zhhaetqzPJdNx0D/262qu5hc5vLYoxDi1CDGRWGLzu4M7mc4VlKVha/x1A", - "ToPL3qpnspg3lkO+VtMMIIrFFiErhAL6oMIr+qT9qLj8leUs3FuW3DpVir9fsg+bt+CNvL6eE9nj9A1W", - "p61W5IM9AG/31WhcrJu2tDBdqchafurevN8Wmbl1kP7sBLt5f4VkxJt8WAWkBX60Y7Akz9vulliigZsK", - "eSYeBxqPSC8L1LNJKEim5kZQ73mLce7Jjgwu+WRSBlrdbQbmBswZSKRyvWCltGXSReTa+SyqqM4GYGbY", - "2ZXDjjYBhp3NeNipXFt5UxNjfD2yHZSBRQbLkLKziqHVQUo3g3HEg0tTEgwqTffRAMUEM4lSBpu/cqu2", - "OVh+O9TtJIW1yXCpiQlxqoktGNOYzPCcQjkEe6cyLQVikmuqJASMQjv7KOQGaqhUD9XOUL9mEgf380nD", - "oYPZwjasG9TvceYiWvN3wcE3gSqs7AMRvGsz5bXEfv36pGsCGCD00AysFN/oJmpGoAVk1kUF2z//3R8/", - "PI7ICMZdxYqP63Qs5nvDzaogkihpwaNzdqgwAQp4ylQVRD5uZ8iVU7bqR1LKINjPhn8AKJjt3RbTD0kA", - "O1LW92KZ0W/B3JW8AUtpX+LAto+FYVPAXZL/rvjMXghXB2CcDYVSxqadYly3uSUcScVt7atsV4/IdUBI", - "WEWb9L/SNlbefumNlX+FLUBNVmXYvg3xzvXZ9e8ueQrG2kTtYkw/46wH0BhuSS2MhcGls0ApZUYr4V4X", - "4CJGPmxP3wttspnJ9XJa/0auFYBzh2lkENf8rGtFlT2MVlH81lmDTRuai9VF+e+gxpqJN79VlTUbqv4Q", - "hdbsW3dSXK22OudEuXfPLRs1l9gvVRkpXWm5gH/3SjnGxrBSF9kDHm3G6xUW3Jn5vSIWEbZl/iPDMRkl", - "gkzo9RLmMS8Yw7gMGZJvpCyDwYBbrsX4Gu08QcEMC1kZO6PTmYoW5QCcHQ+Qz2dVIBREEeYchW1WPl9N", - "92E92s0uZ7F1n3J8XoDdqdXTsCrpaBlo82F+22a98wlegBen8ZLwyfbOYLC9NbgVarMb1g3IdZh/Yuv1", - "ldtpSqkrfGcv+ktRqsUWsgTmehHYK0EhDzojk1SC4HgfEm8SHBAUkQkgtGXFt1ffLFa7Xj54q1BZsJeM", - "/91C2XVzd/Dlei1ZVxbw2k2j4y4Xy/g+xecrLkQbxExQw3Pz5Nxt9wZ7bza393f39jc37wJpOSNSU7bH", - "kw+bV0+iLTzZiZ4unvy5OXsy3Yq3vXbYJTVladrw6q/63cYom/yQLOMElUQaWrNzSIioVvetVsWWJKKM", - "9GSWIbU6TXGJLDD37yv3/838/GYGS3WH8/IkiyoEVjlxSpz1MNhWdjJL7y6qszk+Wj6LW2UgVQfi57fq", - "UIC92g0GyiNsdj4T9SBlLY+ht4UXWx9ES7PiVh1Fvht22OneVW6guI+9S3KytOGWHeD1Q87jO51yQdUs", - "Xn5aZK9lGNYQN/1BqrCMpdRHx1MGpb2LP2dhckUjSn/c6XaiDzvlPWN/b4+qZeFvMwa0S13UClqEkUHl", - "+OVUgFdyw0OYSHZtq+sx/7TZ23wGcQjRh52fBr1n5YiDrqFWkXyb7u3Sr4M2NCzWn3N1izaf3Sji2tFz", - "GQf9Sn3V0/Jz2QLjWh7PCym7o8Ml3JYWOH9cW+MKSk6jAvq5mp493EZFpSkkEV74gNELjlpZsR6LTIbG", - "ZEqZbOO33R5kjtvdeNjpowOLTg22bF42v9Q8FEwv8AmNYxJSrWMa0785g2GrpS+uakvcrDCG+8qjrfX9", - "6tqz1RAJqxKuVh2T/c/Ix/0s67edxbsMvQP8as5EBXwueLGL6ARhVqmOSdkcRzS0ifSQGAnxavsOBC1n", - "WSsDZK4HOj9JF025QnkKfUt/W8qa/YLZ+Mk1+FuXYGYYhtj6IoAoGTgXXSa+jo9QIniYBnn+aASDzhE/", - "RFqBP1ui5K8Oyb1L/wYkZk+4QKv9G00OjXb+yab1rvgmNcM2L/XmYPVS34lTpNtJk3C1DDMvtZNgN4IN", - "X5GC6HHRlMle0QQLk3nfQqKfFSlYt3mNLznQKlGauAsWzVN1TvJct8AVgy+u94hERB9T9UYQj8I8S4LK", - "XIquFqmbe09nTVeccCNVH8ivhCTaVgH8I+gvxmzhHZireZmdJWsDV89WmguvnqmVY6lVHtyTlZpY41IV", - "XbhN+P1Gyld83gYv5bLg/s7ws4uqWR0BxQn8kpJ21ow/b790YW+N/uO7cMs9pJL22l49VCBRHWBvhjzu", - "+s9jgbVaV2beHd/1vI8t3ljLuAnmtZoFWvQ6H/T+YbzMaNTf3/jpb/9v7/1fvd7mit0sieiFZAKBRpdk", - "0TOVb7SN3i+DnALsvlamp5ZVCI7BhwQA4nYzFse7O8iExuI3HNemABFahbI1mysn9Le/NMc3Fcj4FuTk", - "Spb97KoUd1G9U3F3HK3FRExdLLlLJFvvDxnkpl2ShUSFYlhWpXGM+oPMPilEoKMLowb2CZtfoDGF6oJy", - "yLRVi4OAJNqasPVVqCmRzUH6CIKjYju2KJdL/LYXkiaegKB3JzWE3Ndv3zx//fa3o9Hr0xe/HRyPfn3x", - "vxDicdUzPYQ9zXs7u3u2MHaRkpueJf4MDP/PAqn1sZuBuvTwFyRZQrVxj8JMJUAkuHCDwstoDZIVXKVN", - "l6u5fjPozYOsQW+w2xcuqDJ49iWKHr5dWuVwzqOe1qgbgO+9DkxDC2+wNjRlguA7TX7u6dijNlpv4pRO", - "sce17XNhf5HihG5AK5N0auvfWKXJHzp/VK0KYKSBIVUFxb5il0rVa46sj7UiNcorsJfjNVJm0yVpIZyr", - "nBsZM7Vhi4j6IBpCDtjUyxJk813mAPl68NHqvM+lqnxhZoWRNK/NidNYKzr1EgKdatJczYgghYWAD3K0", - "9RuSzKaCtAD+MEXYEiLyMEmXR6IVIbjhlGgtczY4EmQJrnUP7HI0/RN8nfUA3nssa1deMI+83M3my+eA", - "fH7mSvzRiWsChlGxJ/w432UuWkYTx1X1xShyVX3e5n3vxrOyaon0a9pbFebM+yixpo8f/46p+pkLsECa", - "YTbuHC4crJuQCIAdq4KBt0LSpjEJRzxVy/e/rSBuMTayMpB5GVFnbWFg4qCUQdokCxwQRD6GOqU1OUiQ", - "CqoW2jy3yvAYMu9c7U4gJHQEP+cdQ73ET5/ATznx5Ci8JIwIGkA1Sr0fY8xASUfvTgpFyUx9uhpEKKiX", - "rw+PrYXrUGbBYqEKWM+F+h2cHne6nTkRxsrrDPrb/QFs5oQwnNDOfme7v9kfdECRn8EUN6CCuE3ZtSmu", - "ma10HFpN6Ll7SX8pcEwUfPG7J/kcQt3s66D14mnBbkkwFdZwSSJIijcMQ/XXgB7vDtR9cyp3Ddlbu+kg", - "sxUSLkjy2i7ue1AqYe/ANLcGA4ulrezxC+kiJkZ94w8bsJj320qrsyTygKnXLAunW2ak/9Tt7Aw2bzSm", - "ZUOBvevr+C3DNm+UgEG4e0NC3KrTY2YywWxer43AKe44YKTiXvv9vV4zmcYxFgtHsCK1Ei6bFGMiEXbv", - "GohfJVGgRQWUfOmj14zYsvlYIWyCZUXKoJSs+1BzaHkXmLbdIme4OM95uPhiJCz14cziT2VxprfLpxo/", - "fzneydi4vpD2kQN1Nlx7Dwz0HGd1kB9sp+wMnt19p4ecTSIaKNTLGNiGwFIJUSYRQFQ7uBsu0J8pVxhl", - "EeSPaEtbnXWcsVs3P4o2PtLwk9neEfF5Xk+JiDEz8fjmnRWbvradjRc8385LTzXH+FBNAk4qh/tiDipQ", - "5MpbtHhsVZXB+nG040n6t32a6YUPyPg797DD7WSzUpgPueWgviJKJXlM28ne6oxzJcSry70k6mvh+cF9", - "HlkWt/4b3EWPhYFfkkzDy1erdihsJCJlxgD2aoBneY6c/e6HsvL3Jn9SCMwAV7puGiooKHOVh8NFHzma", - "GqNfLQDVRxCYZ1g/Vk718L6WHbZ1HzsMZpxdTnw/pr4fU8t2ueEWNwXYmIVd3sIHcSMPxLfnf7ix9+G7", - "76G976GV54GRK+td+IOP+8gGQUIdejnjaRSiMUEGYseFOygs+tMPCItgRucEcNSgLlgaKZpgAcEMMQqx", - "wubattExsdQtkTW3oZvrudC3nMBV6ARJRgD9NmqCPMyD3ihjJET6E4sWlyPY1apDm73vdbBnDeZHI7qa", - "cUkyCDmmCqc5ZNRKYx1Ds/0he2OxRTUBIX7XyRpJIkBIXeL/4QzhIbMf/OhEiIs9kjjOJRcWAFNHDRii", - "WZZ6NpUe6UgG3Afv8oYwzFRPJiSgExrYaV2ShQ0h9DbYqtSPHrAb57uTLEcAba37IcIAEdCPB3uUPUOW", - "k8r3NwziboMoDfNLLodag8UYR5G3FsQ04mMcjQx9LonnTvAlvGGJUqwi726TGA+JqQieLNSMM/N3Ok6Z", - "Ss3fY8GvJBHDznp/yCD239KahN1cQURXUDssTrjeZ4LHps8NM8SNj5dk8ak/ZAdhTJnjCPgER5Ijcg3f", - "QUklgGkw0quBH8xu8t+DH6ZS8bgItun4zgyTpypJlU1ikER1fUCTQ6Y4+ujgBD9tfMx7/ASXxQSHmk8K", - "r5gpgW7dNGo5wnr2I3jVc91OgADDjj5Ihx3991RgpgxSZIaHiKbFJV3LAPn1Jl2vUjjADCU8McUMgKlm", - "WLNcqQ2AB8BRhBRsJfetVtxhJRvmY9He4nEj1JvB5qpsI8rQyfPCZhrsPPXvJ0kCQXwRJf99/vo3BKey", - "XgPzWh4hZLIImFYYUJjC1amTaS9wMEPmogrq1w07NBx2suvccB3GmkqbMN/rwZ3iT3poP5luujT8qd/X", - "TZnryn30+0fTyr7eS0lsoCeHnU9dVHgwpWqWjrNn7/0EbULMOi8JArRmjrl1kCSYArhJ4cQ3RyRmIeL2", - "FIgWCKNcAhUDV8aUYbFYlrvmIb2lIJ+Y4LkCMT4OIVhu2NkfunC5Yac77BA2h99sTN2w88lPAXtr2Vws", - "Dc6z7HIzY6K9wWB9Nfiypa/nzrLFxcAXtgEbraKs0qNeQYv8+W3dD/xH25/Z1Q9muvMcDccY/s75/ggv", - "IAoae9ES9VxBVNRuzAISObV7taPn/i8P9GIFJIrum0Efij2z67EMHP5RsSMsVr6NlrrvH5jjBvd1qJTc", - "9g/Dv4/Of+7xnlvfOZm7UGd/aQyAPbGmNDIvIyzROYypd66N7xfwa9/+19l+AON3EfHpxb4x3VHEpyii", - "zIagFwKVtXpgaQkfGeST7DsLhOLqkq0ZTeLf//wXDIqy6b//+S8LJ/7vf/4LtvuGQfSCssYXM4KFGhOs", - "LvbRr4QkPRzROXGTgcKjZE7EAm0PrM8fHqFCdXWrpckhG7IzolLBCqH6pkSYtA3aqwI9H8pSIi1yjH6R", - "Tmz9EhPb6PHbuL1sSHmvO7rrQeCDGRQmoE9FxwMAX0ZNbWdriXb8LlMz55LTtBqmWQvWWy1fFLlWhnt7", - "ZoA3FDBAYt++gwd20mjt/PzFeh+BtWW4AmrUgO2QN2PNiP53mbRaJhmJUhYoQGUjmwyc0nKn/5F9p53X", - "37b4Lbn9bdmyG/j9jfMHQBXdCny/A2hxB+Cnm7sP8Dnljxxe2N0FC5ouHihW0PFenebmSYFkD+EMQGsO", - "iAEcqlyg08NjhMNQECnX/7NdBXqmhkvzowNxBkUBHuLW2o6FCwtRZU21MoM8FnFwZkeNsJtXtRpk8Xzb", - "KJWKaDzpsqoR+ZF396dHpdObHCN5xcqc176fJCvj9KgMOBTFyrmlF+AECOnUl2yfFrlolUPKRABmR85S", - "dcmK5+MjtyHvzzVlu05Z9Wy4B6F4VBGIDygIy1maxRqvj4mb32ar6LBRl3iuvi7WHNyfFnTfXiwfmz8m", - "N1ZYIZuWggZPoPEAfUmUQRHo3OFC2x48Ez8nwu1qV5IbZp1Ny3yKDBwCTAiu5pfbvsfmlXamr2nvW7J8", - "gTw30Vgsyb+rKC2M3ZxWywzcY1uw9O7sW+jhRubtl7vxtgzmITKE3Yydx1ooEqI1LBcsWP9+6f3FOdqE", - "ROVGrHDzJiFKIqwgOhKAWDI76770ugNWrI6rdTpr11KGJhGdzpS5/wjpBMJ8VLHaLIxy6x5GmVV1FVgR", - "G930GFMGTzWR7QXSnAiFXh8eG/oXj9SNjxDvttpUcsJr6en69uxVj7CAQ4BjFpzn10ntky9sMBn+L6UB", - "3v+ue4SpcNSpB00K42esv4lDRSZ0tk/5f239HNGxwGLxX1s/4yihjPzX9kGEFZFq/c6YZXBfJ919GzCP", - "mPm0/ULLRAPRxKYAcLhC4c/eaqnzu/e/KbXfTPpGin9G1++6fxvdv0iupeq/XYo7NQBMHw90w5Uxm4/a", - "8Og7GsY9OE0tRxbQMEq3SDkexoxLBY8eX2qkjUelGccVj42W3v98Qy49PhzrHh91gZBQ/xTw123m0T3d", - "Bbhx3Ltya/u9/4uAg3hMpylPZTGpKcYqmBFpE/4iUhbAj03tzo/nRsX7K+bSwX0eHfeuV3/n+zvS+KsL", - "aoS3udBbpfO7t9rq/PZ9rfMbMESbFGlB4ruugMh6Q4ymg0Nsy8Yl1Mh67KhvXD5bBL3VhkpuLiCwIPaH", - "7P9o++N3RXD8/ieXfZUOBlt78Dth8/c/uQQsduJYhTCobA6JuAe/HcEt6hRwJKEkVJ7rWR2HqTALrOdA", - "sP/jDKT8Irm9heS48LuF1MpCKpBruYVk1+JuTaQykP6920iO33wEt3DE36aV9I1fj5QsOJlOJjSghEE5", - "AshplbV4QGPJfb8ZuWUuI7P3kYVgopIm0tqMzKTWCg09r4B674Fkx3nJl/u2Hl2x1ceZk8ETW73Q2mu5", - "ttBssH1t/DC439Pr/g21x8xixiKqky7RSrenqIgppxOnCoJgcxgiiDJGwpg1WYt9dJhln8s0SbhQ0pTk", - "AQvBFO2caQvBV76nXJHHV4IHys5QIrtDBkVZ9WODorFxSRam4A7lLKutk83U1q3x5fqVCx496Db68kqo", - "v5pTKyX0nrexrc/3cErog4mOe1H3jktlT9eyjQEW95hkO5lnyaT0A2XT9UcV8WyEVTa3AmiaR9XawKni", - "rnj/xowb/CQ/hNxphANAkNOvGXAjm51s0MyKTUHKseBRRIQBrUpS5ap7DVk2OMoK1YttKY0L3fwoZYpG", - "F10TTQPIAxJhtrDILUNW6gwrReJECzaLRQQjFCQxI66UNdODpjyV8FYXSV7qEuHoCi/kkAkyiUhg5wYl", - "IAUJDL5bFPXRLxzSvRGeYspsBrJ+0xQF+0EO2QUNIzKy2doXiEokZ1wowkiIYj4nstwvwSKiRMAkDrGm", - "nEQxXgBskkGQM/ThCTHQRKWccK7/jVlIoWSW7jmb8v6QYbQ1GKCYYCYRhbRhiSdEf2XbQDCI0oB+RBjt", - "DJ7ZryrrBtCejvxrer8IQeY8wONogYjmYjgR1TosYLa/oPqkXr4JFdKsV+ZetLWJSgtLpauiGXZRygIo", - "8pgK/S8uUMrs8apbFJAJD/O0l3CEiqw4mk3bH5MAa3oyXu4HANN4EKTCdzjqpS6U7/tPVDIL0zsHUvmk", - "k6YDgj0VwpozrmawpzlspfUfG7gqZ6pv45DxbhIuEEYFvs4dClDpmk3RGgCMXeSFwZirLXmx/qPbO3r7", - "WkHgtr+B+Hos5xMwEZ9MShtw9dFkNvCy9Io6C3+r+/TQVYQsiriQ4injUtHACcNqzeLvxmNr43E5Zb3c", - "POHisqhblfn3Zy4u21pf564Q/6Mywooz/ArvAfTwACL24a8DwBltDBXNNPduoFX5K9uloHRRJV2cMUcR", - "Z1O9i3Kn+L177SsWXRCloJY7U845QbQRMrI/mqKSejK2ZB94+APb6kPLIt37PdwF/cYVonESkZhA0cme", - "YTa92JlWbWpCU4lmWbW/m8lKvauKqcPGFpTm+r/r1CHgK7dga6C915fLK1QjPl0NF5Z17rCxPHhhQ2Zq", - "VhNX4PoCZTJYK7QGnBtdzWgwA+wwsFt1+wZaDCfJRQabur6PXsJGLqLHQudrBpJb85rkETGQYPM4vtiv", - "l1V8d3ICHxnYMFNA8WIfuVKK2fkh9VtFLDA9iwhLhX6zCGdrmTEOK3qhsLY3s/mtW5SwHNZ2yHyIYYxc", - "2QbpBF0UwMMuGtDDnLx9xacPpox1m8HIzVwUR9Z0BN4kLOw0xVjQyI8btjkY+DByW2KYmWHcMYRZbTCv", - "+DQDQi+xMk6StuxrhwlcPI/jJTyM1nIJgqQKear+JlVIhICPLXc3MTdaw4EtgoMvNaMyI5Xcxl4H9vNG", - "EhlkYi+ptFDtdDuEpXFn/3f7r3kcd7odO54CovENlPsVWHDVBusRL3plCoBv39Xym0C5lYV9AcutcnJY", - "c7pZIz8zL3zzN4vOZ/eAbAj6QcWJ+zWpoIXxlh0+jCPJcCJnXD0u9Cjraqpobc2uGjfLnh5emLpKHW1C", - "OM7tp+fuy6/A+l0V2eHGjNx07z3Eoz6Cx5wJK2uzmXBRhRxaFfvx1TPSl1uS2lTbcMh33ry5n68VY2pN", - "vS4i7AehqdyEU8VjrGgAVUOCGeeywPZjMsNzyu1VqbuzyjgTnBvGzrQh9BeaVS+sI/jCKvL71mmFcPGR", - "7aMPn9vAe/8X7lH+xc8FuzyT+F2nfAOyNpQ1FpRMUIJTSbRelcYEBYtAS0VTJobgYIYCnKhUEKiARVBM", - "GY3TuOBq0IaTmGMAqbjYjC+6aJwqFGExBbvIPDTh9IIEPI4JCwl4yIZsRvCcaqNOoAgrwoJFTxKonDkn", - "6IqLy4jjEIx8G4VjKm8JojmQctZFMVE4xAqDqnGhd/zIZPFcZMU0jWHNyHXODeGQiZT9aNDAdbMXbqAX", - "iEiFxxGVs6zoWoBDwgIv1Pb51y3Gvrw3+Jyo6kQfKC7nVrL0IQN1il5PN5yvI4bnkQUjc2GXsY2YX6L0", - "ymYjspz+4NjoP3NLm7m6OT7QFU9G4mW7+Ou428mY7qu533n4CxwuUJia7gq7Etj8W72VyQRKMdwJUivN", - "Mt72aiarLpWR+UYyb+Oj+/P4Ft60r0QSdhsN+6Y6JvmkvwaRa6l6K5n7QG5E60sqeMUeUAS7mKoHU5+4", - "KEi5x+LutALbbM1MbhelkxIYrC/Ovovtqti2IQe3FdvON1u7VC8Icsp6EKXpl+DWjdsoqq3r4D80F6Qy", - "u4LIfHARmd8d3JtYPM4EoRGNCV5EHIffQpjukhucgAth8B8AUeIx4Y8WvIbFAH3wzXUzCdF1uZXvTk7W", - "m6SEUEtlhFCPWEIUkmL0Z7Gv1P+cCEFDV8L88OTIBsxSiUTK+uh1TKGu+CUhSZ5TAkAefT0/h4RRL8Zc", - "grzodghTYpFwytTKUeSv3s1gPt2qhPM9y0kLaP39Qrr1hTR49h+fOAMpA1kTZgLLLVOFVWMooAuNo8xU", - "aNd6GR7zVLeuZZAmk17PKZyCExoRuZCKxCYucJJGsN2gOIKtnWm/M6vchahYvXNMwlpCREylpJzJIbPZ", - "GgkRum/9uW6/EOLkvRBQOJOvp0ZIfh3hc3owJmIMqyaqAWYRVK7v7Hc2cJJshFjhhhAtO7zPGNLPEA+H", - "5CIe84gGKKLsUqK1iF4a8wTNJYr0H+tLA+pG8N2Xrgx6+52lKX3MJtxbPM3wbMbM31RelRVr7mLy0Ym1", - "l6S4WZz8gYX2izW5Uq4JgqOeojHJkGtQqmhEPxhRpxuhUtHAJP3kkAXvTnLUgiE7IUrodzAkl0URCZRz", - "2Gwkggcbw3Qw2A4SCvBn2wQGBwKv+XEMPR6evjWJoCTmYtEdMv0PaPjNwam53Z1g600oDJQRdcXFJTre", - "eL0ixPgcyPQfHKNnJrgUO8C74N+vBG+OCNK4h2TDFuXJMlOJJ998EKnV4L77FR6nXwEgmbLZrE0FDkAp", - "lrNUhfyK+X0Icx6lsf6H+eN4FbCXwsHsHbz61Wi7Zjgru3ETfBSb0s4pJKa444NcehiCPdaYVU04NwVQ", - "YkrRgN5T4EB9i9z95d33RTp+hdedlqKucOpXs7fu++SzY3AYF0V6PJZtbjjNzUTx5d6nK0ybvU/PIx5c", - "SguGUnQbarsNAMb1jzkgtL0iBDUBcjORBRFC5DqhApDfKg5Ig7kjEUaKiJgyHG3AnE0jAG3tvFh4zimk", - "SAcRhSQ1GgJqUQTodFczwpCeDTiqXAOFG11pS0sV3yleRiqOxiTgMXFw3+s+0+3vmKqfuShjd38tcvFN", - "gf56Pnqqep4r4Mqbe/ws+PITfA2h0mFqL5TdiNZe8vxH4wrqIlibYWd7IIedLhp2tuJhR6/AIQYXKlZo", - "F8WUpYrIPjoy/i1Igt0bIEkCzkLpUMedB297IJtSYg1bNuRX7sF396n2WK4CUp7ZTnziQb+H9PeQtIPW", - "ihvO7smwC5suRDxVxt1v95V9KyQK3CPr935XW9gj3237NpL873b7lmQUrLIWl4WlN5I9SeWMNLvcXplK", - "PqkaA5q1q64pZ+gPPpZdxMiV8YYLqfo1uae/PjUd3AfSvu7qJij7du7fIfZbQOzntPLDJZoAS30kO+4w", - "mInk2iDCYpf2bnkILAnAbuABjtDrw+MhC7QoMuB+gsQcpJMFBDen8MHfz9GLw7MuOoJKj+iXdLzeR69Z", - "tHD1ts0dzZAZTcwIrwAzNDZcS0Lf8WzGDtxzl8HiuoMHKp1sdobnZsWtlQsS73ZmBIegkXzsvOKmMw9G", - "8NkrvYH0+tkvs2XvLFU+OmdEiUXvYKKIqDd7YvOkWIZaYQ9pBwJnFTcDPak7lA77LO/T6AYGnGJ7q+PB", - "qvj0verB3VcIvZ9bMhMnYurNjVPA+mSQZIDDxeOKZZIzlAlHnwgsHtdZ3YCmLGEry5YaGNBlU+T3V+Ry", - "Xyq7Skjw/6m7C2b6aC+aktI6aSbO6o2svOl1ycEzA0hsL6oCnOCAqkUX4SiyZ5Q9CbKIlF6m/o4FwZch", - "v2L9ITvLKp3YhF50ePq26y5qUUjlpWnB3sX20es5ETIdZ4NDsNHMrTHQnIRDpjgKcBSkkVY3yGRCAsjF", - "hQImsuEuNxtK5w73Tt6Jt9pKIao9fXRF3vw8AauXs0WV4zbMUm8IEkSYxs3w31ZRg4BDCDUY60Y5Q5RN", - "IhtSFQguJbJN9UhEp3Qc2QAh2UdvZgRJHJMhSyLMGBEolSYqXg+9lwgiZWoSvHUDAJNrOKqLcmi/RHBl", - "QxMizoU00QSaw9+dIKlIsoTNzkzLJzDnO9JtTeO2pwdyUlfG0OwKsa8gvSCGUwzBNR+lkQtgvNdQdDOg", - "h9YSH8vGfyPodEqE3hXYCFkTjme2tSOn2fSljOXGgo/n2VvtCj5mrRayEgsZe0uh2UY52nXYuVnUn6fz", - "S9qI3mcf3SyL+Ff9Ucu+y9mq/kHYR585y2+ljv55IUmwrQMr5/DH5k4qjLy0VUuJtqthtVpn1t5lpmtr", - "/KwHg816zGhZuJQ+22Twfn2MMLhflIf7Lon2uHmrhHZVsk0bUv5X49l/FRx4N0D2D4xycgsg+68q7x6Q", - "xh8O/8S7UR8qj7509+yqzX7zWPR3lT5vAOkBjq0pfd5IPRu8utRQemffaWcm2Ra/JQ3exjveQH93ZP9u", - "9bcwGQrEWnUFrRmexIlauIA2e1eZB51J+oH0Gy6Cs7jVu7sKvkVI55djD8enjQGd32Zx+AeJGbXF+6hE", - "x0eequuPDGOwuOdKB8uGPnV6WAQzOifNTvfyDrYkSgTpJTyBy5XQEMzSw51lCov+9AOyzVvMVfsvqP4I", - "YPkkRCEVJFDRwlTi1BLB9PGDRIJrSwCec7FojhIxW+RnweMDO5sV56HdU9YZlscZxoteiBXuzZ20WeJC", - "+4zoThdPqQUeogy9fI7WyLUSpsYEmmjLB9FJRlJTbl8CT64XB7w5aPBs0g9kNB23GeWSaiGvbTUWFKRS", - "8dit/fERWoPqY1PC9FpoVX8Cmmwi+JyGJCyNsTPnkaHqZgNBb+p31UpFVjrOGRdmcA+iw7Q5kKYfaFIW", - "C1lIzJgyDINbWZejvKdMEr/uD1PmAnDsGrlRfD/CrOW35owdzYlQCdMSUXFuIJ7Xvx9zj/mYKyZDuTOt", - "dNq58Jzlzut2+VEt05buovBDljt3v27rd19PSg+VjzKbx7rO55lB2uQ2/7pYcHB/58N9u8vfPeIU0JfE", - "Gd8FVzk0oFv0McwriOkOyZxEPImhIjm82+l2UhF19jszpZL9jQ2I/Z5xqfZ3nj3Z7nx6/+n/DwAA//8n", - "ZpPE49ABAA==", + "qol5WEFytO+3O4henr49bRpThq2IiqOrzSmLg1oGhu2iXSYpM+iOBZ8KtqjLxrwxt3K2sz56TqTqkcmE", + "C7VvA1ngQqqA7I4XSJAw1fxghkIjquAOQ9JxBHs061Ur/PoXPaGJwDEZp5MJEeWk9mc+Xbrw9ij2OPd+", + "1s+ReSHLqj95Xtantd7e1mo/LXEDmO0THFA2XW+93B6HYGUa3cLyvffzx5kFB2zCVdCkzAAEDaRCH/2W", + "wWdqUkuUB8t5PIUV6IKmPKXT2ULSAEemRYN9RVnRwQe7obWGfpp/aF2hHj3dj7rqdh5am0+TFPb9+Vnv", + "+PW7jTgk825pTBDgNuMR0eNeL4inuUuWz5OqSlJp3uRpMYwh2+7YAq0ykdGaSAUB4aGO4gpHIxlxX8zQ", + "G/0QwUO09u5nk+2qR9BFSWkp9e8FKpT4e8+7YwCnpqHbc+iw6rItbXCv7Vou12HcO4XplTr1bRWTYlTX", + "seqQx/yyvND8cjXMrmmkud9DlwVVcapbRRaZZCkEyVIuDAKZT43X3LpmJEmwwIpEC6PZZGK2DJtC6jfa", + "5JoEN0jDeqFf/2SgslJBRmomiJzxqBwHsd2tw61LiMWdE4swaeZUcPwrjmIsLuEkdoo8SpmhQDmUe3sV", + "fMlMqeQGk/rlzZtTY90rIuY4qiYDyNoN/xGJ8AKNiboihLmpYIkweskzKM9qTqJsQPkTapQQQXmZhp1t", + "T7/nJj4YTQUOCDJfuWINdkkkxBe1JaXtxQOhHQREyob13Vy2vvbTSRq1W2PfsDZX1jYJbrLAbw5PHVpd", + "Vg3AkXmrTuVTInpmy7myAMuXdksux010XTHOSL0zwccEgBRtBkYxQdLFNwFQpP68lJRYEA6ymEZn+4Fd", + "YEjVNfv8fSvtsrrdfRf5MWahr6qCCbw3md/TNNYLoocsUlD9aGiCn0wyurEDivkDguCQMiJlJW02SEXU", + "6XZ6Ezur/Y2NiAc4mnGp9ne2N59uLA8jXRo/bMOlRiFdZl+6oCoTduOyS011A5h0mSU2cJK08MAZOq44", + "H0A81eMVoZKCPtsKGp4LYB8MahnA1zhQDvsVXHKlK1dc3LaJZsnSfKDBrPrQ7rNnxf058N6B58W/HPtv", + "1Hgf4mIgplHziHF3VOhomPyDV6PiwgfwyYWymXBj4oIes/PQhRTaS51SZ08HT4uzbFVjCYRNZZvbfeeZ", + "qnm7nOtLCuS2+9c2AOFZZZ3DbenlGOWaLit4SkvEGkstJyhPCLsRPXd3trduRs+2Ezl2YWEVueRDHDk8", + "OTI6UcCZwpQRgWKisC0oVxAy4MvSUgYQ5DCJIV1m8uNy0dIQP1GEELkt/OWXun1vQMI9M2HEIYoxoxMt", + "kO2bxZ7lDG/t7u0bvO6QTHZ29/r9/k2BFV7kSAqtlmLDBOkVMBb6cvZ563AH+Alt5vKxc3rw5hctyFIp", + "zKG1IceU7Rf+nf0zfwB/mH+OKfPiLrSCeKeTGrR7OR5NG/zm9/1CRTOn97QqA+R3RkN0MoC9eFHMSlGa", + "dwdXlvG4GcBtcmaX5JBqdeU1FJ9soM2t0drzOiaqgNJezDZrgdhOPyy/X3fuLnjH9gmVyXIw+/rN+q3K", + "EciliM01QM+EsAyjOYrMXwFnc71dfYDNpSPSPWuB9wnHiAX2zLos/pj1XvjxsDiQwu8OL7Twk0Vufn/D", + "kJilCunf63roaink1NEVm9nve8xOhbYo+RbmzBsL/sBn4W3Cxsq9v57+95//V54++WPzz1fv3v3v/OV/", + "H/1G//dddPr6s7A/liPcPShM3RdDpjOVX4rwdG1Z6QSrwOOj0+ZfA4XtE+NxUMEMcL3RmOwPWQ+9oooI", + "HO2jYaeS/DjsoDUClhJ8pdVd3ZTN4V7XH5+aG0398UenBn+qthHaZG1hFyTD4JDpOOQxpmx9yIbMtoXc", + "RCTYBfqvEAU4UeDOoQxp+3eBxgJK2dorprzzLvqIk+TT+pDZWjxK6BkkWKgMcdX1AExhR2XCYO3rJINj", + "MBmJQ5ad1hnEm7ln7GfKP4SbVHPJ/ERZbr9Zy+npwAeGB/kseiEjwHFH2Y0dlcDoWaINejpYr9tzK2yM", + "jIeWsJ+9iDcJjwepz13clCR5REIagFxxeYIzm0mapWgaTrNOvETw6wWszRmxtTNxqmZaFpkQqy4KOL+k", + "pAtL2oXrMIj8gC/Nff6MJ73xojfjSQY+gIWJdsHmRrxsZP/fnp1o7x0RdGJ78lbd0iziUTphy9iZmZTD", + "zLtQm9gbvXCSMK36zIl9nUqUWKT+0IZyq1QwE+Smee/g9NigXGTcR3w6+Y+2brF01YpngOemdDM+SLbO", + "IEuZweMgJJPqv8shDVu7e2DBun9vb7XOWDWkW8ZlaeSxaWMn+lpIbCMmYQBGPRg5R/iKICR9Atp7XPBT", + "KA7/PUeuoXzHZYLE3E6ZHDppgitUJAvZc+veNCe7DSyUxgjb3dTmPKrtwlLWVIsWTHQCfBa1APF4YbI1", + "37w6R4qI2OXPrwV6dWCX2DKxVMpUy0+K0cHhyYv1fovK8rBUS7OqyoP2YC3YaIWmIIzcR4Nj0kXHR5At", + "a4+V3BaD9IafuUCRORXzw2gfvbVY8qWmkImuNowRLfKQB6O2DDvrrsWkerzto7PMBMTZUErV7MtRFPlh", + "As3aADiTe1FrvVurEy2c+WfPY8i0wCqTxVpNbD6/2vscHSSTPqgqHrIbH0jFKJRG/1dh7b80au+X19G3", + "b6aju9qfyQxLH3fPirea8NKS+uC0clUjmmOGSv2urHldOIYU0edc6XNfteLd3ubmm82dm7vvbgq4WoaI", + "KsDHZZir7cFS7wJ0tO7KuqZq1BhcjvRjG0ru/CLvTtAMS/aDgocV78jm9pNWNRJ1r23DsosB2XxihpRJ", + "KYc3lYUTG+StSxpFRoGRdMpwhJ6htfPjl78ev3q1jnro9euT6lIs+8K3Pi2wV52oeHn6Fm7GsRy5yMbm", + "ZD6cJ8SSayqVrMNxtQoQ/hysV/Npu1o3bpKmjbzkzXLA2F9KiKZefLX1O0B6ddHdNXI+AAjqQ2bRfX0A", + "rEshUz8X99Qa4ncEe9oo832QoRXIgN0m8X97ANM7GU4JccUntooqgEtxvjViaLdDPemdB1JLZhKi49O8", + "0k7urXfNV+b0bKu/ufe0vzkY9DcH7UqrBUv6Pjk4bN/5YMtoHPt4vB+E+2DR3vZSxzK20dFxdIUXEg2d", + "FTXsGCuwYP4Vtq21tFoFoNSBWW+Hw1rVcxqQVkHfc5E90tZobtJ5WuTvVVHF4jQyKeLFus5lJXKGJZKJ", + "gdA0NQ8yFXfIYIBdC+UFJweNCcJBINLcN+cK7BmFOE0s3w+ZIDLhTCvJ2thBv5KFRDGFW8Kse4gNlChL", + "EwmHbE24lKIsdyjBqSSh/gEC9Ls29DvUQ6MKajvoD4ZMzlKlhdh6Hx1yJtOYCOu2RGMKVyrrSKbG5oPx", + "AjUWWmBKGhIxZPo1D0jnx0x/398bDAYDuJMwps62/vfAx013ejvYtyC0JikW4PCYhaMFnD2RMpSykAh0", + "xiVRCiNi2KEaQ3bDm8XPRJ91n7dTr+znuV7l35irwGrbodB+LvTnskLV5+US1a2t191/fFY1a9JWf7Xp", + "OPar0U2u+AkKeBqF2iQa69POeKxIaP10kqi8+jcckG/ZJdN7tDR1G5urOPozJWKB3p2clOICBJnY4sYt", + "Jg5SomEdeHKjZdha4URYOZpbAsLeBwhsVVMpaIhfHPK1eAXnsoQNh7a4issNSW9mBWVmaTSfLJlTxb0d", + "kvkoTX2GiH7ksEHevj0+KjEHxnubTwdPn/Wejjf3ejvhYLOHN7f3elu7eDDZDp5sb25tL8nCaJHKdfvs", + "LK816qsX70KmRy502xdR2xQ4Xzn3bTDwFWUhvyodLd7ozGLvNvJzVff1uO7WQ/Bmg0DpYmipQUqcwAFK", + "At22iYauVBNv8LLtvRlsrvCyrZQXMLgG+ftGpMzc6ZkE98x/HRcGXFys8jhvJ05hQC7rYhW1ip23J9pg", + "f/fZ/u7nEs1lDqwaY5Wd7nFxm8KRHABwJTXBpccVfDQOMLNj9Q3jUraZDJ1uJ0u2gL/hoK0E8maPW2UQ", + "NW3Yrl+MLJPfDZm7xyVbAMIkDABcuK+1AKfrA8x+lh+u1YvDiKchKvi5DB4WXAIdF+wC3QzcyVj3l8G3", + "NJkA2n4AQGWA86dMC2K4/NKN2DTfffQS3oVHODYmkx2EKSpRvPfB4cIEa+j95bo2BszyIZ9b2wW+0YYM", + "0v+CaWsyWHfo8iaM5rOPfuPwTWZJMV71q5rXwYSpv171wa5ZrGgH2wCdWTVuH/2cqW6Z8meVvTVJ7J8j", + "K7BytJT1Us66XfGO5pZ85Qr5192OoWin23GEgjztesb225zra/uvyIq+KCaCI1NRPUtQTRWNLEY0zIRK", + "RQNpMxf04jbpF7beDQlHxjBpCog0WY/WeMk+curLuxO0BjCAf0PWaaz/tZ4FT5bOuq1nO8/2nmw922sF", + "9pMPcLXaeQg5ufXBrdRBgyQdWb9D09QPT98av0JgLPYs8OLdSRFbIRFcix49c9dgsfNn/WdFjKOQp+Oo", + "cKtlAdEMpCosmBfGK5NFDUF4f9JoTicT9ueH4HLrD0Hjzes9uTXebMBuNR35XVrHxZvtmv+XjHumdo0f", + "hgYYSshGpKYzImEG6JwoBPzTQzgA0yFLpbUs5/CcLMW9jLWzvb399MnuViu+sqMrbJwROLg8h7IdQWGL", + "wZto7ez8HG0UGM606QANAF6bWbPSv8+Qrd86KCuk/c3Bto9LGg7unGts2/O4keTvrGlmJ2WJDhnBmdlW", + "2+Veam9vD57s7D7dbbeNret1JK6XSxiXL2PIY1Hgiyu/Btrkm4NTBNmoExyU/SYuPOlGo1I3GhVUMDDI", + "4zcY2NMne7s721ub7SDHfBEPFkyvtGHLssuz6TxM4VkNDynqorfbdFr41CnDYGckiDCNDwIX3185fQzC", + "+EiY1/JFaHMwWG9/7eBq8W0rx1HmDjLZIUY14AKlLKtr0V993flFbi2bpbY5HlZLdV9OCNPUs9g4pq7V", + "LUiZCDKnPJVfoCGuTMLmJOJc3OjbJoPljMg0UuZqkUr07uQHkCma15BUJCnbUJYblyAI3XJyN9rPJRbx", + "M3kTsVqtRpulXzbhbsOu7S5DcyhJg0bcrlBLrpStjjw8xFGQQiUXnK2nnhUAUEE+epJECxNYHkWcMxTM", + "MIMLCFGA20EzHoV9bximfjKaeCMW+BWKuEEcviQksUVOzCD0Z1qFoXOC1gr58MiwUqUY5W5shIwtY1Hm", + "xt3YX1UPS1+mVJaHremJFS+A4ZpPSi7HiE8lGIUKQub7VQz2BAsTCY+ZKdozj40t6Yn29QyxIsx9J6o5", + "SfnEGrhW5YAsZ0NJHAguJSIRnUKBmHcnleTZJQlXWQrt6mjC8mBbsK65LPQcZQaKqXVtL9/56Ekm+ZwT", + "EngYEtaWxOk552SMWQplTwqMTK4TKgx7tIvFm3GpRhko0g0HK9UIahmkguRQbVnKd+YPcu94z0Un2m5D", + "Lhv0equva1zlb6ppgM0y1UtRP7W6GQ/62LgOC7UUiSqHtqriGN0EKS0Hv6cSWqUFzCy0BgkXBbFUwD9b", + "bxOA4jdZdT81a9WWbHy1Mzhviym2HELsFKvZMZtwD/DEDW4lnSfahkomRMQUqnmgkDBKQmdLZteT1tUF", + "2cqRJChMiaWc0U8FtgTHZnsDeARzPjLKphVZX+2wjXvYjGF5qQPo177YJpRI+rM534gUaGViASXCeV5n", + "qwBLKkf+66x6w4JM0wgLZNEA2wxZLuKIsss2rctFPOYRDZD+oHrnPOFRxK9G+pH8Ceay3mp2+oNRU12c", + "czM4m5RmFqTSbz6Fn/Qs1yspseCJ2TDfbwBcSZvILG+Y8s80IhZa7i2j1wVGL2OB72wNmlK1GxotJWnX", + "YQlvKrkty3p3fCo9iW1LtRxX0YeEFqDdqD1JKk1xkRankkMAdVeAt7vRKWctfB4uxqGR1xVUDDQmkHTi", + "plaXGi3EYpupeOsapHKG/uDjskO0bUitp1rWGsvxGQSZeIPLYUWXOqTNGzWaFFb3JgAIIFb1ROGjG+IK", + "rKorlscuNcmTs1qJrRmxJKNujqbcVovyES62Isudt722T6KvFkPzxAIDRIpUCygrCrVcFoUKfxKNuRAA", + "f6w1HM7cbADzQ+s8mtYOdAm9mZEFEiTGlA0ZZZmTFJC8CGJkTkQhRZMLbWRNSdhHfy+YeAAYHSdqYZHI", + "wXn+g0T8imVjHLLiIHXjqdTtHDDjWRQpVHDPX9LNgtWnGQVSZuESTAko3kfVDE0EkbPi3H0FG7WOd8VF", + "2FgJZ4HcK1BgBe5YkeKXhBVFWdaM1zQ0DY3MV/UIOVNtFZ5a+xOVCqCiaoHT5f2lkgi/kphNKXulVehK", + "YasUjBODQAJwHFDYzv5lRHwGwdECcCNv/n9ck/lPp1nj5d8qrxVANRzG7YFx23pdsIFJkakE+5Rv0laG", + "qkAO1jIolfpdAlpz4cmuOkhZEyhU6Wh1TrZLA6sG4rvRbEgSlHvfebr7ZK9lmZTPuqwz0FFf+mpuHi+5", + "kmtYqZM29z5Pd58+e7a9s/ts60Y3LC5no2F9mvI2iuuD1si10ps1+vc///XupHLrswvxzYMbDcpkbfiH", + "1JC5UR7Qu5N///NfblS3HpBP0NThqRvu7RujdKLiSrpAgfIVXrtLsiX2/UHJSYAzMYPWyGRCwA06MnTr", + "5YOpYFC004JxggOqFh5Bjq9MJHn2SgVmuc11UHmwPpXXtG0hObXkkuk4z3hcc52jv5q74QovPG1dbUmm", + "46Z76NfVXs0tdH5rUYxxaBFiILNC33UHdzafKyxLwdL67wByGlz2Vj2TxbyxHPK1mmYAUSy2qFghFNAH", + "FV7RJ+1HxeWvLGfh3rLk1qlS/P2Sfdi8BW/k9fWcyB6nb7A6bbUiH+wBeLuvRuNiHbSlheZKRdPyU/fm", + "/bbIzK2D9Gcn2M37KyQj3uTDKiAt8KMdgyV53na3xBIN3FTIM/E40HhEelmgnk1CQTI1N4J6z1uMc092", + "ZHDJJ5My0OpuMzA3YM5AIpXrBSulLZMuItfOZ1FFdTYAM8POrhx2tAkw7GzGw07l2sqbmhjj65HtoAws", + "MliGlJ1VAK0OUroZjCMeXJoSX1A5uo8GKCaYSZQy2PyVW7XNwfLboW4nKaxNhktNTIhTTWzBmMZkhucU", + "yiHYO5VpKRCTXFMlIWAU2tlHITdQQ6X6pnaG+jWTOLifTxoOHcwWtmHdoH6PMxfRmr8LDr4JVFVlH4jg", + "XZspryX269cnXRPAAKGHZmCl+EY3UTMCLSCzLirY/vnv/vjhcURGMO4qVnxcp2Mx3xtuVgWRREkLHp2z", + "Q4UJUMBTpqog8nE7Q66cslU/klIGwX42/ANAwWzvtjh+SALYkbK+F8uMfgvmruQNWEr7Ege2fSwMmwLu", + "kvx3xWf2Qrg6AONsKJQmNu0U47rNLeFIKm5rWWW7ekSuA0LCKtqk/5W2sfL2S2+s/CtsAWqyqsH2bYh3", + "rs+uf3fJUzDWJmoXY/oZZz2AxnBLamEsDC6dBUopM1oJ97oAFzHyYXv6XmiTzUyul9P6N3KtAJw7TCOD", + "uOZnXSuq7GG0iuK3zhps2tBcrC6yfwc100y8+a2qptlQ9YconGbfupNiabXVOSfKvXtu2ai5ZH6pykjp", + "SssF/LtXyjE2hpW6yB7waDNer7DgzszvFbGIsC3zHxmOySgRZEKvlzCPecEYxmXIkHwjZRkMBtxyLcbX", + "aOcJCmZYyMrYGZ3OVLQoB+DseIB8PquioCCKMOcobLPy+Wq6D+vRbnY5i637lOPzAuxOrZ6GVUlHy0Cb", + "D/PbNuudT/ACvDiNl4RPtncGg+2twa1Qm92wbkCuw/wTW3+v3E5TSl3hO3vRX4pSLbaQJTDXi7peCQp5", + "0BmZpBIEx/uQeJPggKCITAChLSumvfpmsdr18sFbhcqCvWT87xbKrpu7gy/Xa8m6soDXbhodd7lYxvcp", + "Pl9xIdogZoIanpsn5267N9h7s7m9v7u3v7l5F0jLGZGasj2efNi8ehJt4clO9HTx5M/N2ZPpVrzttcMu", + "qSlL04ZXf9XvNkbZ5IdkGSeoJNLQmp1DQkS1Wm+1yrUkEWWkJ7MMqdVpiktkgbl/X7n/b+bnNzNYqjuc", + "lydZVCGwyolT4qyHwbayk1l6d1GdzfHR8lncKgOpOhA/v1WHAuzVbjBQHmGz85moBylreQy9LbzY+iBa", + "mhW36ijy3bDDTveucgPFfexdkpOlDbfsAK8fch7f6ZQLqmbx8tMiey3DsIa46Q9ShWUspT46njIo1V38", + "OQuTKxpR+uNOtxN92CnvGft7e1QtC3+bMaBd6qJW0CKMDCrBL6cCvJIbHsJEsmtbXY/5p83e5jOIQ4g+", + "7Pw06D0rRxx0DbWK5Nt0b5d+HbShYbH+nKtbtPnsRhHXjp7LOOhX6quelp/LFhjX8nheGNkdHS7htrTA", + "+ePaGldQchoV0M/V9OzhNioqTSGJ8MIHjF5w1MqK9VhkMjQmU8pkG7/t9iBz3O7Gw04fHVh0arBl8zL4", + "peahAHqBT2gck5BqHdOY/s0ZDFstfXFVW+JmhTHcVx5tre9X156thkhYlXC16pjsf0Y+7mdZv+0s3mXo", + "HeBXcyYq4HPBi11EJwizSnVMyuY4oqFNpIfESIhX23cgaDnLWhkgcz3Q+Um6aMoVylPoW/rbUtbsF8zG", + "T67B37oEM8MwxNYXAUTJwLnoMvF1fIQSwcM0yPNHIxh0jvgh0gr82RIlf3VI7l36NyAxe8IFWu3faHJo", + "tPNPNq13xTepGbZ5qTcHq5f6Tpwi3U6ahKtlmHmpnQS7EWz4ihREj4umTPaKJliYzPsWEv2sSMG6zWt8", + "yYFWidLEXbBonqpzkue6Ba4YfHG9RyQi+piqN4J4FOZZElTmUnS1SN3cezpruuKEG6n6QH4lJNG2CuAf", + "QX8xZgvvwFzNy+wsWRu4erbSXHj1TK0cS63y4J6s1MQal6rowm3C7zdSvuLzNngplwX3d4afXVTN6ggo", + "TuCXlLSzZvx5+6ULe2v0H9+FW+4hlbTX9uqhAonqAHsz5HHXfx4LrNW6MvPu+K7nfWzxxlrGTTCv1SzQ", + "otf5oPcP42VGo/7+xk9/+3977//q9TZX7GZJRC8kEwg0uiSLnql8o230fhnkFGD3tTI9taxCcAw+JAAQ", + "t5uxON7dQSY0Fr/huDYFiNAqlK3ZXDmhv/2lOb6pQMa3ICdXsuxnV6W4i+qdirvjaC0mYupiyV0i2Xp/", + "yCA37ZIsJCoUw7IqjWPUH2T2SSECHV0YNbBP2PwCjSlUF5RDpq1aHAQk0daEra9CTYlsDtJHEBwV27FF", + "uVzit72QNPEEBL07qSHkvn775vnrt78djV6fvvjt4Hj064v/hRCPq57pIexp3tvZ3bOFsYuU3PQs8Wdg", + "+H8WSK2P3QzUpYe/IMkSqo17FGYqASLBhRsUXkZrkKzgKm26XM31m0FvHmQNeoPdvnBBlcGzL1H08O3S", + "KodzHvW0Rt0AfO91YBpaeIO1oSkTBN9p8nNPxx610XoTp3SKPa5tnwv7ixQndANamaRTW//GKk3+0Pmj", + "alUAIw0MqSoo9hW7VKpec2R9rBWpUV6BvRyvkTKbLkkL4Vzl3MiYqQ1bRNQH0RBywKZeliCb7zIHyNeD", + "j1bnfS5V5QszK4ykeW1OnMZa0amXEOhUk+ZqRgQpLAR8kKOt35BkNhWkBfCHKcKWEJGHSbo8Eq0IwQ2n", + "RGuZs8GRIEtwrXtgl6Ppn+DrrAfw3mNZu/KCeeTlbjZfPgfk8zNX4o9OXBMwjIo94cf5LnPRMpo4rqov", + "RpGr6vM273s3npVVS6Rf096qMGfeR4k1ffz4d0zVz1yABdIMs3HncOFg3YREAOxYFQy8FZI2jUk44qla", + "vv9tBXGLsZGVgczLiDprCwMTB6UM0iZZ4IAg8jHUKa3JQYJUULXQ5rlVhseQeedqdwIhoSP4Oe8Y6iV+", + "+gR+yoknR+ElYUTQAKpR6v0YYwZKOnp3UihKZurT1SBCQb18fXhsLVyHMgsWC1XAei7U7+D0uNPtzIkw", + "Vl5n0N/uD2AzJ4ThhHb2O9v9zf6gA4r8DKa4ARXEbcquTXHNbKXj0GpCz91L+kuBY6Lgi989yecQ6mZf", + "B60XTwt2S4KpsIZLEkFSvGEYqr8G9Hh3oO6bU7lryN7aTQeZrZBwQZLXdnHfg1IJewemuTUYWCxtZY9f", + "SBcxMeobf9iAxbzfVlqdJZEHTL1mWTjdMiP9p25nZ7B5ozEtGwrsXV/Hbxm2eaMEDMLdGxLiVp0eM5MJ", + "ZvN6bQROcccBIxX32u/v9ZrJNI6xWDiCFamVcNmkGBOJsHvXQPwqiQItKqDkSx+9ZsSWzccKYRMsK1IG", + "pWTdh5pDy7vAtO0WOcPFec7DxRcjYakPZxZ/KoszvV0+1fj5y/FOxsb1hbSPHKiz4dp7YKDnOKuD/GA7", + "ZWfw7O47PeRsEtFAoV7GwDYElkqIMokAotrB3XCB/ky5wiiLIH9EW9rqrOOM3br5UbTxkYafzPaOiM/z", + "ekpEjJmJxzfvrNj0te1svOD5dl56qjnGh2oScFI53BdzUIEiV96ixWOrqgzWj6MdT9K/7dNML3xAxt+5", + "hx1uJ5uVwnzILQf1FVEqyWPaTvZWZ5wrIV5d7iVRXwvPD+7zyLK49d/gLnosDPySZBpevlq1Q2EjESkz", + "BrBXAzzLc+Tsdz+Ulb83+ZNCYAa40nXTUEFBmas8HC76yNHUGP1qAag+gsA8w/qxcqqH97XssK372GEw", + "4+xy4vsx9f2YWrbLDbe4KcDGLOzyFj6IG3kgvj3/w429D999D+19D608D4xcWe/CH3zcRzYIEurQyxlP", + "oxCNCTIQOy7cQWHRn35AWAQzOieAowZ1wdJI0QQLCGaIUYgVNte2jY6JpW6JrLkN3VzPhb7lBK5CJ0gy", + "Aui3URPkYR70RhkjIdKfWLS4HMGuVh3a7H2vgz1rMD8a0dWMS5JByDFVOM0ho1Ya6xia7Q/ZG4stqgkI", + "8btO1kgSAULqEv8PZwgPmf3gRydCXOyRxHEuubAAmDpqwBDNstSzqfRIRzLgPniXN4RhpnoyIQGd0MBO", + "65IsbAiht8FWpX70gN04351kOQJoa90PEQaIgH482KPsGbKcVL6/YRB3G0RpmF9yOdQaLMY4iry1IKYR", + "H+NoZOhzSTx3gi/hDUuUYhV5d5vEeEhMRfBkoWacmb/TccpUav4eC34liRh21vtDBrH/ltYk7OYKIrqC", + "2mFxwvU+Ezw2fW6YIW58vCSLT/0hOwhjyhxHwCc4khyRa/gOSioBTIORXg38YHaT/x78MJWKx0WwTcd3", + "Zpg8VUmqbBKDJKrrA5ocMsXRRwcn+GnjY97jJ7gsJjjUfFJ4xUwJdOumUcsR1rMfwaue63YCBBh29EE6", + "7Oi/pwIzZZAiMzxENC0u6VoGyK836XqVwgFmKOGJKWYATDXDmuVKbQA8AI4ipGAruW+14g4r2TAfi/YW", + "jxuh3gw2V2UbUYZOnhc202DnqX8/SRII4oso+e/z178hOJX1GpjX8gghk0XAtMKAwhSuTp1Me4GDGTIX", + "VVC/btih4bCTXeeG6zDWVNqE+V4P7hR/0kP7yXTTpeFP/b5uylxX7qPfP5pW9vVeSmIDPTnsfOqiwoMp", + "VbN0nD177ydoE2LWeUkQoDVzzK2DJMEUwE0KJ745IjELEbenQLRAGOUSqBi4MqYMi8Wy3DUP6S0F+cQE", + "zxWI8XEIwXLDzv7QhcsNO91hh7A5/GZj6oadT34K2FvL5mJpcJ5ll5sZE+0NBuurwZctfT13li0uBr6w", + "DdhoFWWVHvUKWuTPb+t+4D/a/syufjDTnedoOMbwd873R3gBUdDYi5ao5wqionZjFpDIqd2rHT33f3mg", + "FysgUXTfDPpQ7Jldj2Xg8I+KHWGx8m201H3/wBw3uK9DpeS2fxj+fXT+c4/33PrOydyFOvtLYwDsiTWl", + "kXkZYYnOYUy9c218v4Bf+/a/zvYDGL+LiE8v9o3pjiI+RRFlNgS9EKis1QNLS/jIIJ9k31kgFFeXbM1o", + "Ev/+579gUJRN//3Pf1k48X//81+w3TcMoheUNb6YESzUmGB1sY9+JSTp4YjOiZsMFB4lcyIWaHtgff7w", + "CBWqq1stTQ7ZkJ0RlQpWCNU3JcKkbdBeFej5UJYSaZFj9It0YuuXmNhGj9/G7WVDynvd0V0PAh/MoDAB", + "fSo6HgD4MmpqO1tLtON3mZo5l5ym1TDNWrDeavmiyLUy3NszA7yhgAES+/YdPLCTRmvn5y/W+wisLcMV", + "UKMGbIe8GWtG9L/LpNUyyUiUskABKhvZZOCUljv9j+w77bz+tsVvye1vy5bdwO9vnD8AquhW4PsdQIs7", + "AD/d3H2Azyl/5PDC7i5Y0HTxQLGCjvfqNDdPCiR7CGcAWnNADOBQ5QKdHh4jHIaCSLn+n+0q0DM1XJof", + "HYgzKArwELfWdixcWIgqa6qVGeSxiIMzO2qE3byq1SCL59tGqVRE40mXVY3Ij7y7Pz0qnd7kGMkrVua8", + "9v0kWRmnR2XAoShWzi29ACdASKe+ZPu0yEWrHFImAjA7cpaqS1Y8Hx+5DXl/rinbdcqqZ8M9CMWjikB8", + "QEFYztIs1nh9TNz8NltFh426xHP1dbHm4P60oPv2YvnY/DG5scIK2bQUNHgCjQfoS6IMikDnDhfa9uCZ", + "+DkRble7ktww62xa5lNk4BBgQnA1v9z2PTavtDN9TXvfkuUL5LmJxmJJ/l1FaWHs5rRaZuAe24Kld2ff", + "Qg83Mm+/3I23ZTAPkSHsZuw81kKREK1huWDB+vdL7y/O0SYkKjdihZs3CVESYQXRkQDEktlZ96XXHbBi", + "dVyt01m7ljI0ieh0psz9R0gnEOajitVmYZRb9zDKrKqrwIrY6KbHmDJ4qolsL5DmRCj0+vDY0L94pG58", + "hHi31aaSE15LT9e3Z696hAUcAhyz4Dy/TmqffGGDyfB/KQ3w/nfdI0yFo049aFIYP2P9TRwqMqGzfcr/", + "a+vniI4FFov/2voZRwll5L+2DyKsiFTrd8Ysg/s66e7bgHnEzKftF1omGogmNgWAwxUKf/ZWS53fvf9N", + "qf1m0jdS/DO6ftf92+j+RXItVf/tUtypAWD6eKAbrozZfNSGR9/RMO7BaWo5soCGUbpFyvEwZlwqePT4", + "UiNtPCrNOK54bLT0/ucbcunx4Vj3+KgLhIT6p4C/bjOP7ukuwI3j3pVb2+/9XwQcxGM6TXkqi0lNMVbB", + "jEib8BeRsgB+bGp3fjw3Kt5fMZcO7vPouHe9+jvf35HGX11QI7zNhd4qnd+91Vbnt+9rnd+AIdqkSAsS", + "33UFRNYbYjQdHGJbNi6hRtZjR33j8tki6K02VHJzAYEFsT9k/0fbH78rguP3P7nsq3Qw2NqD3wmbv//J", + "JWCxE8cqhEFlc0jEPfjtCG5Rp4AjCSWh8lzP6jhMhVlgPQeC/R9nIOUXye0tJMeF3y2kVhZSgVzLLSS7", + "FndrIpWB9O/dRnL85iO4hSP+Nq2kb/x6pGTByXQyoQElDMoRQE6rrMUDGkvu+83ILXMZmb2PLAQTlTSR", + "1mZkJrVWaOh5BdR7DyQ7zku+3Lf16IqtPs6cDJ7Y6oXWXsu1hWaD7Wvjh8H9nl73b6g9ZhYzFlGddIlW", + "uj1FRUw5nThVEASbwxBBlDESxqzJWuyjwyz7XKZJwoWSpiQPWAimaOdMWwi+8j3lijy+EjxQdoYS2R0y", + "KMqqHxsUjY1LsjAFdyhnWW2dbKa2bo0v169c8OhBt9GXV0L91ZxaKaH3vI1tfb6HU0IfTHTci7p3XCp7", + "upZtDLC4xyTbyTxLJqUfKJuuP6qIZyOssrkVQNM8qtYGThV3xfs3ZtzgJ/kh5E4jHACCnH7NgBvZ7GSD", + "ZlZsClKOBY8iIgxoVZIqV91ryLLBUVaoXmxLaVzo5kcpUzS66JpoGkAekAizhUVuGbJSZ1gpEidasFks", + "IhihIIkZcaWsmR405amEt7pI8lKXCEdXeCGHTJBJRAI7NygBKUhg8N2iqI9+4ZDujfAUU2YzkPWbpijY", + "D3LILmgYkZHN1r5AVCI540IRRkIU8zmR5X4JFhElAiZxiDXlJIrxAmCTDIKcoQ9PiIEmKuWEc/1vzEIK", + "JbN0z9mU94cMo63BAMUEM4kopA1LPCH6K9sGgkGUBvQjwmhn8Mx+VVk3gPZ05F/T+0UIMucBHkcLRDQX", + "w4mo1mEBs/0F1Sf18k2okGa9MveirU1UWlgqXRXNsItSFkCRx1Tof3GBUmaPV92igEx4mKe9hCNUZMXR", + "bNr+mARY05Pxcj8AmMaDIBW+w1EvdaF833+iklmY3jmQyiedNB0Q7KkQ1pxxNYM9zWErrf/YwFU5U30b", + "h4x3k3CBMCrwde5QgErXbIrWAGDsIi8MxlxtyYv1H93e0dvXCgK3/Q3E12M5n4CJ+GRS2oCrjyazgZel", + "V9RZ+Fvdp4euImRRxIUUTxmXigZOGFZrFn83Hlsbj8sp6+XmCReXRd2qzL8/c3HZ1vo6d4X4H5URVpzh", + "V3gPoIcHELEPfx0AzmhjqGimuXcDrcpf2S4FpYsq6eKMOYo4m+pdlDvF791rX7HogigFtdyZcs4Joo2Q", + "kf3RFJXUk7El+8DDH9hWH1oW6d7v4S7oN64QjZOIxASKTvYMs+nFzrRqUxOaSjTLqv3dTFbqXVVMHTa2", + "oDTX/12nDgFfuQVbA+29vlxeoRrx6Wq4sKxzh43lwQsbMlOzmrgC1xcok8FaoTXg3OhqRoMZYIeB3arb", + "N9BiOEkuMtjU9X30EjZyET0WOl8zkNya1ySPiIEEm8fxxX69rOK7kxP4yMCGmQKKF/vIlVLMzg+p3ypi", + "gelZRFgq9JtFOFvLjHFY0QuFtb2ZzW/dooTlsLZD5kMMY+TKNkgn6KIAHnbRgB7m5O0rPn0wZazbDEZu", + "5qI4sqYj8CZhYacpxoJGftywzcHAh5HbEsPMDOOOIcxqg3nFpxkQeomVcZK0ZV87TODieRwv4WG0lksQ", + "JFXIU/U3qUIiBHxsubuJudEaDmwRHHypGZUZqeQ29jqwnzeSyCATe0mlhWqn2yEsjTv7v9t/zeO40+3Y", + "8RQQjW+g3K/Agqs2WI940StTAHz7rpbfBMqtLOwLWG6Vk8Oa080a+Zl54Zu/WXQ+uwdkQ9APKk7cr0kF", + "LYy37PBhHEmGEznj6nGhR1lXU0Vra3bVuFn29PDC1FXqaBPCcW4/PXdffgXW76rIDjdm5KZ77yEe9RE8", + "5kxYWZvNhIsq5NCq2I+vnpG+3JLUptqGQ77z5s39fK0YU2vqdRFhPwhN5SacKh5jRQOoGhLMOJcFth+T", + "GZ5Tbq9K3Z1Vxpng3DB2pg2hv9CsemEdwRdWkd+3TiuEi49sH3343Abe+79wj/Ivfi7Y5ZnE7zrlG5C1", + "oayxoGSCEpxKovWqNCYoWARaKpoyMQQHMxTgRKWCQAUsgmLKaJzGBVeDNpzEHANIxcVmfNFF41ShCIsp", + "2EXmoQmnFyTgcUxYSMBDNmQzgudUG3UCRVgRFix6kkDlzDlBV1xcRhyHYOTbKBxTeUsQzYGUsy6KicIh", + "VhhUjQu940cmi+ciK6ZpDGtGrnNuCIdMpOxHgwaum71wA71ARCo8jqicZUXXAhwSFnihts+/bjH25b3B", + "50RVJ/pAcTm3kqUPGahT9Hq64XwdMTyPLBiZC7uMbcT8EqVXNhuR5fQHx0b/mVvazNXN8YGueDISL9vF", + "X8fdTsZ0X839zsNf4HCBwtR0V9iVwObf6q1MJlCK4U6QWmmW8bZXM1l1qYzMN5J5Gx/dn8e38KZ9JZKw", + "22jYN9UxySf9NYhcS9VbydwHciNaX1LBK/aAItjFVD2Y+sRFQco9FnenFdhma2ZyuyidlMBgfXH2XWxX", + "xbYNObit2Ha+2dqlekGQU9aDKE2/BLdu3EZRbV0H/6G5IJXZFUTmg4vI/O7g3sTicSYIjWhM8CLiOPwW", + "wnSX3OAEXAiD/wCIEo8Jf7TgNSwG6INvrptJiK7LrXx3crLeJCWEWiojhHrEEqKQFKM/i32l/udECBq6", + "EuaHJ0c2YJZKJFLWR69jCnXFLwlJ8pwSAPLo6/k5JIx6MeYS5EW3Q5gSi4RTplaOIn/1bgbz6VYlnO9Z", + "TlpA6+8X0q0vpMGz//jEGUgZyJowE1humSqsGkMBXWgcZaZCu9bL8JinunUtgzSZ9HpO4RSc0IjIhVQk", + "NnGBkzSC7QbFEWztTPudWeUuRMXqnWMS1hIiYiol5UwOmc3WSIjQfevPdfuFECfvhYDCmXw9NULy6wif", + "04MxEWNYNVENMIugcn1nv7OBk2QjxAo3hGjZ4X3GkH6GeDgkF/GYRzRAEWWXEq1F9NKYJ2guUaT/WF8a", + "UDeC7750ZdDb7yxN6WM24d7iaYZnM2b+pvKqrFhzF5OPTqy9JMXN4uQPLLRfrMmVck0QHPUUjUmGXINS", + "RSP6wYg63QiVigYm6SeHLHh3kqMWDNkJUUK/gyG5LIpIoJzDZiMRPNgYpoPBdpBQgD/bJjA4EHjNj2Po", + "8fD0rUkEJTEXi+6Q6X9Aw28OTs3t7gRbb0JhoIyoKy4u0fHG6xUhxudApv/gGD0zwaXYAd4F/34leHNE", + "kMY9JBu2KE+WmUo8+eaDSK0G992v8Dj9CgDJlM1mbSpwAEqxnKUq5FfM70OY8yiN9T/MH8ergL0UDmbv", + "4NWvRts1w1nZjZvgo9iUdk4hMcUdH+TSwxDsscasasK5KYASU4oG9J4CB+pb5O4v774v0vErvO60FHWF", + "U7+avXXfJ58dg8O4KNLjsWxzw2luJoov9z5dYdrsfXoe8eBSWjCUottQ220AMK5/zAGh7RUhqAmQm4ks", + "iBAi1wkVgPxWcUAazB2JMFJExJThaAPmbBoBaGvnxcJzTiFFOogoJKnREFCLIkCnu5oRhvRswFHlGijc", + "6EpbWqr4TvEyUnE0JgGPiYP7XveZbn/HVP3MRRm7+2uRi28K9Nfz0VPV81wBV97c42fBl5/gawiVDlN7", + "oexGtPaS5z8aV1AXwdoMO9sDOex00bCzFQ87egUOMbhQsUK7KKYsVUT20ZHxb0ES7N4ASRJwFkqHOu48", + "eNsD2ZQSa9iyIb9yD767T7XHchWQ8sx24hMP+j2kv4ekHbRW3HB2T4Zd2HQh4qky7n67r+xbIVHgHlm/", + "97vawh75btu3keR/t9u3JKNglbW4LCy9kexJKmek2eX2ylTySdUY0KxddU05Q3/wsewiRq6MN1xI1a/J", + "Pf31qengPpD2dVc3Qdm3c/8Osd8CYj+nlR8u0QRY6iPZcYfBTCTXBhEWu7R3y0NgSQB2Aw9whF4fHg9Z", + "oEWRAfcTJOYgnSwguDmFD/5+jl4cnnXREVR6RL+k4/U+es2ihau3be5ohsxoYkZ4BZihseFaEvqOZzN2", + "4J67DBbXHTxQ6WSzMzw3K26tXJB4tzMjOASN5GPnFTedeTCCz17pDaTXz36ZLXtnqfLROSNKLHoHE0VE", + "vdkTmyfFMtQKe0g7EDiruBnoSd2hdNhneZ9GNzDgFNtbHQ9WxafvVQ/uvkLo/dySmTgRU29unALWJ4Mk", + "AxwuHlcsk5yhTDj6RGDxuM7qBjRlCVtZttTAgC6bIr+/Ipf7UtlVQoL/T91dMNNHe9GUlNZJM3FWb2Tl", + "Ta9LDp4ZQGJ7URXgBAdULboIR5E9o+xJkEWk9DL1dywIvgz5FesP2VlW6cQm9KLD07ddd1GLQiovTQv2", + "LraPXs+JkOk4GxyCjWZujYHmJBwyxVGAoyCNtLpBJhMSQC4uFDCRDXe52VA6d7h38k681VYKUe3poyvy", + "5ucJWL2cLaoct2GWekOQIMI0bob/tooaBBxCqMFYN8oZomwS2ZCqQHApkW2qRyI6pePIBgjJPnozI0ji", + "mAxZEmHGiECpNFHxeui9RBApU5PgrRsAmFzDUV2UQ/slgisbmhBxLqSJJtAc/u4ESUWSJWx2Zlo+gTnf", + "kW5rGrc9PZCTujKGZleIfQXpBTGcYgiu+SiNXADjvYaimwE9tJb4WDb+G0GnUyL0rsBGyJpwPLOtHTnN", + "pi9lLDcWfDzP3mpX8DFrtZCVWMjYWwrNNsrRrsPOzaL+PJ1f0kb0PvvoZlnEv+qPWvZdzlb1D8I++sxZ", + "fit19M8LSYJtHVg5hz82d1Jh5KWtWkq0XQ2r1Tqz9i4zXVvjZz0YbNZjRsvCpfTZJoP362OEwf2iPNx3", + "SbTHzVsltKuSbdqQ8r8az/6r4MC7AbJ/YJSTWwDZf1V594A0/nD4J96N+lB59KW7Z1dt9pvHor+r9HkD", + "SA9wbE3p80bq2eDVpYbSO/tOOzPJtvgtafA23vEG+rsj+3erv4XJUCDWqitozfAkTtTCBbTZu8o86EzS", + "D6TfcBGcxa3e3VXwLUI6vxx7OD5tDOj8NovDP0jMqC3eRyU6PvJUXX9kGIPFPVc6WDb0qdPDIpjROWl2", + "upd3sCVRIkgv4QlcroSGYJYe7ixTWPSnH5Bt3mKu2n9B9UcAyychCqkggYoWphKnlgimjx8kElxbAvCc", + "i0VzlIjZIj8LHh/Y2aw4D+2ess6wPM4wXvRCrHBv7qTNEhfaZ0R3unhKLfAQZejlc7RGrpUwNSbQRFs+", + "iE4ykppy+xJ4cr044M1Bg2eTfiCj6bjNKJdUC3ltq7GgIJWKx27tj4/QGlQfmxKm10Kr+hPQZBPB5zQk", + "YWmMnTmPDFU3Gwh6U7+rViqy0nHOuDCDexAdps2BNP1Ak7JYyEJixpRhGNzKuhzlPWWS+HV/mDIXgGPX", + "yI3i+xFmLb81Z+xoToRKmJaIinMD8bz+/Zh7zMdcMRnKnWml086F5yx3XrfLj2qZtnQXhR+y3Ln7dVu/", + "+3pSeqh8lNk81nU+zwzSJrf518WCg/s7H+7bXf7uEaeAviTO+C64yqEB3aKPYV5BTHdI5iTiSQwVyeHd", + "TreTiqiz35kplexvbEDs94xLtb/z7Ml259P7T/9/AAAA///ULxEss9ABAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index 2e7eeea4e..d69b1e2a4 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -199,7 +199,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuProfileSlots, err := meter.Int64ObservableGauge( "hypeman_resources_gpu_profile_slots", - metric.WithDescription("Estimated concurrently creatable instances per vGPU profile (best-effort snapshot)"), + metric.WithDescription("Virtual functions able to create each vGPU profile (best-effort snapshot)"), ) if err != nil { return err diff --git a/openapi.yaml b/openapi.yaml index cab8de2d3..f7d051c54 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1585,7 +1585,7 @@ components: example: 1024 available: type: integer - description: "Conservative estimate of instances concurrently creatable with this profile across all GPUs, bounded per GPU by free virtual functions and remaining framebuffer. Best-effort: recomputed from driver state on each query." + description: "Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer." example: 59 PassthroughDevice: From 6e2ea9218a7fede810f68d268100816965576eb4 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:23:13 +0000 Subject: [PATCH 16/19] Retain vendor vGPU assignments after rollback failure --- lib/devices/types.go | 10 ++++++++++ lib/devices/vendor_vfio_linux.go | 27 ++++++++++++++++----------- lib/devices/vendor_vfio_linux_test.go | 18 +++++++++++++++--- 3 files changed, 41 insertions(+), 14 deletions(-) diff --git a/lib/devices/types.go b/lib/devices/types.go index c76d239ed..31ebd90ef 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -100,6 +100,16 @@ type VGPUDevice struct { MdevUUID string } +// VGPUCreateCleanupPendingError reports a failed create whose assignment could +// not be rolled back. Device identifies the assignment that still needs release. +type VGPUCreateCleanupPendingError struct { + Device VGPUDevice + Err error +} + +func (e *VGPUCreateCleanupPendingError) Error() string { return e.Err.Error() } +func (e *VGPUCreateCleanupPendingError) Unwrap() error { return e.Err } + // MdevDevice represents an active mediated device (vGPU instance) type MdevDevice struct { UUID string `json:"uuid"` // e.g., "aa618089-8b16-4d01-a136-25a0f3c73123" diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index ceab85ec5..6615e035c 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -159,14 +159,21 @@ func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID str if err := os.WriteFile(currentTypePath, []byte(requested.TypeName), 0200); err != nil { return nil, fmt.Errorf("create vGPU on VF %s: %w", targetVF, err) } + device := VGPUDevice{ + Framework: VGPUFrameworkVendorVFIO, + VFAddress: targetVF, + ProfileType: requested.TypeName, + ProfileName: profileName, + SysfsPath: filepath.Join(s.pciDevicesPath, targetVF), + } currentType, err := readCurrentVGPUType(currentTypePath) if err != nil { verifyErr := fmt.Errorf("verify vGPU on VF %s: %w", targetVF, err) - return nil, rollbackVendorVFIOCreate(currentTypePath, targetVF, verifyErr) + return nil, s.rollbackCreate(currentTypePath, targetVF, instanceID, device, verifyErr) } if currentType != requested.TypeName { verifyErr := fmt.Errorf("verify vGPU on VF %s: type is %s, want %s", targetVF, currentType, requested.TypeName) - return nil, rollbackVendorVFIOCreate(currentTypePath, targetVF, verifyErr) + return nil, s.rollbackCreate(currentTypePath, targetVF, instanceID, device, verifyErr) } s.owners[targetVF] = instanceID @@ -175,13 +182,7 @@ func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID str "vf", targetVF, "instance_id", instanceID, ) - return &VGPUDevice{ - Framework: VGPUFrameworkVendorVFIO, - VFAddress: targetVF, - ProfileType: requested.TypeName, - ProfileName: profileName, - SysfsPath: filepath.Join(s.pciDevicesPath, targetVF), - }, nil + return &device, nil } func (s vendorVFIOSysfs) destroy(ctx context.Context, vfAddress, instanceID string) error { @@ -475,9 +476,13 @@ func framebufferFromProfileName(name string) int { return gb * 1024 } -func rollbackVendorVFIOCreate(currentTypePath, vfAddress string, verifyErr error) error { +func (s vendorVFIOSysfs) rollbackCreate(currentTypePath, vfAddress, instanceID string, device VGPUDevice, verifyErr error) error { if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { - return errors.Join(verifyErr, fmt.Errorf("roll back vGPU on VF %s: %w", vfAddress, err)) + s.owners[vfAddress] = instanceID + return &VGPUCreateCleanupPendingError{ + Device: device, + Err: errors.Join(verifyErr, fmt.Errorf("roll back vGPU on VF %s: %w", vfAddress, err)), + } } return verifyErr } diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 4555f5c2a..b45884640 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -428,21 +428,33 @@ func TestRollbackVendorVFIOCreate(t *testing.T) { t.Parallel() verifyErr := errors.New("verification failed") + device := VGPUDevice{ + Framework: VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } t.Run("preserves verification error", func(t *testing.T) { currentTypePath := filepath.Join(t.TempDir(), "current_vgpu_type") require.NoError(t, os.WriteFile(currentTypePath, []byte("1148"), 0644)) + sysfs := vendorVFIOSysfs{owners: make(map[string]string)} - err := rollbackVendorVFIOCreate(currentTypePath, "0000:82:00.4", verifyErr) + err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) require.ErrorIs(t, err, verifyErr) assertFileValue(t, currentTypePath, "0") + assert.Empty(t, sysfs.owners) }) - t.Run("surfaces rollback error", func(t *testing.T) { + t.Run("retains assignment when rollback fails", func(t *testing.T) { currentTypePath := filepath.Join(t.TempDir(), "missing", "current_vgpu_type") + sysfs := vendorVFIOSysfs{owners: make(map[string]string)} - err := rollbackVendorVFIOCreate(currentTypePath, "0000:82:00.4", verifyErr) + err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) require.ErrorIs(t, err, verifyErr) assert.ErrorContains(t, err, "roll back vGPU on VF 0000:82:00.4") + var pending *VGPUCreateCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, device, pending.Device) + assert.Equal(t, "instance-1", sysfs.owners[device.VFAddress]) }) } From 4a65d4efc9cf813f518017a9a4bf25b81534a79e Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 15:39:33 +0000 Subject: [PATCH 17/19] Degrade vendor VFIO discovery per VF instead of failing the host A single unreadable current_vgpu_type failed discoverVFs wholesale, and GetGPUStatus turns a discovery error into a host with no GPU, so one flaky sysfs read blanked out the host's entire GPU capacity for admission and monitoring. Skip unreadable VFs with a warning and keep the readable inventory: a skipped VF is never selected for placement and never reconciled, both safe directions. When no VF is readable, discovery still fails so a wholesale sysfs outage cannot demote a vGPU host to passthrough while assignments exist. Also document that vendor VFIO vGPUs are known broken on Cloud Hypervisor upstream and QEMU is the required hypervisor for GPU instances. --- lib/devices/GPU.md | 7 +++++++ lib/devices/vendor_vfio_linux.go | 19 ++++++++++++++++-- lib/devices/vendor_vfio_linux_test.go | 28 +++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index cdd269dc8..0cba075fa 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -99,6 +99,13 @@ Instance Stop/Delete → Release profile → VF available again Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. +### Hypervisor Support + +Hypervisor selection for vGPU instances is caller policy; hypeman does not enforce it. In practice **QEMU is the only hypervisor with working vGPU support**: + +- **QEMU**: fully supported and validated on both mdev and vendor VFIO hosts. +- **Cloud Hypervisor**: vendor VFIO vGPUs are known broken upstream ([cloud-hypervisor#7572](https://github.com/cloud-hypervisor/cloud-hypervisor/issues/7572)) — the VM boots and the VF attaches, but VFIO region reads fail, the guest driver cannot initialize, and the vGPU is non-functional. Do not place vGPU instances on Cloud Hypervisor. + ## Passthrough Mode Passthrough mode assigns entire physical GPUs to instances via VFIO. diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index 6615e035c..bdd026ea8 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -6,6 +6,7 @@ import ( "context" "errors" "fmt" + "log/slog" "os" "path/filepath" "sort" @@ -50,6 +51,7 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { } vfs := make([]VirtualFunction, 0) + var vfErrs []error for _, entry := range entries { vfPath := filepath.Join(s.pciDevicesPath, entry.Name()) nvidiaPath := filepath.Join(vfPath, "nvidia") @@ -57,12 +59,14 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { if os.IsNotExist(err) { continue } - return nil, fmt.Errorf("stat creatable vGPU types for VF %s: %w", entry.Name(), err) + vfErrs = append(vfErrs, fmt.Errorf("stat creatable vGPU types for VF %s: %w", entry.Name(), err)) + continue } currentType, err := readCurrentVGPUType(filepath.Join(nvidiaPath, "current_vgpu_type")) if err != nil { - return nil, fmt.Errorf("read current vGPU type for VF %s: %w", entry.Name(), err) + vfErrs = append(vfErrs, fmt.Errorf("read current vGPU type for VF %s: %w", entry.Name(), err)) + continue } parentGPU := "" @@ -76,6 +80,17 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { ProfileType: currentType, }) } + if len(vfErrs) > 0 { + // One unreadable VF must not blank out the host's GPU capacity: skip + // it and keep the readable inventory. A skipped VF is never selected + // for placement and never reconciled, both safe directions. Only when + // no VF is readable does discovery fail, so a wholesale sysfs outage + // cannot demote a vGPU host to passthrough while assignments exist. + if len(vfs) == 0 { + return nil, errors.Join(vfErrs...) + } + slog.Default().Warn("skipping unreadable vendor VFIO VFs", "error", errors.Join(vfErrs...)) + } sort.Slice(vfs, func(i, j int) bool { return vfs[i].PCIAddress < vfs[j].PCIAddress }) return vfs, nil diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index b45884640..0879af3e7 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -29,6 +29,34 @@ func TestParseCreatableVGPUTypes(t *testing.T) { assert.Equal(t, profileMetadata{TypeName: "1159", Name: "NVIDIA L40S-48Q", FramebufferMB: 48 * 1024}, profiles[2]) } +func TestVendorVFIODiscoverSkipsUnreadableVF(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + // Replace one VF's current_vgpu_type with a directory so reading it fails. + badType := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.5", "nvidia", "current_vgpu_type") + require.NoError(t, os.Remove(badType)) + require.NoError(t, os.Mkdir(badType, 0755)) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + require.Len(t, vfs, 1) + assert.Equal(t, "0000:82:00.4", vfs[0].PCIAddress) +} + +func TestVendorVFIODiscoverFailsWhenNoVFIsReadable(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + badType := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type") + require.NoError(t, os.Remove(badType)) + require.NoError(t, os.Mkdir(badType, 0755)) + + // With every VF unreadable, discovery must fail rather than report an + // empty inventory that would demote the host to passthrough. + _, err := sysfs.discoverVFs() + require.Error(t, err) +} + func TestVendorVFIOCreateAndDestroy(t *testing.T) { t.Parallel() From 4b016ad8a0b2038576b539b0b36360e1f52e143f Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:10:26 +0000 Subject: [PATCH 18/19] Degrade listProfiles per VF and document deliberate probe strictness listProfiles failed wholesale when one VF's creatable_vgpu_types read failed, blanking every advertised profile while discoverVFs directly above it already skips unreadable VFs for exactly that reason. Skip and warn instead; underreporting is the safe direction for status and admission. Also document why openVFIOPaths stays strict where mdev's scan is lax (it authorizes clearing a reused VF path), and the 0Q/0B parsing caveat in framebufferFromProfileName. --- lib/devices/vendor_vfio_linux.go | 15 ++++++++++++++- lib/devices/vendor_vfio_linux_test.go | 18 ++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index bdd026ea8..dbb9ed1f7 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -106,7 +106,11 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro for _, vf := range vfs { creatable, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { - return nil, err + // Mirror discoverVFs: one unreadable VF must not blank the host's + // advertised capacity. Skipping only underreports availability, + // the safe direction for status and admission. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + continue } for _, profile := range creatable { profilesByType[profile.TypeName] = profile @@ -411,6 +415,11 @@ func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string] return false, nil } +// openVFIOPaths hard-fails on any unreadable /proc entry, unlike mdev's +// isVFIOGroupInUse which skips them. That is deliberate: this scan authorizes +// clearing current_vgpu_type on a VF path that is reused across assignments, +// so an incomplete scan must fail the release (which retains metadata for a +// later retry) rather than risk a false "not in use" answer. func (s vendorVFIOSysfs) openVFIOPaths() (map[string]struct{}, error) { processes, err := os.ReadDir(s.procPath) if err != nil { @@ -475,6 +484,10 @@ func parseCreatableVGPUTypes(value string) ([]profileMetadata, error) { return profiles, nil } +// framebufferFromProfileName parses the framebuffer size from names like +// "NVIDIA L40S-12Q". NVIDIA's 512 MB 0Q/0B profiles parse as 0 and would read +// as free VRAM in least-loaded placement; no supported GPU exposes them today, +// so placement only skews if such a profile ever appears. func framebufferFromProfileName(name string) int { series := strings.LastIndexAny(name, "ABCQ") if series <= 0 { diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 0879af3e7..50e4d9f00 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -57,6 +57,24 @@ func TestVendorVFIODiscoverFailsWhenNoVFIsReadable(t *testing.T) { require.Error(t, err) } +func TestVendorVFIOListProfilesSkipsUnreadableVF(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + badCreatable := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.5", "nvidia", "creatable_vgpu_types") + require.NoError(t, os.Remove(badCreatable)) + require.NoError(t, os.Mkdir(badCreatable, 0755)) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + require.NotEmpty(t, profiles, "readable VFs must keep advertising capacity") + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-1Q"), + "only the readable VF counts toward availability") +} + func TestVendorVFIOCreateAndDestroy(t *testing.T) { t.Parallel() From d15d3266d899369aa8b7430ae8015c0b0bb91627 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 11 Aug 2026 19:20:06 +0000 Subject: [PATCH 19/19] Skip unreadable VFs in create placement like listProfiles does listProfiles skips an unreadable VF but create still failed placement wholesale when profileMetadata or selectLeastLoadedVF hit the same VF, so /resources could advertise capacity a create then failed to use. Skip the VF in both loops; it simply stops being a placement candidate. --- lib/devices/vendor_vfio_linux.go | 9 +++++++-- lib/devices/vendor_vfio_linux_test.go | 14 ++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go index dbb9ed1f7..566a0d772 100644 --- a/lib/devices/vendor_vfio_linux.go +++ b/lib/devices/vendor_vfio_linux.go @@ -322,7 +322,9 @@ func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType } profiles, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { - return "", err + // An unreadable free VF is just not a placement candidate. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + continue } for _, profile := range profiles { if profile.TypeName == profileType { @@ -356,7 +358,10 @@ func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetada for _, vf := range vfs { profiles, err := s.readCreatableProfiles(vf.PCIAddress) if err != nil { - return nil, err + // Mirror listProfiles: an unreadable VF must not fail placement + // while /resources still advertises the remaining capacity. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + continue } for _, profile := range profiles { profilesByType[profile.TypeName] = profile diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go index 50e4d9f00..4f2e40844 100644 --- a/lib/devices/vendor_vfio_linux_test.go +++ b/lib/devices/vendor_vfio_linux_test.go @@ -75,6 +75,20 @@ func TestVendorVFIOListProfilesSkipsUnreadableVF(t *testing.T) { "only the readable VF counts toward availability") } +func TestVendorVFIOCreateSkipsUnreadableVF(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + badCreatable := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.5", "nvidia", "creatable_vgpu_types") + require.NoError(t, os.Remove(badCreatable)) + require.NoError(t, os.Mkdir(badCreatable, 0755)) + + // Create places on the readable VF instead of failing the placement. + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "inst-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.4", device.VFAddress) +} + func TestVendorVFIOCreateAndDestroy(t *testing.T) { t.Parallel()