diff --git a/cmd/api/api/builds.go b/cmd/api/api/builds.go index ae8e74c27..2f4174b3b 100644 --- a/cmd/api/api/builds.go +++ b/cmd/api/api/builds.go @@ -44,9 +44,15 @@ func (s *ApiService) ListBuilds(ctx context.Context, request oapi.ListBuildsRequ // CreateBuild creates a new build job func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRequestObject) (oapi.CreateBuildResponseObject, error) { if s.Config != nil && s.Config.MacOSOnly { - return oapi.CreateBuild501JSONResponse{Code: "unsupported", Message: "Linux builder VMs are unavailable in macOS-only mode"}, nil + if !s.Config.MacOSBuilds { + return oapi.CreateBuild501JSONResponse{Code: "unsupported", Message: "Linux builder VMs are unavailable in macOS-only mode"}, nil + } + req, source, err := parseMachineBuildMultipart(request.Body) + if err != nil { + return oapi.CreateBuild400JSONResponse{Code: "invalid_request", Message: err.Error()}, nil + } + return s.submitBuild(ctx, req, source) } - log := logger.FromContext(ctx) // Parse multipart form fields var sourceData []byte @@ -69,6 +75,8 @@ func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRe } switch part.FormName() { + case "machine_base_image", "machine_network_mode": + return oapi.CreateBuild400JSONResponse{Code: "unsupported", Message: "machine builds require macos_only and macos_builds"}, nil case "source": sourceData, err = io.ReadAll(part) if err != nil { @@ -256,6 +264,11 @@ func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRe } } + return s.submitBuild(ctx, domainReq, sourceData) +} + +func (s *ApiService) submitBuild(ctx context.Context, domainReq builds.CreateBuildRequest, sourceData []byte) (oapi.CreateBuildResponseObject, error) { + log := logger.FromContext(ctx) build, err := s.BuildManager.CreateBuild(ctx, domainReq, sourceData) if err != nil { switch { diff --git a/cmd/api/api/capabilities.go b/cmd/api/api/capabilities.go index e19ca8035..63c82e31f 100644 --- a/cmd/api/api/capabilities.go +++ b/cmd/api/api/capabilities.go @@ -6,7 +6,6 @@ import ( "sync" "github.com/kernel/hypeman/lib/hypervisor" - "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" @@ -26,6 +25,8 @@ const ( featureLogs = "logs" featureDevices = "devices" featureRosettaEmulation = "rosetta-emulation" + featureMacOSImages = "macos-machine-images" + featureMacOSBuilds = "macos-machine-builds-v1" ) // apiVersion is the API contract version from the embedded OpenAPI document. @@ -133,13 +134,11 @@ func (s *ApiService) GetCapabilities(ctx context.Context, _ oapi.GetCapabilities }, Runtimes: runtimes, Network: *networkCaps, - Images: oapi.CapabilitiesImages{ - Platforms: imagePlatforms(runtime.GOARCH, emulation), - DefaultPlatform: images.HostPlatformString(), - }, + Images: hostImageCapabilities(runtime.GOOS, runtime.GOARCH, emulation, s.Config.MacOSOnly), Features: serverFeatures(runtime.GOOS, emulation), } + resp.Features = append(resp.Features, machineImageFeatures(runtime.GOOS, runtime.GOARCH, s.Config.MacOSBuilds)...) return oapi.GetCapabilities200JSONResponse(resp), nil } @@ -210,6 +209,33 @@ func imagePlatforms(goarch string, emulation bool) []string { return platforms } +func hostImageCapabilities(goos, goarch string, emulation, macOSOnly bool) oapi.CapabilitiesImages { + caps := oapi.CapabilitiesImages{ + Platforms: imagePlatforms(goarch, emulation), + DefaultPlatform: "linux/" + goarch, + } + if goos == "darwin" && goarch == "arm64" { + if macOSOnly { + caps.Platforms = []string{"darwin/arm64"} + caps.DefaultPlatform = "darwin/arm64" + } else { + caps.Platforms = append(caps.Platforms, "darwin/arm64") + } + } + return caps +} + +func machineImageFeatures(goos, goarch string, builds bool) []string { + if goos != "darwin" || goarch != "arm64" { + return nil + } + features := []string{featureMacOSImages} + if builds { + features = append(features, featureMacOSBuilds) + } + return features +} + // serverFeatures builds the server-level feature ID list: API surfaces that // are always present plus host-platform conditionals. func serverFeatures(goos string, emulation bool) []string { diff --git a/cmd/api/api/capabilities_machine_test.go b/cmd/api/api/capabilities_machine_test.go new file mode 100644 index 000000000..705eabf86 --- /dev/null +++ b/cmd/api/api/capabilities_machine_test.go @@ -0,0 +1,34 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestHostMachineImageCapabilities(t *testing.T) { + for _, tc := range []struct { + name, os, arch, defaultPlatform string + emulation, macOSOnly bool + platforms []string + }{ + {"Linux unchanged", "linux", "amd64", "linux/amd64", false, false, []string{"linux/amd64"}}, + {"Linux arm unchanged", "linux", "arm64", "linux/arm64", false, false, []string{"linux/arm64"}}, + {"Mac mixed", "darwin", "arm64", "linux/arm64", false, false, []string{"linux/arm64", "darwin/arm64"}}, + {"Mac mixed Rosetta", "darwin", "arm64", "linux/arm64", true, false, []string{"linux/arm64", "linux/amd64", "darwin/arm64"}}, + {"Mac only", "darwin", "arm64", "darwin/arm64", true, true, []string{"darwin/arm64"}}, + } { + t.Run(tc.name, func(t *testing.T) { + caps := hostImageCapabilities(tc.os, tc.arch, tc.emulation, tc.macOSOnly) + require.Equal(t, tc.defaultPlatform, caps.DefaultPlatform) + require.Equal(t, tc.platforms, caps.Platforms) + }) + } +} + +func TestMachineImageFeatureActivation(t *testing.T) { + require.Empty(t, machineImageFeatures("linux", "arm64", true)) + require.Empty(t, machineImageFeatures("darwin", "amd64", true)) + require.Equal(t, []string{"macos-machine-images"}, machineImageFeatures("darwin", "arm64", false)) + require.Equal(t, []string{"macos-machine-images", "macos-machine-builds-v1"}, machineImageFeatures("darwin", "arm64", true)) +} diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index d5aa47fcf..ced753723 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -63,7 +63,11 @@ func (s *ApiService) ListInstances(ctx context.Context, request oapi.ListInstanc func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInstanceRequestObject) (oapi.CreateInstanceResponseObject, error) { log := logger.FromContext(ctx) - macOSCreate, err := s.isMacOSImage(ctx, request.Body.Image) + platform := derefString(request.Body.Platform) + if s.Config != nil && s.Config.MacOSOnly && platform == "" { + platform = "darwin/arm64" + } + macOSCreate, err := s.isMacOSImage(ctx, request.Body.Image, platform) if err != nil { log.ErrorContext(ctx, "failed to look up image for create", "error", err, "image", request.Body.Image) return oapi.CreateInstance500JSONResponse{Code: "internal_error", Message: "failed to look up image"}, nil @@ -337,7 +341,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Name: request.Body.Name, Image: request.Body.Image, Size: size, - Platform: derefString(request.Body.Platform), + Platform: platform, HotplugSize: hotplugSize, OverlaySize: overlaySize, Vcpus: vcpus, diff --git a/cmd/api/api/macos.go b/cmd/api/api/macos.go index 5dc42cbdb..62e6652ce 100644 --- a/cmd/api/api/macos.go +++ b/cmd/api/api/macos.go @@ -16,11 +16,19 @@ import ( // macOSHost is true where macOS guests can exist. Tests override it. var macOSHost = runtime.GOOS == "darwin" -// isMacOSImage reports whether a create names an imported macOS image. Other -// hosts skip the lookup, so Linux creates are unchanged. An image that is not -// cached yet is not a macOS image; any other lookup error is returned. -func (s *ApiService) isMacOSImage(ctx context.Context, name string) (bool, error) { - if !macOSHost || s.ImageManager == nil { +// isMacOSImage identifies macOS creates before Linux defaults are applied. An +// explicit Darwin platform and macOS-only servers can resolve uncached images. +func (s *ApiService) isMacOSImage(ctx context.Context, name, platform string) (bool, error) { + if !macOSHost { + return false, nil + } + if s.Config != nil && s.Config.MacOSOnly { + return true, nil + } + if p, err := images.ParsePlatform(platform); err == nil && p.OS == "darwin" { + return true, nil + } + if s.ImageManager == nil { return false, nil } img, err := s.ImageManager.GetImage(ctx, name) diff --git a/cmd/api/api/macos_builds.go b/cmd/api/api/macos_builds.go new file mode 100644 index 000000000..9ec28c825 --- /dev/null +++ b/cmd/api/api/macos_builds.go @@ -0,0 +1,125 @@ +package api + +import ( + "fmt" + "io" + "mime/multipart" + "net/http" + "strconv" + + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/images" + mw "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/scopes" +) + +// maxConcurrentMachineBuildUploads bounds how many machine build bodies, each up +// to 65 MiB, are read into memory at once. +const maxConcurrentMachineBuildUploads = 4 + +// LimitMachineBuildBody must wrap the validator, which also consumes request bodies. +// A build upload is authenticated before it takes an upload slot, so unauthenticated +// requests cannot hold the slots. +func LimitMachineBuildBody(jwtSecret string) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + uploads := make(chan struct{}, maxConcurrentMachineBuildUploads) + upload := mw.JwtAuth(jwtSecret)(scopes.RequireScope(scopes.BuildWrite)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case uploads <- struct{}{}: + defer func() { <-uploads }() + default: + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusTooManyRequests) + _, _ = w.Write([]byte(`{"code":"too_many_requests","message":"too many machine build uploads in progress"}`)) + return + } + r.Body = http.MaxBytesReader(w, r.Body, 65<<20) + next.ServeHTTP(w, r) + }))) + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && r.URL.Path == "/builds" { + upload.ServeHTTP(w, r) + return + } + next.ServeHTTP(w, r) + }) + } +} + +// Machine-only HTTP admission is bounded regardless of multipart field order. +// Linux retains its existing parser and source contract. +func parseMachineBuildMultipart(body *multipart.Reader) (builds.CreateBuildRequest, []byte, error) { + req := builds.CreateBuildRequest{BuildPolicy: &builds.BuildPolicy{}} + policy := req.BuildPolicy + numbers := map[string]*int{"timeout_seconds": &policy.TimeoutSeconds, "memory_mb": &policy.MemoryMB, "cpus": &policy.CPUs} + var source []byte + seen := map[string]bool{} + if body == nil { + return req, nil, fmt.Errorf("multipart source is required") + } + for { + part, err := body.NextPart() + if err == io.EOF { + break + } + if err != nil { + return req, nil, fmt.Errorf("invalid multipart form") + } + name := part.FormName() + if seen[name] { + return req, nil, fmt.Errorf("duplicate multipart field") + } + seen[name] = true + limit := int64(64 << 10) + switch name { + case "source": + limit = 64 << 20 + case "machine_base_image", "machine_network_mode", "timeout_seconds", "memory_mb", "cpus", "tags": + default: + return req, nil, fmt.Errorf("unsupported machine build field: %s", name) + } + data, err := io.ReadAll(io.LimitReader(part, limit+1)) + if err != nil || int64(len(data)) > limit { + return req, nil, fmt.Errorf("invalid or oversized %s field", name) + } + if err = part.Close(); err != nil { + return req, nil, fmt.Errorf("invalid multipart field") + } + switch name { + case "source": + source = data + case "machine_base_image": + req.MachineBaseImage = string(data) + case "machine_network_mode": + if string(data) != "isolated" && string(data) != "egress" { + return req, nil, fmt.Errorf("machine_network_mode must be isolated or egress") + } + req.BuildPolicy.NetworkMode = string(data) + case "tags": + req.Tags, err = parseTagsJSON(string(data)) + if err != nil { + return req, nil, fmt.Errorf("tags must be a JSON object of strings") + } + default: + value, err := strconv.Atoi(string(data)) + if err != nil || value < 0 { + return req, nil, fmt.Errorf("%s must be a nonnegative integer", name) + } + *numbers[name] = value + } + } + if len(source) == 0 { + return req, nil, fmt.Errorf("source is required") + } + ref, err := images.ParseNormalizedRef(req.MachineBaseImage) + if err != nil || !ref.IsDigest() || !builds.ValidMachineDigest(ref.Digest()) { + return req, nil, fmt.Errorf("machine_base_image must be pinned by sha256 digest") + } + if req.BuildPolicy.TimeoutSeconds > 86400 { + return req, nil, fmt.Errorf("timeout_seconds exceeds 86400") + } + if err = req.BuildPolicy.Validate(); err != nil { + return req, nil, err + } + return req, source, nil +} diff --git a/cmd/api/api/macos_builds_test.go b/cmd/api/api/macos_builds_test.go new file mode 100644 index 000000000..2fe07942f --- /dev/null +++ b/cmd/api/api/macos_builds_test.go @@ -0,0 +1,226 @@ +package api + +import ( + "bytes" + "context" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/oapi" + "github.com/stretchr/testify/require" +) + +func machineMultipart(t *testing.T, fields ...[2]string) *multipart.Reader { + t.Helper() + var data bytes.Buffer + writer := multipart.NewWriter(&data) + for _, field := range fields { + require.NoError(t, writer.WriteField(field[0], field[1])) + } + require.NoError(t, writer.Close()) + return multipart.NewReader(&data, writer.Boundary()) +} + +type machineZeroReader struct{} + +func (machineZeroReader) Read(p []byte) (int, error) { clear(p); return len(p), nil } + +func TestMachineBuildBodyBoundBeforeValidation(t *testing.T) { + for _, path := range []string{"/builds", "/images"} { + r := authorizedMachineBuildRequest(t, path, io.LimitReader(machineZeroReader{}, (65<<20)+1)) + LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n, err := io.Copy(io.Discard, r.Body) + if path == "/builds" { + require.Error(t, err) + require.EqualValues(t, 65<<20, n) + } else { + require.NoError(t, err) + require.EqualValues(t, (65<<20)+1, n) + } + })).ServeHTTP(httptest.NewRecorder(), r) + } +} + +func TestMachineBuildUploadsAreBounded(t *testing.T) { + release := make(chan struct{}) + entered := make(chan struct{}, maxConcurrentMachineBuildUploads) + handler := LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + entered <- struct{}{} + <-release + })) + done := make(chan struct{}) + for range maxConcurrentMachineBuildUploads { + go func() { + handler.ServeHTTP(httptest.NewRecorder(), authorizedMachineBuildRequest(t, "/builds", nil)) + done <- struct{}{} + }() + <-entered + } + + w := httptest.NewRecorder() + handler.ServeHTTP(w, authorizedMachineBuildRequest(t, "/builds", nil)) + require.Equal(t, http.StatusTooManyRequests, w.Code) + + close(release) + for range maxConcurrentMachineBuildUploads { + <-done + } + w = httptest.NewRecorder() + go func() { <-entered }() + handler.ServeHTTP(w, authorizedMachineBuildRequest(t, "/builds", nil)) + require.Equal(t, http.StatusOK, w.Code, "a slot frees once an upload finishes") +} + +func TestMachineBuildMultipartAdmission(t *testing.T) { + base := "example.test/base@sha256:" + strings.Repeat("a", 64) + valid := [][2]string{{"source", "synthetic"}, {"machine_base_image", base}} + for _, tc := range []struct { + name string + extra [][2]string + ok bool + }{ + {name: "inherit", ok: true}, + {name: "policy", extra: [][2]string{{"cpus", "4"}, {"memory_mb", "8192"}, {"timeout_seconds", "1200"}}, ok: true}, + {name: "isolated network", extra: [][2]string{{"machine_network_mode", "isolated"}}, ok: true}, + {name: "egress network", extra: [][2]string{{"machine_network_mode", "egress"}}, ok: true}, + {name: "invalid network", extra: [][2]string{{"machine_network_mode", "bridge"}}}, + {name: "empty network", extra: [][2]string{{"machine_network_mode", ""}}}, + {name: "duplicate", extra: [][2]string{{"source", "second"}}}, + {name: "linux", extra: [][2]string{{"dockerfile", "FROM scratch"}}}, + {name: "secret", extra: [][2]string{{"secrets", "[]"}}}, + {name: "invalid number", extra: [][2]string{{"cpus", "nan"}}}, + {name: "negative", extra: [][2]string{{"memory_mb", "-1"}}}, + {name: "timeout", extra: [][2]string{{"timeout_seconds", "86401"}}}, + {name: "metadata bound", extra: [][2]string{{"tags", strings.Repeat("x", (64<<10)+1)}}}, + } { + t.Run(tc.name, func(t *testing.T) { + req, source, err := parseMachineBuildMultipart(machineMultipart(t, append(append([][2]string{}, valid...), tc.extra...)...)) + if !tc.ok { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, base, req.MachineBaseImage) + require.Equal(t, []byte("synthetic"), source) + if tc.name == "egress network" { + require.Equal(t, "egress", req.BuildPolicy.NetworkMode) + } + }) + } + _, _, err := parseMachineBuildMultipart(machineMultipart(t, [2]string{"source", "synthetic"}, [2]string{"machine_base_image", "example.test/base:latest"})) + require.Error(t, err) +} + +type machineHTTPManager struct { + builds.Manager + req builds.CreateBuildRequest + calls int +} + +func (m *machineHTTPManager) CreateBuild(_ context.Context, req builds.CreateBuildRequest, source []byte) (*builds.Build, error) { + m.calls++ + m.req = req + return &builds.Build{ID: "synthetic-build", Status: builds.StatusQueued}, nil +} + +func TestMachineBuildHTTPGateAndDispatch(t *testing.T) { + base := "example.test/base@sha256:" + strings.Repeat("a", 64) + for _, tc := range []struct { + name string + cfg *config.Config + accepted bool + }{ + {"disabled", &config.Config{MacOSOnly: true}, false}, + {"linux", &config.Config{}, false}, + {"enabled", &config.Config{MacOSOnly: true, MacOSBuilds: true}, true}, + } { + t.Run(tc.name, func(t *testing.T) { + manager := &machineHTTPManager{} + service := &ApiService{Config: tc.cfg, BuildManager: manager} + response, err := service.CreateBuild(context.Background(), oapi.CreateBuildRequestObject{Body: machineMultipart(t, [2]string{"source", "synthetic"}, [2]string{"machine_base_image", base})}) + require.NoError(t, err) + if tc.accepted { + require.IsType(t, oapi.CreateBuild202JSONResponse{}, response) + require.Equal(t, 1, manager.calls) + require.Equal(t, base, manager.req.MachineBaseImage) + } else { + // Disabled machine builds are unsupported (501) on macOS-only servers, and + // a Linux server rejects the machine selector as a bad request (400). + if tc.cfg.MacOSOnly { + require.IsType(t, oapi.CreateBuild501JSONResponse{}, response) + } else { + require.IsType(t, oapi.CreateBuild400JSONResponse{}, response) + } + require.Zero(t, manager.calls) + } + }) + } +} + +const machineBuildTestSecret = "machine-build-test-secret" + +func signedMachineBuildToken(t *testing.T, claims jwt.MapClaims) string { + t.Helper() + token, err := jwt.NewWithClaims(jwt.SigningMethodHS256, claims).SignedString([]byte(machineBuildTestSecret)) + require.NoError(t, err) + return token +} + +func authorizedMachineBuildRequest(t *testing.T, path string, body io.Reader) *http.Request { + t.Helper() + token := signedMachineBuildToken(t, jwt.MapClaims{ + "sub": "user", + "iat": time.Now().Unix(), + "exp": time.Now().Add(time.Hour).Unix(), + }) + r := httptest.NewRequest(http.MethodPost, path, body) + r.Header.Set("Authorization", "Bearer "+token) + return r +} + +func TestMachineBuildUploadsRequireAuthBeforeTakingASlot(t *testing.T) { + readOnly := signedMachineBuildToken(t, jwt.MapClaims{ + "sub": "user", + "iat": time.Now().Unix(), + "exp": time.Now().Add(time.Hour).Unix(), + "permissions": []string{"build:read"}, + }) + for _, tc := range []struct { + name string + header string + status int + }{ + {"unauthenticated", "", http.StatusUnauthorized}, + {"read-only token", "Bearer " + readOnly, http.StatusForbidden}, + } { + t.Run(tc.name, func(t *testing.T) { + reached := false + handler := LimitMachineBuildBody(machineBuildTestSecret)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + reached = true + })) + for range maxConcurrentMachineBuildUploads + 1 { + r := httptest.NewRequest(http.MethodPost, "/builds", nil) + if tc.header != "" { + r.Header.Set("Authorization", tc.header) + } + w := httptest.NewRecorder() + handler.ServeHTTP(w, r) + require.Equal(t, tc.status, w.Code) + } + require.False(t, reached, "rejected uploads must not reach processing") + + w := httptest.NewRecorder() + handler.ServeHTTP(w, authorizedMachineBuildRequest(t, "/builds", nil)) + require.Equal(t, http.StatusOK, w.Code, "rejected requests hold no upload slot") + require.True(t, reached) + }) + } +} diff --git a/cmd/api/api/macos_test.go b/cmd/api/api/macos_test.go index ce19cce6b..fc19ca81f 100644 --- a/cmd/api/api/macos_test.go +++ b/cmd/api/api/macos_test.go @@ -84,6 +84,38 @@ func TestLinuxCreateDefaultsMatchSchema(t *testing.T) { require.EqualValues(t, linuxDefaultVcpus, props["vcpus"].Value.Default) } +func TestMacOSCreateDefaultsToDarwinPlatform(t *testing.T) { + useMacOSHost(t) + darwin := "darwin/arm64" + for _, tc := range []struct { + name string + cfg *config.Config + images images.Manager + body string + platform *string + }{ + {"macOS-only server", &config.Config{MacOSOnly: true}, nil, `{"name":"macos","image":"registry.example/macos/app:latest"}`, nil}, + {"uncached darwin image", &config.Config{}, missingMacOSImageManager{}, `{"name":"macos","image":"registry.example/macos/app:latest","platform":"darwin/arm64"}`, &darwin}, + } { + t.Run(tc.name, func(t *testing.T) { + manager := newCaptureCreateManager(nil) + service := &ApiService{InstanceManager: manager, ImageManager: tc.images, Config: tc.cfg} + ctx, status := recordCreateBody(t, tc.body) + require.Equal(t, http.StatusOK, status) + size, overlay, vcpus := linuxDefaultSize, linuxDefaultOverlaySize, linuxDefaultVcpus + _, err := service.CreateInstance(ctx, oapi.CreateInstanceRequestObject{Body: &oapi.CreateInstanceRequest{ + Name: "macos", Image: "registry.example/macos/app:latest", Platform: tc.platform, + Size: &size, OverlaySize: &overlay, Vcpus: &vcpus, + }}) + require.NoError(t, err) + require.Equal(t, "darwin/arm64", manager.lastReq.Platform) + require.Zero(t, manager.lastReq.Size) + require.Zero(t, manager.lastReq.OverlaySize) + require.Zero(t, manager.lastReq.Vcpus) + }) + } +} + func TestMacOSCreateIgnoresInjectedLinuxDefaults(t *testing.T) { useMacOSHost(t) m := newCaptureCreateManager(nil) @@ -172,3 +204,9 @@ type macOSImageManager struct{ images.Manager } func (macOSImageManager) GetImage(context.Context, string) (*images.Image, error) { return &images.Image{Platform: "darwin/arm64", MacOS: &images.MacOSImage{}}, nil } + +type missingMacOSImageManager struct{ images.Manager } + +func (missingMacOSImageManager) GetImage(context.Context, string) (*images.Image, error) { + return nil, images.ErrNotFound +} diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 0bffb655b..544a5d494 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -285,7 +285,8 @@ type Config struct { JwtSecret string `koanf:"jwt_secret"` Env string `koanf:"env"` Version string `koanf:"version"` - MacOSOnly bool `koanf:"macos_only"` // Experimental: omit Linux boot downloads and reject Linux creates. + MacOSBuilds bool `koanf:"macos_builds"` // Experimental: enable machine builds on the macOS-only server. + MacOSOnly bool `koanf:"macos_only"` // Experimental: omit Linux boot downloads and reject Linux creates. Network NetworkConfig `koanf:"network"` Caddy CaddyConfig `koanf:"caddy"` @@ -606,6 +607,9 @@ func (c *Config) Validate() error { if err := c.validateAPIListenPorts(); err != nil { return err } + if err := c.validateMacOSBuilds(); err != nil { + return err + } if strings.TrimSpace(c.Metrics.ListenAddress) == "" { return fmt.Errorf("metrics.listen_address must not be empty") } @@ -852,3 +856,21 @@ func (c *Config) validateFirecrackerUFFDGraduation() error { func intPtr(v int) *int { return &v } + +// validateMacOSBuilds checks the experimental machine-build gate. The publisher +// validates the registry URL itself, since the base publication tool shares it. +func (c *Config) validateMacOSBuilds() error { + if !c.MacOSBuilds { + return nil + } + if !c.MacOSOnly { + return fmt.Errorf("macos_builds requires macos_only") + } + if c.Registry.URL == "" { + return fmt.Errorf("macos_builds requires an explicit registry URL") + } + if c.Registry.CACertFile != "" || c.Registry.Insecure { + return fmt.Errorf("macos_builds requires system-trusted registry TLS or loopback HTTP") + } + return nil +} diff --git a/cmd/api/config/macos_builds_test.go b/cmd/api/config/macos_builds_test.go new file mode 100644 index 000000000..6a4cd08d9 --- /dev/null +++ b/cmd/api/config/macos_builds_test.go @@ -0,0 +1,26 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMacOSBuildsFailClosedConfiguration(t *testing.T) { + require.False(t, (&Config{}).MacOSBuilds) + c := *defaultConfig() + c.MacOSBuilds = true + require.ErrorContains(t, c.validateMacOSBuilds(), "requires macos_only") + c.MacOSOnly = true + c.Hypervisor.Default = "vz" + c.Registry.URL = "" + require.ErrorContains(t, c.validateMacOSBuilds(), "explicit registry URL") + c.Registry.URL = "http://localhost:4973" + c.Registry.CACertFile = "custom.pem" + require.ErrorContains(t, c.validateMacOSBuilds(), "system-trusted registry TLS") + c.Registry.CACertFile = "" + c.Registry.Insecure = true + require.ErrorContains(t, c.validateMacOSBuilds(), "system-trusted registry TLS") + c.Registry.Insecure = false + require.NoError(t, c.validateMacOSBuilds()) +} diff --git a/cmd/api/main.go b/cmd/api/main.go index 347b395ff..1c8241323 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -535,6 +535,11 @@ func run() error { r.Use(api.RecordCreateFields) } + // Bound machine multipart input before OpenAPI validation reads the body. + if app.Config.MacOSBuilds { + r.Use(api.LimitMachineBuildBody(app.Config.JwtSecret)) + } + // OpenAPI request validation with authentication validatorOptions := &nethttpmiddleware.Options{ Options: openapi3filter.Options{ @@ -632,10 +637,10 @@ func run() error { logger.Error("failed to start builders manager", "error", err) return err } - if err := app.BuildManager.Start(gctx); err != nil { - logger.Error("failed to start build manager", "error", err) - return err - } + } + if err := app.BuildManager.Start(gctx); err != nil { + logger.Error("failed to start build manager", "error", err) + return err } grp.Go(func() error { diff --git a/cmd/vz-shim/macos_vm.go b/cmd/vz-shim/macos_vm.go index 269e9af82..4241f57dd 100644 --- a/cmd/vz-shim/macos_vm.go +++ b/cmd/vz-shim/macos_vm.go @@ -45,6 +45,9 @@ func createMacVM(config *shimconfig.ShimConfig) (*vz.VirtualMachine, *vz.Virtual if err := configureMacPlatform(vmConfig, config); err != nil { return nil, nil, fmt.Errorf("configure Mac platform: %w", err) } + if err := configureMacAudio(vmConfig); err != nil { + return nil, nil, fmt.Errorf("configure audio: %w", err) + } vsockConfig, err := vz.NewVirtioSocketDeviceConfiguration() if err != nil { @@ -65,3 +68,26 @@ func createMacVM(config *shimconfig.ShimConfig) (*vz.VirtualMachine, *vz.Virtual } return vm, vmConfig, nil } + +// configureMacAudio gives the guest an audio device. Virtualization attaches audio +// only through a sound device, so without one the guest boots with no CoreAudio +// devices at all: `system_profiler SPAudioDataType` lists none and `afplay` fails +// with `AudioQueueStart failed (-66680)`. +// +// The output stream has no host sink. The samples are discarded on the host, which +// is what a headless browser VM wants -- nothing should play on the machine running +// the guests -- while the guest still has a real output device, so applications can +// render audio and ScreenCaptureKit has a mix to capture for the live view. +func configureMacAudio(vmConfig *vz.VirtualMachineConfiguration) error { + soundConfig, err := vz.NewVirtioSoundDeviceConfiguration() + if err != nil { + return fmt.Errorf("create sound device: %w", err) + } + output, err := vz.NewVirtioSoundDeviceOutputStreamConfiguration() + if err != nil { + return fmt.Errorf("create output stream: %w", err) + } + soundConfig.SetStreams(output) + vmConfig.SetAudioDevicesVirtualMachineConfiguration([]vz.AudioDeviceConfiguration{soundConfig}) + return nil +} diff --git a/config.example.darwin.yaml b/config.example.darwin.yaml index e7745fb1a..d838b9887 100644 --- a/config.example.darwin.yaml +++ b/config.example.darwin.yaml @@ -27,6 +27,14 @@ data_dir: ~/Library/Application Support/hypeman # Server configuration port: "4973" +# Native macOS guests/builds are opt-in; these defaults retain Linux guests on VZ. +macos_only: false +macos_builds: false +# For a dedicated native Mac builder, set both true after the release gates in +# docs/macos-builds.md pass, and replace registry.url below with an explicit +# system-trusted HTTPS address reachable by the build and destination hosts. +# Machine publication runs on the host, not at the Linux guest NAT gateway. + # ============================================================================= # Hypervisor Configuration (IMPORTANT FOR MACOS) # ============================================================================= diff --git a/docs/macos-builds.md b/docs/macos-builds.md new file mode 100644 index 000000000..c5e488cee --- /dev/null +++ b/docs/macos-builds.md @@ -0,0 +1,126 @@ +# macOS image builds (experimental, opt-in) + +A build job can provision a digest-pinned installed macOS base in an isolated VZ +VM, run the source's `build.sh` in it, and publish the result as an OCI machine +image. This is not macOS installation, Linux rootfs conversion or memory forking. +It depends on the shared system GuestService. + +The normal build manager owns the queue, persisted request, deadline, source +staging, status, provenance and the image-readiness gate. `GuestMachineBuildDriver` +runs the guest and `StreamingMachinePublisher` writes the registry. + +## Enabling + +Off by default. Set `macos_only: true` and `macos_builds: true` on an Apple-silicon +VZ server, with `registry.url` set to an address the host can reach (not a guest +gateway), including its scheme: `http://localhost:4973` for loopback, or a +system-trusted `https://` endpoint. Credentials, paths, queries, fragments, custom +CAs and insecure HTTPS are rejected. Linux servers reject the machine selector. + +```sh +curl -H "Authorization: Bearer $TOKEN" \ + -F "machine_base_image=registry.example/base@sha256:$BASE_DIGEST" \ + -F "source=@source.tar.gz;type=application/gzip" \ + -F "timeout_seconds=1200" "$API/builds" +``` + +The base must already be Ready. A 202 queues an ordinary job; status, cancellation +and provenance use the shared `/builds` APIs. Only `source`, `machine_base_image`, +`machine_network_mode`, `timeout_seconds`, `memory_mb`, `cpus` and `tags` are +accepted. Duplicate or unknown fields, malformed numbers and mutable bases return +400. Limits: 64 KiB per metadata field, 64 MiB source, 65 MiB total body. Linux +source parsing is unchanged. + +## Using the result + +`GET /builds/{id}` returns a digest-pinned `image_ref` for instance create. Other +Mac hosts need `registry.url` reachable from them; loopback only works on the build +host. A host that hasn't cached the image pulls on create, which can exceed the +readiness wait for large images, so retry or pre-pull. Distribute by digest with +`POST /pushes`. Consumers should pin the digest, not a tag. + +## Base and policy + +The base must be pinned by canonical SHA256, be `darwin/arm64`, and declare a +provisioned system agent on port 2222. The driver also checks guest UID 0 and never +installs an agent or falls back to host SSH. CPU and memory match the base exactly; +zero request values inherit them. Timeout defaults to 600 s, capped at 24 h. + +Networking defaults to isolated. `machine_network_mode=egress` gives unrestricted +VZ NAT for checksum-verified downloads; it is not a domain allowlist. Linux builder, +cache, build-arg, secret and admin options are rejected. + +Each builder gets its own machine identifier and MAC, and its SSH host keys are +rekeyed on first start, the same way a cold clone is. The output keeps that identity. +Builds on one base run concurrently, and an output can run beside its base. + +## Source + +The source tar.gz is staged privately and SHA256-verified, including on recovery, +and `provenance.source_hash` pins exactly what ran. Input is capped at 64 MiB +compressed, 128 MiB uncompressed, 1024 entries and 64 MiB per file. Only regular +files and directories are accepted; links, devices, duplicates and path escapes fail +before transfer. The archive must contain a non-empty `build.sh` at its root. It is +copied to `/var/root/hypeman-build-{id}` in the guest and run as root with +`/bin/sh build.sh`. + +Before stopping, the guest's build directory is removed and its absence verified, +and the host copy is removed. No registry token, host credential or secret is +injected. Guest stdout and stderr are discarded, so build logs never contain guest +output. Failures report the phase and, for the script, its exit code. + +## Stop, export and cleanup + +Publication is two instance operations. `StopMacOSBuildInstance` does a strict +graceful stop and rejects forced fallback, an instance already Stopped, non-macOS +guests, and any stop without shutdown acknowledgement and confirmed VMM exit. +`ExportStoppedMacOSBuildInstance` then copies the closed storage of a still-Stopped +instance under the instance lock, so a concurrent Start makes export fail. The output +is a private 0700 directory holding only `disk.img`, `aux.img` and `config.json`. + +Failure cleanup uses a forced stop and exports nothing. Storage is deleted only after +confirmed VMM exit; a VM with no confirmed exit stays quarantined for an operator. +Cleanup has its own 2-minute budget. A re-enqueued attempt first reaps the builder +left by the interrupted attempt, and fails rather than start a second one if that +delete fails. Builders orphaned outside this path have no general reaper. + +The builder is destroyed **before** publication. Source stays staged until a terminal +status is persisted, so a job interrupted while pushing is rebuilt on recovery. +Startup on a macOS-only server recovers machine jobs without preparing Linux builder +state, reaps cancelled builders and removes terminal-job source. Cancellation +interrupts the running job and keeps Cancelled terminal. Guest errors that may carry +secrets are masked in strings and logs. + +## Publication + +The publisher uses a server-configured registry and a token scoped to `builds/{id}`. +Plaintext is allowed only on loopback; redirects are never followed and proxies are +disabled. The layer is never written to disk: a first pass computes its digest, size +and DiffID, the upload regenerates the same bytes, and the registry rejects any +mismatch. Blobs use monolithic POST uploads, every persisted blob is re-read and +hashed, the manifest is committed last, and the digest content is verified. Ambiguous +errors never return success. + +A derivative built from a base in the same registry publishes a delta: the base's +layers are mounted by digest and only a layer of changed 64 KiB disk chunks is +uploaded. See `lib/images/macos_delta.go`. Bases outside the registry publish +standalone. There is no garbage-collection rule yet; deleting a base that a derivative +references breaks a later re-pull of that derivative. + +## Status and gates + +Race tests cover admission, the shared queue, cancellation without publication, phase +failures and quarantine, export validation, and authenticated publication with +manifest-last ordering. Test disks are synthetic and prove nothing about bootability. +A development base has been run end to end through the HTTP API on Apple silicon, +including a cold boot of the result; this trimmed branch has not been re-run natively. + +Before production use: a reviewed fresh base with approved artifact pins (base +publication tooling is a separate PR), cross-host publish and pull with sequential +cold boot, disk-space monitoring (this change has no disk admission, so run machine +builds serially on a dedicated host), SDK and CLI release, and Apple licence review. +Keep `macos_builds` off until then. + +**Rollback:** stop new submissions, let or cancel running jobs through the normal +APIs, repoint workloads to the prior validated digest, then set `macos_builds: false`. +Disabling does not delete persisted jobs or images. diff --git a/docs/macos-support.md b/docs/macos-support.md index 2ee42b466..13afe299e 100644 --- a/docs/macos-support.md +++ b/docs/macos-support.md @@ -14,7 +14,7 @@ owns the behavior. | Stats | Partial | CPU and memory from the VM process. Network counters are 0 under NAT. | PR 1 | | Memory reclaim | Unsupported | Reported per guest as `unsupported`. No balloon change. | PR 1 | | Guest port forwarding | Agent-gated | HTTP and WebSocket to a guest loopback port at `/instances/{id}/ports/{port}/*`. The image starts its own services. See [macos-guest-ports.md](macos-guest-ports.md). | PR 5 | -| Builds | Partial | Opt-in `macos_builds`. Restricted recipe. Linux Dockerfile builds unsupported. | PR 6 | +| Builds | Partial | Opt-in `macos_builds`. Runs the source's build.sh as root in the guest. Linux Dockerfile builds unsupported. | PR 6 | | Stopped snapshots, same-instance restore | Experimental | Graceful stop required. Stopped and Running targets; restored metadata and storage are staged together. | PR 4 | | Standby and restore (same instance) | Experimental | Saves memory, restores the same instance. Restore can fail (about 2% observed, cause unknown); a failed restore leaves the instance in Standby. Recover by deleting, as on Linux. Standby auto-policies rejected: no host-visible connection tracking on NAT. | PR 4 | | Same-instance memory snapshots | Experimental | Running/Standby capture, with exact VZ configuration validation and an uncompressed disk/aux/memory bundle. Same-instance restore only; new path needs native QA. | PR 4 | diff --git a/lib/builds/macos_build.go b/lib/builds/macos_build.go new file mode 100644 index 000000000..d58a55d27 --- /dev/null +++ b/lib/builds/macos_build.go @@ -0,0 +1,305 @@ +package builds + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" +) + +// MachineBuildRequest is the internal backend contract for one machine build. +// The normal build manager owns ID, source staging and publish authorization. A concrete driver must require a provisioned system agent. +type MachineBuildRequest struct { + ID string + BaseImage string // Installed darwin/arm64 image, pinned by sha256 digest. + SourceHash string // Optional expected lowercase SHA256 of the source archive. + Policy BuildPolicy // Zero CPU/memory inherit the base machine's exact resources. + onStarted func(string) error // Normal manager records the owned builder before provisioning. + onPublished func(context.Context, MachinePublication) error // Hold storage until the shared image pipeline is terminal. + deltaBase *MachineDeltaBase // Set when the base is in the publisher's registry; publishes a derivative. +} + +type MachineStopReceipt = instances.MacOSBuildExportReceipt +type MachinePublication struct{ Reference, Digest string } +type MachineBuildResult struct { + Publication MachinePublication + Provenance BuildProvenance +} + +type MachineBuildDriver interface { + // Start owns an isolated instance and identity lease, named name. A partially + // created session must be returned even with an error so cleanup can stop it. + Start(ctx context.Context, name string, base *images.Image, policy BuildPolicy) (MachineBuildSession, error) +} + +type MachineBuildSession interface { + ID() string + Provision(context.Context, string) error // Build-owned staged source, never a client host path. + Sanitize(context.Context) error // Remove build credentials, source and secret injection artifacts. + // Stop must await actual VMM exit. Only a strict graceful stop may produce a + // receipt that permits export. With force, a failed strict stop falls back to + // a forced stop, which is failure-only cleanup: nothing is exported after it. + Stop(ctx context.Context, force bool) (MachineStopReceipt, error) + Export(context.Context, string) error // Stopped disk.img, aux.img, config.json only. + // Destroy removes instance storage only after proven VMM exit, not the export. + Destroy(context.Context) error +} + +type MachineBuildPublisher interface { + // Publisher owns registry address, scoped token and builds/{id} destination. + // It must stream blobs and commit/verify the manifest last. An ambiguous + // network error is not a verified successful publication. + Publish(context.Context, string, string) (MachinePublication, error) +} + +type MachineBuildBackend struct { + Driver MachineBuildDriver + Publisher MachineBuildPublisher +} + +// machinePhaseError keeps potentially secret-bearing guest/command errors out of +// ordinary log/error text, while retaining errors.Is/As for internal handling. +type machinePhaseError struct { + phase string + cause error +} + +func (e *machinePhaseError) Error() string { + if exit := (*machineExitError)(nil); errors.As(e.cause, &exit) { + return "machine build " + e.phase + " failed: " + exit.Error() + } + // Host admission errors, such as a template identity already in use, carry + // no guest output. + if errors.Is(e.cause, instances.ErrInsufficientResources) || errors.Is(e.cause, instances.ErrInvalidState) { + return "machine build " + e.phase + " failed: " + e.cause.Error() + } + return "machine build " + e.phase + " failed" +} +func (e *machinePhaseError) Unwrap() error { return e.cause } +func machineFailure(phase string, err error) error { return &machinePhaseError{phase, err} } + +// machineExitError is the entrypoint's nonzero exit code. It holds no guest +// output, so it is safe to show. +type machineExitError struct{ code int } + +func (e *machineExitError) Error() string { + return fmt.Sprintf("%s exited with code %d", machineEntrypoint, e.code) +} + +// normalizeMachinePolicy applies machine build policy defaults and rejects what +// machine builds do not support. Admission, the runner and the guest driver share it. +func normalizeMachinePolicy(policy *BuildPolicy) error { + if policy.TimeoutSeconds == 0 { + policy.TimeoutSeconds = 600 + } + if policy.TimeoutSeconds < 1 || policy.TimeoutSeconds > 86400 { + return fmt.Errorf("invalid machine build timeout") + } + if policy.NetworkMode == "" { + policy.NetworkMode = "isolated" + } + if len(policy.AllowedDomains) != 0 || (policy.NetworkMode != "isolated" && policy.NetworkMode != "egress") { + return fmt.Errorf("unsupported machine network policy") + } + return nil +} + +// runPrepared executes only machine-specific phases. The shared build manager +// owns resolution, source staging/hash verification, deadline, queue and status. +func (r *MachineBuildBackend) runPrepared(ctx context.Context, req MachineBuildRequest, base *images.Image, sourcePath, workDir string) (result *MachineBuildResult, err error) { + if r.Driver == nil || r.Publisher == nil || !filepath.IsAbs(workDir) || !filepath.IsAbs(sourcePath) || len(req.SourceHash) != 64 || strings.Trim(req.SourceHash, "0123456789abcdef") != "" { + return nil, fmt.Errorf("machine driver, publisher, private workspace and verified source required") + } + if _, bounded := ctx.Deadline(); !bounded { + return nil, fmt.Errorf("prepared machine job requires a deadline") + } + if len(req.ID) == 0 || len(req.ID) > 128 || strings.Trim(req.ID, "abcdefghijklmnopqrstuvwxyz0123456789-") != "" { + return nil, fmt.Errorf("invalid machine build ID") + } + ref, e := images.ParseNormalizedRef(req.BaseImage) + if e != nil || !ref.IsDigest() || !ValidMachineDigest(ref.Digest()) { + return nil, fmt.Errorf("machine base must be pinned by sha256 digest") + } + policy := req.Policy + if e = normalizeMachinePolicy(&policy); e != nil { + return nil, e + } + if base == nil || base.Status != images.StatusReady || base.Platform != "darwin/arm64" || base.MacOS == nil || !base.MacOS.GuestAgent || base.Digest != ref.Digest() || base.MacOS.CPUs < 2 || base.MacOS.CPUs > MaxBuildCPUs || base.MacOS.Memory < 4<<30 || base.MacOS.Memory > uint64(MaxBuildMemoryMB)<<20 || base.MacOS.Memory%(1<<20) != 0 { + return nil, fmt.Errorf("base is not the pinned ready macOS machine") + } + if base.MacOS.Validate() != nil { + return nil, fmt.Errorf("base has invalid machine identity") + } + memory := int(base.MacOS.Memory >> 20) + cpus := int(base.MacOS.CPUs) + if (policy.MemoryMB != 0 && policy.MemoryMB != memory) || (policy.CPUs != 0 && policy.CPUs != cpus) { + return nil, fmt.Errorf("machine resources must match installed base") + } + policy.MemoryMB = memory + policy.CPUs = cpus + if e = policy.Validate(); e != nil { + return nil, e + } + workspace, e := os.MkdirTemp(workDir, "machine-build-") + if e != nil { + return nil, machineFailure("workspace", e) + } + defer func() { + if cleanupErr := os.RemoveAll(workspace); cleanupErr != nil { + err = errors.Join(err, machineFailure("cleanup_workspace", cleanupErr)) + result = nil + } + }() + if e = verifyBuildSource(ctx, sourcePath, req.SourceHash); e != nil { + return nil, machineFailure("verify_source", e) + } + if e = ctx.Err(); e != nil { + return nil, e + } + session, e := r.Driver.Start(ctx, machineBuilderName(req.ID), base, policy) + if session == nil { + if e == nil { + e = fmt.Errorf("driver returned no session") + } + return nil, machineFailure("start", e) + } + stopped, destroyed := false, false + defer func() { + cleanup, cancel := context.WithTimeout(context.WithoutCancel(ctx), machineCleanupTimeout) + defer cancel() + if !stopped { + receipt, stopErr := session.Stop(cleanup, true) + stopped = receipt.VMMExited + if !stopped { + err = errors.Join(err, machineFailure("quarantine_unconfirmed_vmm_exit", stopErr)) + result = nil + return + } + } + if !destroyed { + if destroyErr := session.Destroy(cleanup); destroyErr != nil { + err = errors.Join(err, machineFailure("cleanup", destroyErr)) + result = nil + } + } + }() + instanceID := session.ID() + if req.onStarted != nil { + if recordErr := req.onStarted(instanceID); recordErr != nil { + return nil, machineFailure("record_builder", recordErr) + } + } + if e != nil { + return nil, machineFailure("start", e) + } + if e = session.Provision(ctx, sourcePath); e != nil { + return nil, machineFailure("provision", e) + } + if e = session.Sanitize(ctx); e != nil { + return nil, machineFailure("sanitize", e) + } + receipt, e := session.Stop(ctx, false) + stopped = receipt.VMMExited + if e != nil || !receipt.Graceful || !stopped { + return nil, machineFailure("graceful_stop", e) + } + bundle := filepath.Join(workspace, "bundle") + if e = os.Mkdir(bundle, 0700); e != nil { + return nil, machineFailure("export", e) + } + if e = session.Export(ctx, bundle); e != nil { + return nil, machineFailure("export", e) + } + if e = validateMachineExport(bundle, base.MacOS); e != nil { + return nil, machineFailure("validate_export", e) + } + if e = session.Destroy(ctx); e != nil { + return nil, machineFailure("cleanup", e) + } + destroyed = true + if e = ctx.Err(); e != nil { + return nil, e + } + publication, e := r.publishBuild(ctx, req, bundle) + if e != nil { + return nil, machineFailure("publish", e) + } + output, e := images.ParseNormalizedRef(publication.Reference) + if e != nil || !output.IsDigest() || !ValidMachineDigest(publication.Digest) || output.Digest() != publication.Digest { + return nil, fmt.Errorf("machine publication receipt is unverified") + } + if req.onPublished != nil { + if e = req.onPublished(ctx, publication); e != nil { + return nil, machineFailure("image_ready", e) + } + } + provenance := BuildProvenance{BaseImageDigest: base.Digest, SourceHash: req.SourceHash, Timestamp: time.Now().UTC()} + return &MachineBuildResult{Publication: publication, Provenance: provenance}, nil +} + +// machineCleanupTimeout covers a strict graceful stop attempt (30s power-off +// budget) plus forced fallback and storage deletion. Export is outside cleanup. +const machineCleanupTimeout = 2 * time.Minute + +// ValidMachineDigest reports whether s is a canonical lowercase sha256 digest. +// It is the single validation rule for machine base and output digests. +func ValidMachineDigest(s string) bool { + if !strings.HasPrefix(s, "sha256:") || len(s) != 71 { + return false + } + return strings.Trim(s[7:], "0123456789abcdef") == "" +} + +func validateMachineExport(root string, base *images.MacOSImage) error { + directory, err := os.Lstat(root) + if err != nil || !directory.IsDir() { + return fmt.Errorf("export root must be a real directory") + } + entries, err := os.ReadDir(root) + if err != nil { + return err + } + if len(entries) != 3 { + return fmt.Errorf("export must contain only machine payload files") + } + for _, entry := range entries { + if entry.Name() != "disk.img" && entry.Name() != "aux.img" && entry.Name() != "config.json" { + return fmt.Errorf("unexpected export file") + } + info, err := os.Lstat(filepath.Join(root, entry.Name())) + if err != nil || !info.Mode().IsRegular() { + return fmt.Errorf("export files must be regular") + } + } + platform, err := images.ValidateMacOSBundle(root) + if err != nil { + return err + } + if base != nil { + if !bytes.Equal(platform.HardwareModel, base.HardwareModel) || platform.CPUs != base.CPUs || platform.Memory != base.Memory { + return fmt.Errorf("machine export changed hardware model or resources") + } + if bytes.Equal(platform.MachineIdentifier, base.MachineIdentifier) || strings.EqualFold(platform.MAC, base.MAC) { + return fmt.Errorf("machine export kept the base identity") + } + } + for _, entry := range entries { + if err = os.Chmod(filepath.Join(root, entry.Name()), 0600); err != nil { + return err + } + } + return nil +} + +// machineBuilderName names a build's builder instance after the build, so a retry +// can find and reap a builder that an interrupted attempt created but never recorded. +func machineBuilderName(buildID string) string { + return "macos-build-" + strings.ToLower(buildID) +} diff --git a/lib/builds/macos_build_cancel.go b/lib/builds/macos_build_cancel.go new file mode 100644 index 000000000..09b984000 --- /dev/null +++ b/lib/builds/macos_build_cancel.go @@ -0,0 +1,70 @@ +package builds + +import ( + "context" + "fmt" + "sync" + "time" +) + +type machineBuildControl struct { + mu sync.Mutex + cancel context.CancelFunc +} + +func (m *manager) recordMachineBuilder(id, instanceID string) error { + if active, ok := m.machineBuilds.Load(id); ok { + control := active.(*machineBuildControl) + control.mu.Lock() + defer control.mu.Unlock() + } + meta, err := readMetadata(m.paths, id) + if err != nil { + return err + } + if meta.Status == StatusCancelled { + return context.Canceled + } + meta.BuilderInstance = &instanceID + return writeMetadata(m.paths, meta) +} + +func (m *manager) cancelQueuedMachineBuild(meta *buildMetadata) error { + meta.Status = StatusCancelled + now := time.Now() + meta.CompletedAt = &now + if err := writeMetadata(m.paths, meta); err != nil { + return err + } + m.removeMachineSource(meta.ID) + m.notifyStatusChange(meta.ID, StatusCancelled) + return nil +} + +func (m *manager) cancelMachineBuild(id string) error { + active, ok := m.machineBuilds.Load(id) + if !ok { + return ErrBuildInProgress + } + control := active.(*machineBuildControl) + control.mu.Lock() + defer control.mu.Unlock() + meta, err := readMetadata(m.paths, id) + if err != nil { + return err + } + if meta.Status != StatusBuilding && meta.Status != StatusPushing { + return fmt.Errorf("machine build no longer running") + } + // Persist cancellation before interrupting phases. The backend performs bounded + // graceful cleanup or quarantine; Cancel must not force-delete mutable storage. + meta.Status = StatusCancelled + now := time.Now() + meta.CompletedAt = &now + if err := writeMetadata(m.paths, meta); err != nil { + return err + } + control.cancel() + m.notifyStatusChange(id, StatusCancelled) + return nil +} diff --git a/lib/builds/macos_build_failure_test.go b/lib/builds/macos_build_failure_test.go new file mode 100644 index 000000000..632bd7ee7 --- /dev/null +++ b/lib/builds/macos_build_failure_test.go @@ -0,0 +1,198 @@ +package builds + +import ( + "context" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func runMachineBuildFailure(t *testing.T, f *machineFixture, ctx context.Context, req MachineBuildRequest, source io.Reader) (*machineRunnerFixture, error) { + t.Helper() + runner := machineTestRunner(t, f) + result, err := runner.Run(ctx, req, source) + if err == nil || result != nil { + t.Fatal("failed build returned a result") + } + return runner, err +} + +type failingMachineSource struct{} + +func (failingMachineSource) Read(p []byte) (int, error) { + return copy(p, "partial"), errors.New("secret-source-error") +} + +type cancelMachineSource struct{ cancel context.CancelFunc } + +func (s cancelMachineSource) Read(p []byte) (int, error) { + s.cancel() + return copy(p, "source"), io.EOF +} + +func TestMachineBuildSourceFailureDoesNotStart(t *testing.T) { + for _, tc := range []struct { + name string + source func(context.CancelFunc) io.Reader + wantCancel bool + }{ + {name: "read failure", source: func(context.CancelFunc) io.Reader { return failingMachineSource{} }}, + {name: "canceled", source: func(cancel context.CancelFunc) io.Reader { return cancelMachineSource{cancel} }, wantCancel: true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + f := newMachineFixture() + r, err := runMachineBuildFailure(t, f, ctx, machineTestRequest(), tc.source(cancel)) + if tc.wantCancel { + require.ErrorIs(t, err, context.Canceled) + } else { + require.NotContains(t, err.Error(), "secret-source-error", "source failure was not masked") + } + require.Empty(t, f.calls, "source failure started guest") + entries, err := os.ReadDir(r.WorkDir) + require.NoError(t, err) + require.Empty(t, entries, "partial source retained") + }) + } +} + +// Cancellation and timeout both interrupt a running build. Cleanup must still +// run to completion on a live context, and nothing may publish. +func TestMachineBuildInterruptedCleanup(t *testing.T) { + assertCleanup := func(t *testing.T, f *machineFixture, err, want error) { + t.Helper() + require.ErrorIs(t, err, want) + require.False(t, f.publisherCalled, "interrupted build published") + require.True(t, f.cleanupContextLive, "cleanup ran on a cancelled context") + require.Equal(t, []string{"start", "abort", "destroy"}, f.calls) + } + t.Run("cancellation", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + f := newMachineFixture() + f.cancel = cancel + _, err := runMachineBuildFailure(t, f, ctx, machineTestRequest(), strings.NewReader("source")) + assertCleanup(t, f, err, context.Canceled) + }) + t.Run("timeout", func(t *testing.T) { + f := newMachineFixture() + f.waitForCancel = true + req := machineTestRequest() + req.Policy.TimeoutSeconds = 1 + parent, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + _, err := runMachineBuildFailure(t, f, parent, req, strings.NewReader("source")) + assertCleanup(t, f, err, context.DeadlineExceeded) + }) +} + +func TestMachineBuildPublicationReceipt(t *testing.T) { + for _, kind := range []string{"ambiguous error", "tag receipt", "digest mismatch", "invalid digest"} { + t.Run(kind, func(t *testing.T) { + f := newMachineFixture() + switch kind { + case "ambiguous error": + f.fail = "publish" + case "tag receipt": + f.publicationOverride = &MachinePublication{Reference: "localhost/builds/build-test:latest", Digest: machineTestDigest} + case "digest mismatch": + f.publicationOverride = &MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: "sha256:" + strings.Repeat("b", 64)} + case "invalid digest": + f.publicationOverride = &MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: "not-a-digest"} + } + _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) + if !f.publisherCalled { + t.Fatal("unverified receipt became ready") + } + if strings.Contains(err.Error(), "sensitive-command-argument") { + t.Fatal("publisher error leaked") + } + count := 0 + for _, phase := range f.calls { + if phase == "publish" { + count++ + } + } + if count != 1 { + t.Fatal("ambiguous publication was retried") + } + }) + } +} + +func TestMachineBuildMalformedExport(t *testing.T) { + for _, kind := range []string{"empty disk", "empty aux", "symlink disk", "hardlink payloads", "truncated config", "oversized config", "resource change", "bad MAC", "identity", "extra file", "unknown metadata"} { + t.Run(kind, func(t *testing.T) { + f := newMachineFixture() + f.mutateExport = func(root string) error { + disk, aux, config := filepath.Join(root, "disk.img"), filepath.Join(root, "aux.img"), filepath.Join(root, "config.json") + switch kind { + case "empty disk": + return os.Truncate(disk, 0) + case "empty aux": + return os.Truncate(aux, 0) + case "symlink disk": + if err := os.Remove(disk); err != nil { + return err + } + return os.Symlink(aux, disk) + case "hardlink payloads": + if err := os.Remove(aux); err != nil { + return err + } + return os.Link(disk, aux) + case "truncated config": + return os.WriteFile(config, []byte(`{"hardware_model":`), 0600) + case "oversized config": + return os.WriteFile(config, []byte(strings.Repeat(" ", 65<<10)), 0600) + case "resource change": + data, err := os.ReadFile(config) + if err != nil { + return err + } + return os.WriteFile(config, []byte(strings.Replace(string(data), `"cpus":4`, `"cpus":2`, 1)), 0600) + case "bad MAC": + data, err := os.ReadFile(config) + if err != nil { + return err + } + return os.WriteFile(config, []byte(strings.Replace(string(data), "02:00:00:00:00:02", "02:00:00:00:00:02:03:04", 1)), 0600) + case "identity", "unknown metadata": + var raw map[string]any + data, err := os.ReadFile(config) + if err == nil { + err = json.Unmarshal(data, &raw) + } + if err != nil { + return err + } + if kind == "identity" { + raw["machine_identifier"] = f.image.MacOS.MachineIdentifier // kept the base identity + } else { + raw["password"] = "must-not-publish" + } + data, _ = json.Marshal(raw) + return os.WriteFile(config, data, 0600) + case "extra file": + return os.WriteFile(filepath.Join(root, "secret.txt"), []byte("must-not-publish"), 0600) + } + return nil + } + _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) + if f.publisherCalled { + t.Fatal("malformed export published") + } + if strings.Contains(err.Error(), "must-not-publish") { + t.Fatal("export secret leaked in failure") + } + }) + } +} diff --git a/lib/builds/macos_build_guest.go b/lib/builds/macos_build_guest.go new file mode 100644 index 000000000..3794b92ab --- /dev/null +++ b/lib/builds/macos_build_guest.go @@ -0,0 +1,186 @@ +package builds + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "time" + + "github.com/kernel/hypeman/lib/guest" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" +) + +type machineExportManager interface { + StopMacOSBuildInstance(context.Context, string) (instances.MacOSBuildExportReceipt, error) + ExportStoppedMacOSBuildInstance(context.Context, string, string) error +} + +// GuestMachineBuildDriver uses ordinary instance lifecycle and the shared system +// GuestService. It never installs an agent or silently upgrades an unmanaged base. +// The backend validates the base and policy before Start. The builder gets its own +// machine identifier and MAC, so it and its output can run beside the base. +type GuestMachineBuildDriver struct { + Instances instances.Manager + WorkDir string +} + +func (d *GuestMachineBuildDriver) Start(ctx context.Context, name string, base *images.Image, policy BuildPolicy) (MachineBuildSession, error) { + exporter, ok := d.Instances.(machineExportManager) + if !ok || !filepath.IsAbs(d.WorkDir) { + return nil, fmt.Errorf("strict machine export and private workspace required") + } + if _, ok = ctx.Deadline(); !ok { + return nil, fmt.Errorf("machine start requires a deadline") + } + ref, err := images.ParseNormalizedRef(base.Name) + if err != nil { + return nil, err + } + // Workspaces are named per builder, so one left by an interrupted attempt + // of this job is removed here. + prefix := "guest-machine-" + name + "-" + stale, _ := filepath.Glob(filepath.Join(d.WorkDir, prefix+"*")) + for _, dir := range stale { + _ = os.RemoveAll(dir) + } + dir, err := os.MkdirTemp(d.WorkDir, prefix) + if err != nil { + return nil, err + } + inst, err := d.Instances.CreateInstance(ctx, instances.CreateInstanceRequest{Name: name, Image: ref.Repository() + "@" + base.Digest, Platform: "darwin/arm64", Hypervisor: hypervisor.TypeVZ, FreshMacOSIdentity: true, Size: int64(base.MacOS.Memory), Vcpus: int(base.MacOS.CPUs), NetworkEnabled: policy.NetworkMode == "egress"}) + if inst == nil { + _ = os.RemoveAll(dir) + if err == nil { + err = fmt.Errorf("instance creation returned no instance") + } + return nil, err + } + session := &guestMachineSession{manager: d.Instances, exporter: exporter, instance: inst, workspace: dir, guestDir: "/var/root/hypeman-build-" + inst.Id} + return session, err +} + +type guestMachineSession struct { + manager instances.Manager + exporter machineExportManager + instance *instances.Instance + workspace, guestDir string + receipt MachineStopReceipt + sanitized bool +} + +func (s *guestMachineSession) ID() string { return s.instance.Id } +func (s *guestMachineSession) dialer() (hypervisor.VsockDialer, error) { + return hypervisor.NewVsockDialer(s.instance.HypervisorType, s.instance.VsockSocket, s.instance.VsockCID) +} +func (s *guestMachineSession) exec(ctx context.Context, command []string, cwd string, env map[string]string, wait time.Duration) error { + dialer, err := s.dialer() + if err != nil { + return err + } + // Drain output without retaining or publishing it. Redaction cannot reliably + // make arbitrary guest output safe: even successful commands may print secrets. + exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{Command: command, Cwd: cwd, Env: env, WaitForAgent: wait, Stdout: io.Discard, Stderr: io.Discard}) + if err != nil { + return err + } + if exit == nil { + return fmt.Errorf("machine guest command did not succeed") + } + if exit.Code != 0 { + return &machineExitError{code: exit.Code} + } + return nil +} + +func (s *guestMachineSession) Provision(ctx context.Context, source string) error { + root := filepath.Join(s.workspace, "source") + if err := extractMachineSource(ctx, source, root); err != nil { + return err + } + err := s.exec(ctx, []string{"/bin/sh", "-c", `test "$(/usr/bin/id -u)" -eq 0`}, "", nil, 30*time.Second) + if err != nil { + return err + } + if err = s.exec(ctx, []string{"/bin/test", "!", "-e", s.guestDir}, "", nil, 30*time.Second); err != nil { + return err + } + if err = s.exec(ctx, []string{"/bin/mkdir", "-m", "700", s.guestDir}, "", nil, 0); err != nil { + return err + } + dialer, err := s.dialer() + if err != nil { + return err + } + if err = guest.CopyToInstance(ctx, dialer, guest.CopyToInstanceOptions{SrcPath: root, DstPath: s.guestDir}); err != nil { + return err + } + return s.exec(ctx, []string{"/bin/sh", machineEntrypoint}, s.guestDir, nil, 0) +} + +func (s *guestMachineSession) Sanitize(ctx context.Context) error { + if err := s.exec(ctx, []string{"/bin/rm", "-rf", "--", s.guestDir}, "", nil, 0); err != nil { + return err + } + if err := s.exec(ctx, []string{"/bin/test", "!", "-e", s.guestDir}, "", nil, 0); err != nil { + return err + } + if err := os.RemoveAll(filepath.Join(s.workspace, "source")); err != nil { + return err + } + s.sanitized = true + return nil +} + +// Stop is the publication path: only an acknowledged graceful power-off with +// confirmed VMM exit may produce a receipt that permits export. With force, a +// failed strict stop falls back to the ordinary forced hypervisor stop. Nothing +// from a failed build is exported, and Destroy still requires confirmed VMM exit, +// so a non-exited instance remains quarantined. +func (s *guestMachineSession) Stop(ctx context.Context, force bool) (MachineStopReceipt, error) { + if s.receipt.VMMExited { + return s.receipt, nil + } + receipt, err := s.exporter.StopMacOSBuildInstance(ctx, s.ID()) + if !force { + s.receipt = receipt + return s.receipt, err + } + if err == nil && receipt.VMMExited { + s.receipt = receipt + return s.receipt, nil + } + if inst, err := s.manager.GetInstance(ctx, s.ID()); err != nil || inst.State != instances.StateStopped { + inst, err = s.manager.StopInstance(ctx, s.ID()) + if err != nil { + return s.receipt, err + } + if inst == nil || inst.State != instances.StateStopped { + return s.receipt, fmt.Errorf("forced stop did not confirm VMM exit") + } + } + s.receipt = MachineStopReceipt{VMMExited: true} + return s.receipt, nil +} + +// Export copies only from closed storage of a Stopped instance, under the +// instance lock, after a strict graceful receipt and sanitation. +func (s *guestMachineSession) Export(ctx context.Context, destination string) error { + if !s.receipt.Graceful || !s.receipt.VMMExited || !s.sanitized { + return fmt.Errorf("sanitized strict stop receipt required") + } + return s.exporter.ExportStoppedMacOSBuildInstance(ctx, s.ID(), destination) +} + +func (s *guestMachineSession) Destroy(ctx context.Context) error { + if !s.receipt.VMMExited { + return fmt.Errorf("unconfirmed VMM exit: instance quarantined") + } + if err := s.manager.DeleteInstance(ctx, s.ID()); err != nil { + return err + } + return os.RemoveAll(s.workspace) +} diff --git a/lib/builds/macos_build_job_test.go b/lib/builds/macos_build_job_test.go new file mode 100644 index 000000000..12394b213 --- /dev/null +++ b/lib/builds/macos_build_job_test.go @@ -0,0 +1,37 @@ +package builds + +import ( + "context" + "io" + "os" + "path/filepath" + "time" +) + +type machineRunnerFixture struct { + MachineBuildBackend + WorkDir string +} + +func (r *machineRunnerFixture) Run(ctx context.Context, req MachineBuildRequest, source io.Reader) (*MachineBuildResult, error) { + timeout := req.Policy.TimeoutSeconds + if timeout == 0 { + timeout = 600 + } + ctx, cancel := context.WithTimeout(ctx, time.Duration(timeout)*time.Second) + defer cancel() + job, err := os.MkdirTemp(r.WorkDir, "prepared-job-") + if err != nil { + return nil, err + } + defer os.RemoveAll(job) + sourcePath := filepath.Join(job, "source.tar.gz") + hash, err := stageBuildSource(ctx, source, sourcePath, maxBuildSourceBytes) + if err != nil { + return nil, machineFailure("stage_source", err) + } + if req.SourceHash == "" { + req.SourceHash = hash + } + return r.runPrepared(ctx, req, r.Driver.(*machineFixture).image, sourcePath, r.WorkDir) +} diff --git a/lib/builds/macos_build_layer_test.go b/lib/builds/macos_build_layer_test.go new file mode 100644 index 000000000..6798c5582 --- /dev/null +++ b/lib/builds/macos_build_layer_test.go @@ -0,0 +1,70 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "io" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestMachineLayerRegeneratesHashedBytesWithoutSpooling(t *testing.T) { + parent := t.TempDir() + bundle := filepath.Join(parent, "bundle") + require.NoError(t, os.Mkdir(bundle, 0700)) + disk := make([]byte, 3<<20) // mostly zeros, like a sparse machine disk + copy(disk[1<<20:], "data") + for name, data := range map[string][]byte{"disk.img": disk, "aux.img": []byte("aux"), "config.json": []byte("{}")} { + require.NoError(t, os.WriteFile(filepath.Join(bundle, name), data, 0600)) + } + + layer, err := machineTarballLayer(context.Background(), bundle) + require.NoError(t, err) + diffID, err := layer.DiffID() + require.NoError(t, err) + require.NotEmpty(t, diffID.Hex) + digest, err := layer.Digest() + require.NoError(t, err) + size, err := layer.Size() + require.NoError(t, err) + for range 2 { + body, err := layer.Compressed() + require.NoError(t, err) + sum := sha256.New() + n, err := io.Copy(sum, body) + require.NoError(t, err) + require.NoError(t, body.Close()) + require.Equal(t, size, n) + require.Equal(t, digest.Hex, hex.EncodeToString(sum.Sum(nil))) + } + entries, err := os.ReadDir(parent) + require.NoError(t, err) + require.Len(t, entries, 1, "nothing is spooled beside the bundle") +} + +func TestMachineLayerAbandonedUploadStopsGenerator(t *testing.T) { + bundle := t.TempDir() + for name, data := range map[string][]byte{"disk.img": make([]byte, 8<<20), "aux.img": []byte("aux"), "config.json": []byte("{}")} { + require.NoError(t, os.WriteFile(filepath.Join(bundle, name), data, 0600)) + } + layer, err := machineTarballLayer(context.Background(), bundle) + require.NoError(t, err) + before := runtime.NumGoroutine() + body, err := layer.Compressed() + require.NoError(t, err) + _, err = io.CopyN(io.Discard, body, 16) + require.NoError(t, err) + require.NoError(t, body.Close()) + // Polled directly: require.Eventually runs its condition in another goroutine. + deadline := time.Now().Add(5 * time.Second) + for runtime.NumGoroutine() > before && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + require.LessOrEqual(t, runtime.NumGoroutine(), before, "the layer generator exits when the upload is abandoned") +} diff --git a/lib/builds/macos_build_lifecycle_test.go b/lib/builds/macos_build_lifecycle_test.go new file mode 100644 index 000000000..f7359e95c --- /dev/null +++ b/lib/builds/macos_build_lifecycle_test.go @@ -0,0 +1,135 @@ +package builds + +import ( + "context" + "errors" + "log/slog" + "testing" + + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +type machineLifecycleInstances struct { + instances.Manager + lookedUp string + state, stoppedState instances.State + missing, stopped bool + deleted bool + stopErr, deleteErr error +} + +func (m *machineLifecycleInstances) GetInstance(_ context.Context, id string) (*instances.Instance, error) { + m.lookedUp = id + if m.missing { + return nil, instances.ErrNotFound + } + return &instances.Instance{StoredMetadata: instances.StoredMetadata{Id: "prior"}, State: m.state}, nil +} +func (m *machineLifecycleInstances) StopInstance(context.Context, string) (*instances.Instance, error) { + m.stopped = true + if m.stopErr != nil { + return nil, m.stopErr + } + if m.stoppedState != "" { + m.state = m.stoppedState + } + return &instances.Instance{State: m.state}, nil +} +func (m *machineLifecycleInstances) DeleteInstance(context.Context, string) error { + m.deleted = true + return m.deleteErr +} + +type abortExporter struct { + receipt instances.MacOSBuildExportReceipt + err error +} + +func (e abortExporter) StopMacOSBuildInstance(context.Context, string) (instances.MacOSBuildExportReceipt, error) { + return e.receipt, e.err +} +func (abortExporter) ExportStoppedMacOSBuildInstance(context.Context, string, string) error { + return errors.New("abort must not export") +} + +func TestMachineForcedStopCleanup(t *testing.T) { + for _, tc := range []struct { + name string + state, stoppedState instances.State + graceful, forceError, wantExit bool + }{ + {"graceful exit", instances.StateRunning, "", true, false, true}, + {"already stopped", instances.StateStopped, "", false, false, true}, + {"force running VM", instances.StateRunning, instances.StateStopped, false, false, true}, + {"unconfirmed stop", instances.StateRunning, "", false, false, false}, + {"failed force stop", instances.StateRunning, "", false, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + m := &machineLifecycleInstances{state: tc.state, stoppedState: tc.stoppedState} + stop := MachineStopReceipt{Graceful: tc.graceful, VMMExited: tc.graceful} + var stopErr error + if !tc.graceful { + stopErr = errors.New("agent never became ready") + } + if tc.forceError { + m.stopErr = errors.New("hypervisor still alive") + } + s := &guestMachineSession{manager: m, exporter: abortExporter{receipt: stop, err: stopErr}, instance: &instances.Instance{StoredMetadata: instances.StoredMetadata{Id: "builder"}}} + receipt, err := s.Stop(context.Background(), true) + if tc.wantExit { + require.NoError(t, err) + require.Equal(t, MachineStopReceipt{Graceful: tc.graceful, VMMExited: true}, receipt) + } else { + wantErr := "forced stop did not confirm VMM exit" + if tc.forceError { + wantErr = "hypervisor still alive" + } + require.EqualError(t, err, wantErr) + require.False(t, receipt.VMMExited) + require.Error(t, s.Destroy(context.Background()), "unconfirmed exit must remain quarantined") + } + require.Equal(t, !tc.graceful && tc.state != instances.StateStopped, m.stopped) + }) + } +} + +func TestReapPriorMachineBuilder(t *testing.T) { + for _, tc := range []struct { + name string + state instances.State + missing bool + unrecorded bool + deleteErr error + wantStop bool + wantDelete bool + wantError bool + }{ + {"stop and delete", instances.StateRunning, false, false, nil, true, true, false}, + {"delete failure blocks retry", instances.StateStopped, false, false, errors.New("storage still open"), false, true, true}, + {"missing builder", "", true, false, nil, false, false, false}, + {"find unrecorded builder", instances.StateRunning, false, true, nil, true, true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + p := paths.New(t.TempDir()) + meta := &buildMetadata{ID: "job", Status: StatusQueued} + if !tc.unrecorded { + prior := "prior" + meta.BuilderInstance = &prior + } + require.NoError(t, writeMetadata(p, meta)) + fake := &machineLifecycleInstances{state: tc.state, missing: tc.missing, deleteErr: tc.deleteErr} + m := &manager{paths: p, instanceManager: fake, logger: slog.Default()} + err := m.reapPriorMachineBuilder(context.Background(), "job") + if tc.wantError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.Equal(t, tc.unrecorded, fake.lookedUp == machineBuilderName("job")) + require.Equal(t, tc.wantStop, fake.stopped) + require.Equal(t, tc.wantDelete, fake.deleted) + }) + } +} diff --git a/lib/builds/macos_build_manager.go b/lib/builds/macos_build_manager.go new file mode 100644 index 000000000..eebd4b9f6 --- /dev/null +++ b/lib/builds/macos_build_manager.go @@ -0,0 +1,192 @@ +package builds + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" +) + +// machineBuild reports whether a request selects the native macOS build path. +func machineBuild(req *CreateBuildRequest) bool { + return req != nil && req.MachineBaseImage != "" +} + +func (m *manager) skipMachineBuildRecovery(meta *buildMetadata) bool { + if !m.config.MachineBuildOnly { + return false + } + if machineBuild(meta.Request) && m.config.MachineBuild != nil { + return false + } + m.logger.Warn("leaving unsupported build pending in machine-build-only mode", "id", meta.ID) + return true +} + +// executeMachineBuild is a backend of the same queued build job. runBuild owns +// deadline/status/log completion, and CreateBuild already staged verified source. +func (m *manager) executeMachineBuild(ctx context.Context, id string, req CreateBuildRequest, policy *BuildPolicy) (*BuildResult, error) { + backend := m.config.MachineBuild + if backend == nil { + return nil, fmt.Errorf("machine build backend is not configured") + } + if err := m.reapPriorMachineBuilder(ctx, id); err != nil { + return nil, machineFailure("reap_prior_builder", err) + } + base, err := m.imageManager.GetImage(ctx, req.MachineBaseImage) + if err != nil { + return nil, machineFailure("resolve_base", err) + } + sourcePath, err := filepath.Abs(filepath.Join(m.paths.BuildSourceDir(id), "source.tar.gz")) + if err != nil { + return nil, err + } + // An interrupted attempt skips its own cleanup; drop its workspace first. + removeMachineWorkspaces(m.paths.BuildDir(id)) + defer removeMachineWorkspaces(m.paths.BuildDir(id)) + job := MachineBuildRequest{ + ID: id, BaseImage: req.MachineBaseImage, SourceHash: req.SourceHash, Policy: *policy, + deltaBase: m.machineDeltaBase(backend.Publisher, req.MachineBaseImage, base), + onStarted: func(instanceID string) error { return m.recordMachineBuilder(id, instanceID) }, + onPublished: func(ctx context.Context, publication MachinePublication) error { + return m.waitMachinePublication(ctx, id, publication.Digest) + }, + } + result, err := backend.runPrepared(ctx, job, base, sourcePath, filepath.Dir(filepath.Dir(sourcePath))) + if err != nil { + return nil, err + } + ref, err := images.ParseNormalizedRef(result.Publication.Reference) + if err != nil || ref.Repository() != strings.TrimSuffix(stripRegistryScheme(m.config.RegistryURL), "/")+"/builds/"+id { + return nil, fmt.Errorf("machine publication is outside the authorized job repository") + } + return &BuildResult{Success: true, ImageDigest: result.Publication.Digest, ImageRef: result.Publication.Reference, Provenance: result.Provenance}, nil +} + +func (m *manager) waitMachinePublication(ctx context.Context, id, digest string) error { + if !ValidMachineDigest(digest) { + return fmt.Errorf("canonical machine publication digest required") + } + if err := m.imageManager.WaitForReady(ctx, "builds/"+id+"@"+digest); err != nil { + return fmt.Errorf("machine image import: %w", err) + } + return nil +} + +// reapPriorMachineBuilder removes the builder recorded by an interrupted attempt of +// this same job before a re-enqueued attempt starts. Nothing from that builder is +// published, so a forced stop is acceptable; a failed delete fails this attempt +// rather than leaving two builders with storage in play. +func (m *manager) reapPriorMachineBuilder(ctx context.Context, id string) error { + meta, err := readMetadata(m.paths, id) + if err != nil { + return err + } + // An attempt interrupted before it recorded its builder still named it after the build. + prior := machineBuilderName(id) + if meta.BuilderInstance != nil && *meta.BuilderInstance != "" { + prior = *meta.BuilderInstance + } + inst, err := m.instanceManager.GetInstance(ctx, prior) + if errors.Is(err, instances.ErrNotFound) { + return nil + } + if err != nil { + return err + } + if inst.State != instances.StateStopped { + if _, err := m.instanceManager.StopInstance(ctx, inst.Id); err != nil { + return fmt.Errorf("stop prior builder: %w", err) + } + } + if err := m.instanceManager.DeleteInstance(ctx, inst.Id); err != nil && !errors.Is(err, instances.ErrNotFound) { + return fmt.Errorf("delete prior builder: %w", err) + } + m.logger.Info("reaped prior machine builder", "build_id", id, "instance_id", inst.Id) + return nil +} + +// storeMachineSource stages machine source privately and records its SHA256 +// provenance. A caller-supplied source hash must match. +func (m *manager) storeMachineSource(ctx context.Context, meta *buildMetadata, data []byte) error { + sourceDir := m.paths.BuildSourceDir(meta.ID) + if err := os.MkdirAll(sourceDir, 0700); err != nil { + return fmt.Errorf("store source: %w", err) + } + hash, err := stageBuildSource(ctx, bytes.NewReader(data), filepath.Join(sourceDir, "source.tar.gz"), int64(len(data))) + if err != nil { + return fmt.Errorf("store source: %w", err) + } + if meta.Request.SourceHash != "" && meta.Request.SourceHash != hash { + return ErrSourceHashMismatch + } + meta.Request.SourceHash = hash + if err := writeMetadata(m.paths, meta); err != nil { + return fmt.Errorf("write source provenance: %w", err) + } + return nil +} + +// admitMachineBuild validates a machine build request and returns its policy. +// Machine resources inherit the installed base, not Linux builder defaults. +func (m *manager) admitMachineBuild(req *CreateBuildRequest, sourceData []byte) (*BuildPolicy, error) { + if m.config.MachineBuild == nil || m.config.MachineBuild.Driver == nil || m.config.MachineBuild.Publisher == nil { + return nil, fmt.Errorf("machine build backend is not configured") + } + if req.Dockerfile != "" || req.BuilderID != "" || len(req.BuildArgs) != 0 || len(req.Secrets) != 0 || req.CacheScope != "" || req.GlobalCacheKey != "" || req.IsAdminBuild { + return nil, fmt.Errorf("machine builds do not support Linux builder/cache/secret options") + } + if len(sourceData) > maxBuildSourceBytes { + return nil, fmt.Errorf("machine build source too large") + } + ref, err := images.ParseNormalizedRef(req.MachineBaseImage) + if err != nil || !ref.IsDigest() || !ValidMachineDigest(ref.Digest()) { + return nil, fmt.Errorf("machine base must be pinned by sha256 digest") + } + policy := &BuildPolicy{} + if req.BuildPolicy != nil { + requested := *req.BuildPolicy + policy = &requested + } + if err := normalizeMachinePolicy(policy); err != nil { + return nil, err + } + req.BuildPolicy = policy + return policy, nil +} + +// enqueueMachineBuild queues a machine build whose source is already staged. Each +// builder has its own identity, so builds on one base are not serialized. +func (m *manager) enqueueMachineBuild(meta *buildMetadata, req CreateBuildRequest, policy *BuildPolicy) *Build { + queuePos := m.queue.EnqueueSerial(meta.ID, req, "", func() { + m.runBuild(context.Background(), meta.ID, req, policy) + }) + build := meta.toBuild() + if queuePos > 0 { + build.QueuePosition = &queuePos + } + m.logger.Info("build created", "id", meta.ID, "queue_position", queuePos) + return build +} + +// removeMachineWorkspaces deletes export workspaces under a build directory. Each +// holds a full exported disk. +func removeMachineWorkspaces(buildDir string) { + matches, _ := filepath.Glob(filepath.Join(buildDir, "machine-build-*")) + for _, dir := range matches { + _ = os.RemoveAll(dir) + } +} + +// removeMachineSource deletes a machine build's staged source after the job is terminal. +func (m *manager) removeMachineSource(id string) { + if err := os.Remove(filepath.Join(m.paths.BuildSourceDir(id), "source.tar.gz")); err != nil && !errors.Is(err, os.ErrNotExist) { + m.logger.Warn("failed to remove machine build source", "id", id, "error", err) + } +} diff --git a/lib/builds/macos_build_manager_test.go b/lib/builds/macos_build_manager_test.go new file mode 100644 index 000000000..c30799a76 --- /dev/null +++ b/lib/builds/macos_build_manager_test.go @@ -0,0 +1,248 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/instances" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +type machineJobImages struct { + images.Manager + fixture *machineFixture + resolveErr error +} + +func (m machineJobImages) GetImage(ctx context.Context, ref string) (*images.Image, error) { + if m.resolveErr != nil { + return nil, m.resolveErr + } + if ref == "registry/base@"+machineTestDigest { + return m.fixture.ResolveBase(ctx, ref) + } + return &images.Image{Status: images.StatusReady, Digest: machineTestDigest}, nil +} + +func (m machineJobImages) WaitForReady(context.Context, string) error { return nil } + +func newMachineTestManager(t *testing.T, f *machineFixture, publisher MachineBuildPublisher, imageManager images.Manager, machineOnly bool) *manager { + t.Helper() + config := DefaultConfig() + config.RegistrySecret = "synthetic-test-secret" + config.MachineBuildOnly = machineOnly + if f != nil && publisher != nil { + config.MachineBuild = &MachineBuildBackend{Driver: f, Publisher: publisher} + } + if imageManager == nil && f != nil { + imageManager = machineJobImages{fixture: f} + } + api, err := NewManager(paths.New(t.TempDir()), config, noPriorBuilder{}, nil, nil, imageManager, nil, nil, nil) + require.NoError(t, err) + return api.(*manager) +} + +type machineJobPublisher struct { + fixture *machineFixture + publishedID string + // sourcePath, when set, must still exist at publish time. + sourcePath func(id string) string +} + +func (p *machineJobPublisher) Publish(ctx context.Context, id, bundle string) (MachinePublication, error) { + p.publishedID = id + if p.sourcePath != nil { + if _, err := os.Stat(p.sourcePath(id)); err != nil { + return MachinePublication{}, errors.New("source removed before publish") + } + } + // Reuse the fixture's export assertions with its fixed ID. + receipt, err := p.fixture.Publish(ctx, "build-test", bundle) + if err == nil { + receipt.Reference = "localhost:4973/builds/" + id + "@" + receipt.Digest + } + return receipt, err +} +func TestMachineBuildUsesSharedQueueStatusAndSourceLifecycle(t *testing.T) { + f := newMachineFixture() + publisher := &machineJobPublisher{fixture: f} + m := newMachineTestManager(t, f, publisher, nil, false) + publisher.sourcePath = func(id string) string { return filepath.Join(m.paths.BuildSourceDir(id), "source.tar.gz") } + source := []byte("synthetic compressed-source bytes") + sum := sha256.Sum256(source) + req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, SourceHash: hex.EncodeToString(sum[:])} + build, err := m.CreateBuild(context.Background(), req, source) + require.NoError(t, err) + require.Eventually(t, func() bool { + b, e := m.GetBuild(context.Background(), build.ID) + return e == nil && (b.Status == StatusReady || b.Status == StatusFailed) + }, 5*time.Second, 10*time.Millisecond) + done, err := m.GetBuild(context.Background(), build.ID) + require.NoError(t, err) + require.Equal(t, StatusReady, done.Status, "failure: %v", done.Error) + require.NotNil(t, done.ImageRef) + require.Equal(t, "localhost:4973/builds/"+build.ID+"@"+machineTestDigest, *done.ImageRef) + require.Equal(t, build.ID, publisher.publishedID) + require.Equal(t, req.SourceHash, done.Provenance.SourceHash) + require.Equal(t, machineTestDigest, done.Provenance.BaseImageDigest) + require.Equal(t, f.ID(), *done.BuilderInstanceID) + meta, err := readMetadata(m.paths, build.ID) + require.NoError(t, err) + require.Equal(t, req.MachineBaseImage, meta.Request.MachineBaseImage) + require.Equal(t, "isolated", meta.Request.BuildPolicy.NetworkMode) + require.Equal(t, 0, meta.Request.BuildPolicy.CPUs, "machine defaults must not use Linux CPU defaults") + _, err = readBuildConfig(m.paths, build.ID) + require.Error(t, err, "machine builds must not persist a builder config or registry token") + require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(build.ID), "source.tar.gz")) +} +func TestMachineBuildsOnOneBaseRunConcurrently(t *testing.T) { + fixture := newMachineFixture() + fixture.waitForCancel = true + m := newMachineTestManager(t, fixture, fixture, nil, false) + req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest} + var ids []string + for range 2 { + build, err := m.CreateBuild(context.Background(), req, []byte("source")) + require.NoError(t, err) + ids = append(ids, build.ID) + } + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 2 }, 5*time.Second, 10*time.Millisecond, "builds on one base must not serialize") + for _, id := range ids { + // Cancel only once the attempt has recorded its builder, so it is Building. + require.Eventually(t, func() bool { + meta, err := readMetadata(m.paths, id) + return err == nil && meta.BuilderInstance != nil + }, 5*time.Second, 10*time.Millisecond) + require.NoError(t, m.CancelBuild(context.Background(), id)) + } + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 0 }, 5*time.Second, 10*time.Millisecond) +} + +func TestMachineBuildAdmissionAndSourceHash(t *testing.T) { + m := newMachineTestManager(t, nil, nil, nil, false) + _, err := m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, []byte("source")) + require.ErrorContains(t, err, "not configured") + f := newMachineFixture() + m.config.MachineBuild = &MachineBuildBackend{Driver: f, Publisher: f} + _, err = m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base:mutable"}, []byte("source")) + require.ErrorContains(t, err, "pinned") + _, err = m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, Secrets: []SecretRef{{ID: "unsupported"}}}, []byte("source")) + require.ErrorContains(t, err, "secret options") + _, err = m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, SourceHash: "wrong"}, []byte("source")) + require.True(t, errors.Is(err, ErrSourceHashMismatch)) + _, err = m.admitMachineBuild(&CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, make([]byte, maxBuildSourceBytes+1)) + require.ErrorContains(t, err, "source too large") + list, err := m.ListBuilds(context.Background()) + require.NoError(t, err) + require.Empty(t, list) +} + +func TestQueuedMachineCancellationRemovesSource(t *testing.T) { + fixture := newMachineFixture() + fixture.waitForCancel = true + m := newMachineTestManager(t, fixture, fixture, nil, false) + req := CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest} + // Fill both concurrency slots so the next build stays queued. + for range m.config.MaxConcurrentBuilds { + first, err := m.CreateBuild(context.Background(), req, []byte("synthetic source")) + require.NoError(t, err) + t.Cleanup(func() { + _ = m.CancelBuild(context.Background(), first.ID) + require.Eventually(t, func() bool { _, active := m.machineBuilds.Load(first.ID); return !active }, 5*time.Second, 10*time.Millisecond) + }) + require.Eventually(t, func() bool { + meta, err := readMetadata(m.paths, first.ID) + return err == nil && meta.BuilderInstance != nil + }, 5*time.Second, 10*time.Millisecond) + } + queued, err := m.CreateBuild(context.Background(), req, []byte("queued source")) + require.NoError(t, err) + source := filepath.Join(m.paths.BuildSourceDir(queued.ID), "source.tar.gz") + require.FileExists(t, source) + require.NoError(t, m.CancelBuild(context.Background(), queued.ID)) + done, err := m.GetBuild(context.Background(), queued.ID) + require.NoError(t, err) + require.Equal(t, StatusCancelled, done.Status) + require.NotNil(t, done.CompletedAt) + require.NoFileExists(t, source) +} + +func TestMachineCancellationInterruptsPhasesWithoutPublishing(t *testing.T) { + fixture := newMachineFixture() + fixture.waitForCancel = true + m := newMachineTestManager(t, fixture, fixture, nil, false) + build, e := m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, []byte("synthetic source")) + require.NoError(t, e) + require.Eventually(t, func() bool { + meta, e := readMetadata(m.paths, build.ID) + return e == nil && meta.BuilderInstance != nil + }, 5*time.Second, 10*time.Millisecond) + require.NoError(t, m.CancelBuild(context.Background(), build.ID)) + require.Eventually(t, func() bool { _, active := m.machineBuilds.Load(build.ID); return !active }, 5*time.Second, 10*time.Millisecond) + done, e := m.GetBuild(context.Background(), build.ID) + require.NoError(t, e) + require.Equal(t, StatusCancelled, done.Status) + require.False(t, fixture.publisherCalled) + require.Contains(t, fixture.calls, "abort") + require.Contains(t, fixture.calls, "destroy") +} + +func TestMachineFailureUsesSharedFailedStatusWithoutPublication(t *testing.T) { + for _, tc := range []struct { + name string + phase string + resolveErr error + wantError string + }{ + {name: "base resolution", resolveErr: errors.New("sensitive-resolution-error"), wantError: "machine build resolve_base failed"}, + {name: "provision", phase: "provision", wantError: "machine build provision failed"}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newMachineFixture() + f.fail = tc.phase + m := newMachineTestManager(t, f, f, machineJobImages{fixture: f, resolveErr: tc.resolveErr}, false) + build, err := m.CreateBuild(context.Background(), CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}, []byte("source")) + require.NoError(t, err) + require.Eventually(t, func() bool { + b, e := m.GetBuild(context.Background(), build.ID) + return e == nil && b.Status == StatusFailed + }, 5*time.Second, 10*time.Millisecond) + done, err := m.GetBuild(context.Background(), build.ID) + require.NoError(t, err) + require.Equal(t, StatusFailed, done.Status) + require.Nil(t, done.ImageDigest) + require.NotContains(t, f.calls, "publish") + require.NotContains(t, *done.Error, "sensitive-resolution-error") + require.Equal(t, tc.wantError, *done.Error) + if tc.phase == "provision" { + require.Contains(t, f.calls, "destroy") + } else { + require.Empty(t, f.calls) + } + }) + } +} + +func TestMachineFailureShowsOnlyExitCode(t *testing.T) { + err := machineFailure("provision", fmt.Errorf("guest said secret-token: %w", &machineExitError{code: 7})) + require.Equal(t, "machine build provision failed: build.sh exited with code 7", err.Error()) + require.Equal(t, "machine build provision failed", machineFailure("provision", fmt.Errorf("secret-token")).Error()) + admission := fmt.Errorf("%w: macOS template identity is already in use", instances.ErrInsufficientResources) + require.Contains(t, machineFailure("start", admission).Error(), "machine build start failed: insufficient resources") +} + +// noPriorBuilder reports that no earlier attempt left a builder instance behind. +type noPriorBuilder struct{ instances.Manager } + +func (noPriorBuilder) GetInstance(context.Context, string) (*instances.Instance, error) { + return nil, instances.ErrNotFound +} diff --git a/lib/builds/macos_build_policy_test.go b/lib/builds/macos_build_policy_test.go new file mode 100644 index 000000000..32c1592f6 --- /dev/null +++ b/lib/builds/macos_build_policy_test.go @@ -0,0 +1,44 @@ +package builds + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMachinePolicyNormalization(t *testing.T) { + for _, tc := range []struct { + name string + policy BuildPolicy + err string + }{ + {name: "defaults"}, + {name: "minimum timeout", policy: BuildPolicy{TimeoutSeconds: 1}}, + {name: "maximum timeout", policy: BuildPolicy{TimeoutSeconds: 86400}}, + {name: "egress", policy: BuildPolicy{NetworkMode: "egress"}}, + {name: "negative timeout", policy: BuildPolicy{TimeoutSeconds: -1}, err: "invalid machine build timeout"}, + {name: "excessive timeout", policy: BuildPolicy{TimeoutSeconds: 86401}, err: "invalid machine build timeout"}, + {name: "unsupported mode", policy: BuildPolicy{NetworkMode: "host"}, err: "unsupported machine network policy"}, + {name: "domains", policy: BuildPolicy{NetworkMode: "egress", AllowedDomains: []string{"example.test"}}, err: "unsupported machine network policy"}, + } { + t.Run(tc.name, func(t *testing.T) { + policy := tc.policy + err := normalizeMachinePolicy(&policy) + if tc.err != "" { + require.EqualError(t, err, tc.err) + return + } + require.NoError(t, err) + want := tc.policy + if want.TimeoutSeconds == 0 { + want.TimeoutSeconds = 600 + } + if want.NetworkMode == "" { + want.NetworkMode = "isolated" + } + require.Equal(t, want, policy) + require.NoError(t, normalizeMachinePolicy(&policy)) + require.Equal(t, want, policy) + }) + } +} diff --git a/lib/builds/macos_build_publisher.go b/lib/builds/macos_build_publisher.go new file mode 100644 index 000000000..59be90116 --- /dev/null +++ b/lib/builds/macos_build_publisher.go @@ -0,0 +1,196 @@ +package builds + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/images" +) + +// StreamingMachinePublisher targets only the configured internal registry and a +// build-scoped repository. go-containerregistry hashes and compresses the layer +// from a reopenable tar stream, so it is never written to disk. Blobs use the +// registry's disk-streaming monolithic upload and are independently verified before +// the manifest is committed last. +// No credentials, redirect destinations or arbitrary output repositories come from the source. +type StreamingMachinePublisher struct { + RegistryURL string + Tokens *RegistryTokenGenerator +} + +type machineBlob struct { + open func() (io.ReadCloser, error) + descriptor v1.Descriptor +} + +// Publish commits bundle as a standalone machine image in builds/{id}. +func (p *StreamingMachinePublisher) Publish(ctx context.Context, id, bundle string) (MachinePublication, error) { + return p.publish(ctx, id, bundle, nil) +} + +// PublishDerived commits bundle as a delta over base. The base layers are mounted +// from base.Repo rather than uploaded; only the delta layer and config are uploaded. +func (p *StreamingMachinePublisher) PublishDerived(ctx context.Context, id, bundle string, base MachineDeltaBase) (MachinePublication, error) { + if err := base.validate(); err != nil { + return MachinePublication{}, err + } + return p.publish(ctx, id, bundle, &base) +} + +func (p *StreamingMachinePublisher) publish(ctx context.Context, id, bundle string, source *MachineDeltaBase) (MachinePublication, error) { + deadline, ok := ctx.Deadline() + if !ok || ctx.Err() != nil { + return MachinePublication{}, fmt.Errorf("bounded publication required") + } + if id == "" || len(id) > 128 || strings.Trim(id, "abcdefghijklmnopqrstuvwxyz0123456789-") != "" || p.Tokens == nil { + return MachinePublication{}, fmt.Errorf("scoped machine publisher required") + } + address := p.RegistryURL + if !strings.Contains(address, "://") { + address = "https://" + address + } + registry, err := url.Parse(address) + if err != nil || registry.Host == "" || registry.User != nil || registry.RawQuery != "" || registry.Fragment != "" || (registry.Path != "" && registry.Path != "/") || (registry.Scheme != "http" && registry.Scheme != "https") { + return MachinePublication{}, fmt.Errorf("invalid internal registry address") + } + if registry.Scheme == "http" && registry.Hostname() != "localhost" && !net.ParseIP(registry.Hostname()).IsLoopback() { + return MachinePublication{}, fmt.Errorf("plaintext registry must be loopback") + } + if err := validateMachineExport(bundle, nil); err != nil { + return MachinePublication{}, err + } + repo := "builds/" + id + access := []RepoPermission{{Repo: repo, Scope: "push"}} + if source != nil { + access = append(access, RepoPermission{Repo: source.Repo, Scope: "pull"}) + } + token, err := p.Tokens.GenerateToken(id, access, time.Until(deadline)) + if err != nil { + return MachinePublication{}, err + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + defer transport.CloseIdleConnections() + client := &http.Client{Transport: transport, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + base := strings.TrimRight(address, "/") + "/v2/" + repo + layers, diffIDs, err := machineLayers(ctx, client, token, address, bundle, source) + if err != nil { + return MachinePublication{}, err + } + config := v1.ConfigFile{OS: "darwin", Architecture: "arm64", RootFS: v1.RootFS{Type: "layers", DiffIDs: diffIDs}, Config: v1.Config{Labels: map[string]string{ + images.MacOSMachineVersionLabel: "1", images.MacOSMachineKindLabel: "macos-image", images.MacOSMachineFormatLabel: "raw", images.MacOSMachineDiskLabel: "disk.img", images.MacOSMachineAuxLabel: "aux.img", images.MacOSMachinePlatformLabel: "config.json", + }}} + configBytes, err := json.Marshal(config) + if err != nil { + return MachinePublication{}, err + } + configBlob := machineBlob{open: func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(configBytes)), nil }, descriptor: v1.Descriptor{MediaType: types.OCIConfigJSON, Digest: machineHash(configBytes), Size: int64(len(configBytes))}} + descriptors := make([]v1.Descriptor, 0, len(layers)) + for _, layer := range layers { + descriptors = append(descriptors, layer.descriptor) + } + manifest := v1.Manifest{SchemaVersion: 2, MediaType: types.OCIManifestSchema1, Config: configBlob.descriptor, Layers: descriptors} + manifestBytes, err := json.Marshal(manifest) + if err != nil { + return MachinePublication{}, err + } + digest := machineHash(manifestBytes).String() + for _, blob := range append([]machineBlob{configBlob}, layers...) { + // The blob store is keyed by digest, so a blob the registry already holds + // (a base layer, or a retried publish) is not uploaded again. + stored, e := machineBlobStored(ctx, client, token, base, blob.descriptor) + if e != nil { + return MachinePublication{}, e + } + if stored { + continue + } + body, e := blob.open() + if e != nil { + return MachinePublication{}, e + } + response, e := machineRegistryRequest(ctx, client, token, http.MethodPost, base+"/blobs/uploads/?digest="+blob.descriptor.Digest.String(), "application/octet-stream", io.LimitReader(body, blob.descriptor.Size), blob.descriptor.Size) + closeErr := body.Close() + if e != nil { + return MachinePublication{}, e + } + if closeErr != nil { + response.Body.Close() + return MachinePublication{}, closeErr + } + status := response.StatusCode + response.Body.Close() + if status != http.StatusCreated { + return MachinePublication{}, fmt.Errorf("machine blob upload was not committed") + } + if _, e = machineFetch(ctx, client, token, base+"/blobs/"+blob.descriptor.Digest.String(), blob.descriptor.Digest.String(), blob.descriptor.Size); e != nil { + return MachinePublication{}, fmt.Errorf("machine blob verification failed: %w", e) + } + } + target := base + "/manifests/latest" + response, err := machineRegistryRequest(ctx, client, token, http.MethodPut, target, string(types.OCIManifestSchema1), bytes.NewReader(manifestBytes), int64(len(manifestBytes))) + if err != nil { + return MachinePublication{}, err + } + status := response.StatusCode + response.Body.Close() + if status != http.StatusCreated { + return MachinePublication{}, fmt.Errorf("machine manifest commit unconfirmed") + } + // The digest covers manifestBytes exactly, so a matching fetch is the same bytes. + if _, err = machineFetch(ctx, client, token, base+"/manifests/"+digest, digest, int64(len(manifestBytes))); err != nil { + return MachinePublication{}, fmt.Errorf("machine manifest verification failed: %w", err) + } + return MachinePublication{Reference: registry.Host + "/" + repo + "@" + digest, Digest: digest}, nil +} + +func machineTarballLayer(ctx context.Context, bundle string) (v1.Layer, error) { + return tarball.LayerFromOpener(func() (io.ReadCloser, error) { + reader, writer := io.Pipe() + go func() { writer.CloseWithError(writeMachineTar(ctx, bundle, writer)) }() + return reader, nil + }, tarball.WithMediaType(types.OCILayer), tarball.WithCompressionLevel(gzip.DefaultCompression)) +} + +// writeMachineTar writes a deterministic tar of the exported bundle. +func writeMachineTar(ctx context.Context, bundle string, w io.Writer) error { + archive := tar.NewWriter(w) + for _, name := range []string{"disk.img", "aux.img", "config.json"} { + source, e := os.Open(filepath.Join(bundle, name)) + if e != nil { + return e + } + info, e := source.Stat() + if e != nil { + source.Close() + return e + } + e = archive.WriteHeader(&tar.Header{Name: name, Mode: 0600, Size: info.Size(), Typeflag: tar.TypeReg}) + if e == nil { + _, e = io.CopyN(archive, buildContextReader{ctx, source}, info.Size()) + } + closeErr := source.Close() + if e != nil { + return e + } + if closeErr != nil { + return closeErr + } + } + return archive.Close() +} diff --git a/lib/builds/macos_build_publisher_test.go b/lib/builds/macos_build_publisher_test.go new file mode 100644 index 000000000..c710a49e7 --- /dev/null +++ b/lib/builds/macos_build_publisher_test.go @@ -0,0 +1,96 @@ +package builds_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/registry" + "github.com/stretchr/testify/require" +) + +type machineImportStub struct { + images.Manager + imported chan string +} + +func (m *machineImportStub) ImportLocalImage(_ context.Context, repo, reference, digest string) (*images.Image, error) { + m.imported <- repo + return &images.Image{Status: images.StatusReady, Digest: digest}, nil +} + +func machinePublisherBundle(t *testing.T) string { + t.Helper() + root := t.TempDir() + platform := images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true} + data, e := json.Marshal(platform) + require.NoError(t, e) + for name, data := range map[string][]byte{"config.json": data, "disk.img": []byte("synthetic disk"), "aux.img": []byte("synthetic aux")} { + require.NoError(t, os.WriteFile(filepath.Join(root, name), data, 0600)) + } + return root +} + +func TestStreamingMachinePublisherAuthenticatedRegistry(t *testing.T) { + // Tiny storage is intentional: this proves auth/streaming/commit ordering, not bootability. + imports := &machineImportStub{imported: make(chan string, 1)} + reg, e := registry.New(paths.New(t.TempDir()), imports) + require.NoError(t, e) + var uploads, verified atomic.Int32 + handler := middleware.JwtAuth("synthetic registry secret")(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.NotEqual(t, http.MethodPatch, r.Method, "uploads must use the disk-streaming monolithic endpoint") + if r.Method == http.MethodPost { + require.NotEmpty(t, r.URL.Query().Get("digest")) + require.Greater(t, r.ContentLength, int64(0)) + uploads.Add(1) + } + if r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/blobs/") { + verified.Add(1) + } + if r.Method == http.MethodPut { + require.Equal(t, int32(2), uploads.Load(), "manifest must be last") + } + reg.Handler().ServeHTTP(w, r) + })) + server := httptest.NewServer(handler) + defer server.Close() + publisher := &builds.StreamingMachinePublisher{RegistryURL: server.URL, Tokens: builds.NewRegistryTokenGenerator("synthetic registry secret")} + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + result, e := publisher.Publish(ctx, "qa-build", machinePublisherBundle(t)) + require.NoError(t, e) + require.Contains(t, result.Reference, "/builds/qa-build@sha256:") + select { + case repo := <-imports.imported: + require.Equal(t, "builds/qa-build", repo) + case <-ctx.Done(): + t.Fatal("manifest did not reach production conversion boundary") + } +} + +func TestStreamingMachinePublisherRejectsRedirectWithoutLeakingToken(t *testing.T) { + var visited atomic.Bool + other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { visited.Store(true) })) + defer other.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, other.URL, http.StatusTemporaryRedirect) + })) + defer server.Close() + publisher := &builds.StreamingMachinePublisher{RegistryURL: server.URL, Tokens: builds.NewRegistryTokenGenerator("synthetic secret")} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, e := publisher.Publish(ctx, "qa-build", machinePublisherBundle(t)) + require.Error(t, e) + require.False(t, visited.Load()) +} diff --git a/lib/builds/macos_build_recovery_test.go b/lib/builds/macos_build_recovery_test.go new file mode 100644 index 000000000..411c12102 --- /dev/null +++ b/lib/builds/macos_build_recovery_test.go @@ -0,0 +1,88 @@ +package builds + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestMachineOnlyStartRecoversInterruptedBuilds(t *testing.T) { + for _, status := range []string{StatusQueued, StatusBuilding, StatusPushing} { + t.Run(status, func(t *testing.T) { + f := newMachineFixture() + m := newMachineTestManager(t, f, &machineJobPublisher{fixture: f}, nil, true) + m.config.BuilderImage = "linux-builder-must-not-start" + policy := &BuildPolicy{} + require.NoError(t, normalizeMachinePolicy(policy)) + meta := &buildMetadata{ID: "build-test", Status: status, Request: &CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest, BuildPolicy: policy}} + require.NoError(t, m.storeMachineSource(context.Background(), meta, []byte("synthetic compressed-source bytes"))) + // A workspace left by the interrupted attempt. + stale := filepath.Join(m.paths.BuildDir(meta.ID), "machine-build-interrupted") + require.NoError(t, os.MkdirAll(filepath.Join(stale, "bundle"), 0700)) + require.NoError(t, os.WriteFile(filepath.Join(stale, "bundle", "disk.img"), []byte("disk"), 0600)) + require.NoError(t, writeMetadata(m.paths, &buildMetadata{ID: "linux-job", Status: StatusQueued, Request: &CreateBuildRequest{BuilderID: "linux-builder"}})) + t.Cleanup(func() { + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 0 }, 5*time.Second, 10*time.Millisecond) + }) + + // Hold the recovered build active so the second recovery sees it running. + f.startGate = make(chan struct{}) + require.NoError(t, m.Start(context.Background())) + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 1 }, 5*time.Second, 10*time.Millisecond) + m.RecoverPendingBuilds() + close(f.startGate) + require.Eventually(t, func() bool { return m.queue.ActiveCount() == 0 && m.queue.PendingCount() == 0 }, 5*time.Second, 10*time.Millisecond) + build, err := m.GetBuild(context.Background(), meta.ID) + require.NoError(t, err) + require.Equal(t, StatusReady, build.Status, "failure: %v", build.Error) + require.NoDirExists(t, stale, "the interrupted attempt's workspace is removed") + require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(meta.ID), "source.tar.gz"), "source is removed once the image is ready") + require.False(t, m.ReadyForBuilds(), "Linux builder image must not be prepared") + var starts int + for _, call := range f.calls { + if call == "start" { + starts++ + } + } + require.Equal(t, 1, starts, "recovery must not duplicate an active build") + linux, err := readMetadata(m.paths, "linux-job") + require.NoError(t, err) + require.Equal(t, StatusQueued, linux.Status, "Linux jobs are preserved for Linux-mode recovery") + require.True(t, m.BuilderHasBuilds("linux-builder"), "unrecovered disk jobs still protect their builders") + }) + } +} + +func TestRecoveryCleansCancelledMachineBuildSource(t *testing.T) { + f := newMachineFixture() + m := newMachineTestManager(t, f, &machineJobPublisher{fixture: f}, nil, false) + meta := &buildMetadata{ + ID: "cancelled-build", + Status: StatusCancelled, + Request: &CreateBuildRequest{ + MachineBaseImage: "registry/base@" + machineTestDigest, + }, + } + require.NoError(t, m.storeMachineSource(context.Background(), meta, []byte("source"))) + meta.Status = StatusCancelled + require.NoError(t, writeMetadata(m.paths, meta)) + + m.RecoverPendingBuilds() + + require.NoFileExists(t, filepath.Join(m.paths.BuildSourceDir(meta.ID), "source.tar.gz")) +} + +func TestMachineOnlyStartPreservesBuildsWhenBackendDisabled(t *testing.T) { + m := newMachineTestManager(t, nil, nil, nil, true) + require.NoError(t, writeMetadata(m.paths, &buildMetadata{ID: "disabled", Status: StatusBuilding, Request: &CreateBuildRequest{MachineBaseImage: "registry/base@" + machineTestDigest}})) + require.NoError(t, m.Start(context.Background())) + require.Equal(t, 0, m.queue.ActiveCount()) + meta, err := readMetadata(m.paths, "disabled") + require.NoError(t, err) + require.Equal(t, StatusBuilding, meta.Status) + require.False(t, m.ReadyForBuilds()) +} diff --git a/lib/builds/macos_build_source.go b/lib/builds/macos_build_source.go new file mode 100644 index 000000000..37b7ee2f5 --- /dev/null +++ b/lib/builds/macos_build_source.go @@ -0,0 +1,119 @@ +package builds + +import ( + "archive/tar" + "compress/gzip" + "context" + "fmt" + "io" + "os" + "path" + "path/filepath" + "strings" + "unicode/utf8" +) + +// machineEntrypoint is the script a machine build source must contain at its +// root. It runs as root with /bin/sh in the guest's private build directory. +const machineEntrypoint = "build.sh" +const maxMachineExtractBytes = 128 << 20 + +// extractMachineSource unpacks a build-owned tar.gz on the host into an exclusive +// directory, never with guest tar or a caller-supplied destination. Links, +// devices, duplicates and decompression bombs fail before anything is copied to +// the VM. The archive must contain the entrypoint as a regular file. +func extractMachineSource(ctx context.Context, source, destination string) error { + file, err := os.Open(source) + if err != nil { + return err + } + defer file.Close() + compressed, err := gzip.NewReader(file) + if err != nil { + return err + } + defer compressed.Close() + compressed.Multistream(false) + if err = os.Mkdir(destination, 0700); err != nil { + return err + } + complete := false + defer func() { + if !complete { + _ = os.RemoveAll(destination) + } + }() + limited := &io.LimitedReader{R: compressed, N: maxMachineExtractBytes + 1} + reader := tar.NewReader(limited) + seen := map[string]bool{} + var total int64 + for count := 0; ; count++ { + if err = ctx.Err(); err != nil { + return err + } + header, e := reader.Next() + if e == io.EOF { + break + } + if e != nil { + return e + } + if count >= 1024 { + return fmt.Errorf("too many machine source entries") + } + name := path.Clean(header.Name) + if name == "." && header.Typeflag == tar.TypeDir { + continue + } + if !utf8.ValidString(name) || strings.ContainsAny(name, "\\\x00") || !filepath.IsLocal(name) || seen[name] { + return fmt.Errorf("invalid or duplicate machine source path") + } + seen[name] = true + if header.Size < 0 || header.Size > 64<<20 || total+header.Size > maxMachineExtractBytes { + return fmt.Errorf("machine source expands beyond limit") + } + total += header.Size + target := filepath.Join(destination, name) + switch header.Typeflag { + case tar.TypeDir: + if header.Size != 0 { + return fmt.Errorf("directory has data") + } + if err = os.MkdirAll(target, 0700); err != nil { + return err + } + case tar.TypeReg, tar.TypeRegA: + if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil { + return err + } + output, e := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600|os.FileMode(header.Mode)&0100) + if e != nil { + return e + } + _, copyErr := io.Copy(output, reader) + closeErr := output.Close() + if copyErr != nil { + return copyErr + } + if closeErr != nil { + return closeErr + } + default: + return fmt.Errorf("machine source must contain only regular files and directories") + } + } + // Consume bounded trailing tar padding to verify gzip integrity without allowing + // an unbounded post-tar decompression stream. + if _, err = io.Copy(io.Discard, limited); err != nil { + return err + } + if limited.N <= 0 { + return fmt.Errorf("machine source expands beyond limit") + } + info, err := os.Lstat(filepath.Join(destination, machineEntrypoint)) + if err != nil || !info.Mode().IsRegular() || info.Size() == 0 { + return fmt.Errorf("machine source must contain %s", machineEntrypoint) + } + complete = true + return nil +} diff --git a/lib/builds/macos_build_source_test.go b/lib/builds/macos_build_source_test.go new file mode 100644 index 000000000..00f0d5f4d --- /dev/null +++ b/lib/builds/macos_build_source_test.go @@ -0,0 +1,67 @@ +package builds + +import ( + "archive/tar" + "compress/gzip" + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func writeMachineSource(t *testing.T, headers []*tar.Header, bodies []string) string { + t.Helper() + name := filepath.Join(t.TempDir(), "source.tar.gz") + f, e := os.Create(name) + require.NoError(t, e) + gz := gzip.NewWriter(f) + writer := tar.NewWriter(gz) + for i, h := range headers { + require.NoError(t, writer.WriteHeader(h)) + if h.Typeflag == tar.TypeReg { + _, e = writer.Write([]byte(bodies[i])) + require.NoError(t, e) + } + } + require.NoError(t, writer.Close()) + require.NoError(t, gz.Close()) + require.NoError(t, f.Close()) + return name +} + +func TestMachineSourceExtraction(t *testing.T) { + source := writeMachineSource(t, []*tar.Header{{Name: machineEntrypoint, Size: 4, Typeflag: tar.TypeReg, Mode: 0755}, {Name: "lib/data", Size: 1, Typeflag: tar.TypeReg}}, []string{"true", "x"}) + root := filepath.Join(t.TempDir(), "source") + require.NoError(t, extractMachineSource(context.Background(), source, root)) + info, e := os.Stat(filepath.Join(root, machineEntrypoint)) + require.NoError(t, e) + require.Equal(t, os.FileMode(0700), info.Mode().Perm()) + info, e = os.Stat(filepath.Join(root, "lib", "data")) + require.NoError(t, e) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) +} + +func TestMachineSourceRejectsUnsafeArchives(t *testing.T) { + entry := &tar.Header{Name: machineEntrypoint, Typeflag: tar.TypeReg, Size: 4} + for _, tc := range []struct { + name string + headers []*tar.Header + bodies []string + }{ + {"missing entrypoint", []*tar.Header{{Name: "other", Typeflag: tar.TypeReg, Size: 1}}, []string{"x"}}, + {"empty entrypoint", []*tar.Header{{Name: machineEntrypoint, Typeflag: tar.TypeReg}}, []string{""}}, + {"entrypoint directory", []*tar.Header{{Name: machineEntrypoint, Typeflag: tar.TypeDir}}, []string{""}}, + {"traversal", []*tar.Header{entry, {Name: "../outside", Typeflag: tar.TypeReg, Size: 1}}, []string{"true", "x"}}, + {"symlink", []*tar.Header{entry, {Name: "link", Typeflag: tar.TypeSymlink, Linkname: "/etc"}}, []string{"true", ""}}, + {"hardlink", []*tar.Header{entry, {Name: "link", Typeflag: tar.TypeLink, Linkname: "other"}}, []string{"true", ""}}, + {"duplicate", []*tar.Header{entry, {Name: "a", Typeflag: tar.TypeReg, Size: 1}, {Name: "a", Typeflag: tar.TypeReg, Size: 1}}, []string{"true", "x", "y"}}, + } { + t.Run(tc.name, func(t *testing.T) { + root := filepath.Join(t.TempDir(), "source") + require.Error(t, extractMachineSource(context.Background(), writeMachineSource(t, tc.headers, tc.bodies), root)) + require.NoDirExists(t, root) + }) + } +} diff --git a/lib/builds/macos_build_test.go b/lib/builds/macos_build_test.go new file mode 100644 index 000000000..1b7f9c5ba --- /dev/null +++ b/lib/builds/macos_build_test.go @@ -0,0 +1,233 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "os" + "path/filepath" + "reflect" + "strings" + "sync" + "testing" + + "github.com/kernel/hypeman/lib/images" +) + +const machineTestDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +type machineFixture struct { + mu sync.Mutex + image *images.Image + calls []string + fail string + stop MachineStopReceipt + cancel context.CancelFunc + waitForCancel bool + mutateExport func(string) error + publicationOverride *MachinePublication + publisherCalled bool + cleanupContextLive bool + startGate chan struct{} +} + +func newMachineFixture() *machineFixture { + return &machineFixture{image: &images.Image{Digest: machineTestDigest, Status: images.StatusReady, Platform: "darwin/arm64", MacOS: &images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true}}, stop: MachineStopReceipt{Graceful: true, VMMExited: true}} +} +func (f *machineFixture) record(phase string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, phase) + if f.fail == phase { + return errors.New("sensitive-command-argument") + } + return nil +} +func (f *machineFixture) ResolveBase(context.Context, string) (*images.Image, error) { + if err := f.record("resolve"); err != nil { + return nil, err + } + return f.image, nil +} +func (f *machineFixture) Start(_ context.Context, _ string, _ *images.Image, p BuildPolicy) (MachineBuildSession, error) { + if p.CPUs != 4 || p.MemoryMB != 8192 { + return nil, errors.New("wrong resource defaults") + } + if f.startGate != nil { + <-f.startGate + } + return f, f.record("start") +} +func (f *machineFixture) ID() string { return "isolated-test-instance" } +func (f *machineFixture) Provision(ctx context.Context, source string) error { + if _, err := os.ReadFile(source); err != nil { + return err + } + if f.cancel != nil { + f.cancel() + return ctx.Err() + } + if f.waitForCancel { + <-ctx.Done() + return ctx.Err() + } + return f.record("provision") +} +func (f *machineFixture) Sanitize(context.Context) error { return f.record("sanitize") } +func (f *machineFixture) Stop(ctx context.Context, force bool) (MachineStopReceipt, error) { + f.cleanupContextLive = ctx.Err() == nil + phase := "stop" + if force { + phase = "abort" + } + return f.stop, f.record(phase) +} +func (f *machineFixture) Export(_ context.Context, root string) error { + if err := f.record("export"); err != nil { + return err + } + // The builder has its own identity; the export carries it, not the base's. + output := *f.image.MacOS + output.MachineIdentifier, output.MAC = []byte{3}, "02:00:00:00:00:02" + data, _ := json.Marshal(output) + for name, content := range map[string][]byte{"disk.img": []byte("disk"), "aux.img": []byte("aux"), "config.json": data} { + if err := os.WriteFile(filepath.Join(root, name), content, 0666); err != nil { + return err + } + } + if f.mutateExport != nil { + return f.mutateExport(root) + } + return nil +} +func (f *machineFixture) Destroy(ctx context.Context) error { + f.cleanupContextLive = ctx.Err() == nil + return f.record("destroy") +} +func (f *machineFixture) Publish(ctx context.Context, id, root string) (MachinePublication, error) { + f.publisherCalled = true + if err := ctx.Err(); err != nil { + return MachinePublication{}, err + } + if id != "build-test" { + return MachinePublication{}, errors.New("wrong job namespace") + } + for _, name := range []string{"disk.img", "aux.img", "config.json"} { + info, err := os.Stat(filepath.Join(root, name)) + if err != nil || info.Mode().Perm() != 0600 { + return MachinePublication{}, errors.New("unsafe export mode") + } + } + publication := MachinePublication{Reference: "localhost/builds/build-test@" + machineTestDigest, Digest: machineTestDigest} + if f.publicationOverride != nil { + publication = *f.publicationOverride + } + return publication, f.record("publish") +} +func machineTestRunner(t *testing.T, f *machineFixture) *machineRunnerFixture { + t.Helper() + return &machineRunnerFixture{MachineBuildBackend: MachineBuildBackend{Driver: f, Publisher: f}, WorkDir: t.TempDir()} +} +func machineTestRequest() MachineBuildRequest { + return MachineBuildRequest{ID: "build-test", BaseImage: "localhost/macos@" + machineTestDigest} +} + +func TestMachineBuildPublicationOrder(t *testing.T) { + f := newMachineFixture() + r := machineTestRunner(t, f) + result, err := r.Run(context.Background(), machineTestRequest(), strings.NewReader("source")) + if err != nil { + t.Fatal(err) + } + expected := []string{"start", "provision", "sanitize", "stop", "export", "destroy", "publish"} + if !reflect.DeepEqual(f.calls, expected) { + t.Fatalf("wrong phase order: %v", f.calls) + } + hash := sha256.Sum256([]byte("source")) + if result.Provenance.SourceHash != hex.EncodeToString(hash[:]) || result.Provenance.BaseImageDigest != machineTestDigest { + t.Fatal("unverified provenance") + } + entries, err := os.ReadDir(r.WorkDir) + if err != nil || len(entries) != 0 { + t.Fatal("workspace not cleaned") + } +} + +func TestMachineBuildFailuresDoNotPublish(t *testing.T) { + for _, tc := range []struct { + name string + phase string + receipt *MachineStopReceipt + }{ + {name: "start", phase: "start"}, + {name: "provision", phase: "provision"}, + {name: "sanitize", phase: "sanitize"}, + {name: "stop", phase: "stop"}, + {name: "export", phase: "export"}, + {name: "destroy", phase: "destroy"}, + {name: "unconfirmed stop", receipt: &MachineStopReceipt{}}, + {name: "forced stop", receipt: &MachineStopReceipt{VMMExited: true}}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newMachineFixture() + f.fail = tc.phase + if tc.receipt != nil { + f.stop = *tc.receipt + } + _, err := runMachineBuildFailure(t, f, context.Background(), machineTestRequest(), strings.NewReader("source")) + if f.publisherCalled { + t.Fatal("failed build reached publication") + } + if tc.phase != "" && strings.Contains(err.Error(), "sensitive-command-argument") { + t.Fatal("raw guest error leaked") + } + for _, phase := range f.calls { + if phase == "export" && tc.receipt != nil { + t.Fatal("unconfirmed stop exported") + } + if phase == "destroy" && tc.receipt != nil && !tc.receipt.VMMExited { + t.Fatal("destroyed unconfirmed live storage") + } + } + }) + } +} + +func TestMachineBuildInputAdmission(t *testing.T) { + for _, kind := range []string{"floating base", "wrong digest", "linux", "pending", "shaping", "source hash", "CPU overflow", "memory precision"} { + t.Run(kind, func(t *testing.T) { + f := newMachineFixture() + req := machineTestRequest() + switch kind { + case "floating base": + req.BaseImage = "localhost/macos:latest" + case "wrong digest": + f.image.Digest = "sha256:" + strings.Repeat("b", 64) + case "linux": + f.image.Platform = "linux/arm64" + case "pending": + f.image.Status = images.StatusPending + case "shaping": + req.Policy.MemoryMB = 4096 + case "source hash": + req.SourceHash = strings.Repeat("0", 64) + case "CPU overflow": + f.image.MacOS.CPUs = ^uint(0) + case "memory precision": + f.image.MacOS.Memory++ + } + runMachineBuildFailure(t, f, context.Background(), req, strings.NewReader("source")) + for _, phase := range f.calls { + if phase == "start" { + t.Fatal("bad input started a VM") + } + } + }) + } +} + +var _ MachineBuildDriver = (*machineFixture)(nil) +var _ MachineBuildSession = (*machineFixture)(nil) +var _ MachineBuildPublisher = (*machineFixture)(nil) diff --git a/lib/builds/macos_delta_publish.go b/lib/builds/macos_delta_publish.go new file mode 100644 index 000000000..f42e2282a --- /dev/null +++ b/lib/builds/macos_delta_publish.go @@ -0,0 +1,173 @@ +package builds + +import ( + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "path/filepath" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/images" +) + +// maxMachineMetadataBytes bounds a base manifest or config fetched for a delta. +const maxMachineMetadataBytes = 4 << 20 + +// MachineDeltaBase is a pinned base that a derivative is published against. Its +// layers are mounted from Repo, so the base must already be in the registry the +// publisher writes to. +type MachineDeltaBase struct { + Repo string // builds/{id} repository holding the base manifest and layers + Digest string // canonical sha256 digest of the base manifest + DiskPath string // absolute path of the base boot disk on this host +} + +func (m MachineDeltaBase) validate() error { + if !strings.HasPrefix(m.Repo, "builds/") || strings.Contains(m.Repo, "..") || len(m.Repo) > 256 { + return fmt.Errorf("delta base repository must be under builds/") + } + if !ValidMachineDigest(m.Digest) { + return fmt.Errorf("delta base digest must be a canonical sha256 digest") + } + if !filepath.IsAbs(m.DiskPath) { + return fmt.Errorf("delta base disk path must be absolute") + } + return nil +} + +// machineLayers returns a machine image's layer blobs and their diff IDs. A +// standalone image has one uploaded layer. A derivative lists the base's layers, +// which the registry already holds, and uploads one delta layer that patches the +// base disk. +func machineLayers(ctx context.Context, client *http.Client, token, registry, bundle string, source *MachineDeltaBase) ([]machineBlob, []v1.Hash, error) { + if source == nil { + layer, err := machineTarballLayer(ctx, bundle) + if err != nil { + return nil, nil, err + } + diffID, err := layer.DiffID() + if err != nil { + return nil, nil, err + } + digest, err := layer.Digest() + if err != nil { + return nil, nil, err + } + size, err := layer.Size() + if err != nil { + return nil, nil, err + } + blob := machineBlob{open: layer.Compressed, descriptor: v1.Descriptor{MediaType: types.OCILayer, Digest: digest, Size: size}} + return []machineBlob{blob}, []v1.Hash{diffID}, nil + } + baseLayers, baseDiffIDs, err := fetchMachineSource(ctx, client, token, registry, source) + if err != nil { + return nil, nil, err + } + blobs := make([]machineBlob, 0, len(baseLayers)+1) + for _, descriptor := range baseLayers { + blobs = append(blobs, machineBlob{descriptor: descriptor}) + } + diskPath := filepath.Join(bundle, "disk.img") + delta, err := images.ComputeMacOSDelta(source.DiskPath, diskPath) + if err != nil { + return nil, nil, fmt.Errorf("compute disk delta: %w", err) + } + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { + reader, writer := io.Pipe() + go func() { writer.CloseWithError(images.WriteMacOSDeltaTar(writer, delta, diskPath, bundle)) }() + return reader, nil + }, tarball.WithMediaType(images.MacOSDeltaMediaType), tarball.WithCompressionLevel(gzip.DefaultCompression)) + if err != nil { + return nil, nil, err + } + diffID, err := layer.DiffID() + if err != nil { + return nil, nil, err + } + digest, err := layer.Digest() + if err != nil { + return nil, nil, err + } + size, err := layer.Size() + if err != nil { + return nil, nil, err + } + blobs = append(blobs, machineBlob{open: layer.Compressed, descriptor: v1.Descriptor{MediaType: images.MacOSDeltaMediaType, Digest: digest, Size: size}}) + return blobs, append(baseDiffIDs, diffID), nil +} + +// fetchMachineSource reads the base manifest and config from the registry, checking +// each against its digest. It returns the base's layer descriptors and diff IDs. +func fetchMachineSource(ctx context.Context, client *http.Client, token, registry string, source *MachineDeltaBase) ([]v1.Descriptor, []v1.Hash, error) { + root := strings.TrimRight(registry, "/") + "/v2/" + source.Repo + manifestBytes, err := machineFetch(ctx, client, token, root+"/manifests/"+source.Digest, source.Digest, 0) + if err != nil { + return nil, nil, fmt.Errorf("base manifest: %w", err) + } + var manifest v1.Manifest + if err := json.Unmarshal(manifestBytes, &manifest); err != nil { + return nil, nil, fmt.Errorf("base manifest: %w", err) + } + configDigest := manifest.Config.Digest.String() + configBytes, err := machineFetch(ctx, client, token, root+"/blobs/"+configDigest, configDigest, 0) + if err != nil { + return nil, nil, fmt.Errorf("base config: %w", err) + } + var config v1.ConfigFile + if err := json.Unmarshal(configBytes, &config); err != nil { + return nil, nil, fmt.Errorf("base config: %w", err) + } + if config.OS != "darwin" || config.Architecture != "arm64" || len(manifest.Layers) == 0 || len(config.RootFS.DiffIDs) != len(manifest.Layers) { + return nil, nil, fmt.Errorf("base is not a darwin/arm64 machine image with matching layers") + } + return manifest.Layers, config.RootFS.DiffIDs, nil +} + +// machineDeltaBase returns the base as a delta source when the publisher writes to +// the registry that stores the base. Any other base publishes standalone. +func (m *manager) machineDeltaBase(publisher MachineBuildPublisher, baseImage string, base *images.Image) *MachineDeltaBase { + publisherWithRegistry, ok := publisher.(*StreamingMachinePublisher) + if !ok || base == nil { + return nil + } + address := publisherWithRegistry.RegistryURL + if !strings.Contains(address, "://") { + address = "https://" + address + } + registry, err := url.Parse(address) + if err != nil || registry.Host == "" { + return nil + } + ref, err := images.ParseNormalizedRef(baseImage) + if err != nil || !ref.IsDigest() { + return nil + } + prefix := registry.Host + "/" + if !strings.HasPrefix(ref.Repository(), prefix) { + return nil + } + disk, err := images.GetDiskPath(m.paths, baseImage, base.Digest) + if err != nil { + return nil + } + return &MachineDeltaBase{Repo: strings.TrimPrefix(ref.Repository(), prefix), Digest: base.Digest, DiskPath: disk} +} + +// publishBuild publishes bundle as a derivative when the base is a registry source, +// and as a standalone image otherwise. +func (r *MachineBuildBackend) publishBuild(ctx context.Context, req MachineBuildRequest, bundle string) (MachinePublication, error) { + if derived, ok := r.Publisher.(interface { + PublishDerived(context.Context, string, string, MachineDeltaBase) (MachinePublication, error) + }); ok && req.deltaBase != nil { + return derived.PublishDerived(ctx, req.ID, bundle, *req.deltaBase) + } + return r.Publisher.Publish(ctx, req.ID, bundle) +} diff --git a/lib/builds/macos_delta_publish_test.go b/lib/builds/macos_delta_publish_test.go new file mode 100644 index 000000000..743f6167f --- /dev/null +++ b/lib/builds/macos_delta_publish_test.go @@ -0,0 +1,160 @@ +package builds_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/builds" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/registry" + "github.com/stretchr/testify/require" +) + +const deltaTestSecret = "synthetic registry secret" + +type deltaTestRegistry struct { + server *httptest.Server + tokens *builds.RegistryTokenGenerator + uploads atomic.Int32 + puts atomic.Int32 +} + +func newDeltaTestRegistry(t *testing.T) *deltaTestRegistry { + t.Helper() + imports := &machineImportStub{imported: make(chan string, 16)} + reg, err := registry.New(paths.New(t.TempDir()), imports) + require.NoError(t, err) + dr := &deltaTestRegistry{tokens: builds.NewRegistryTokenGenerator(deltaTestSecret)} + handler := middleware.JwtAuth(deltaTestSecret)(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPost && r.URL.Query().Get("digest") != "" { + dr.uploads.Add(1) + } + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/manifests/") { + dr.puts.Add(1) + } + reg.Handler().ServeHTTP(w, r) + })) + dr.server = httptest.NewServer(handler) + t.Cleanup(dr.server.Close) + // Each manifest PUT starts a conversion goroutine that writes the OCI layout + // after the response. Its import call is the last step, so wait for every one + // before the TempDir cleanup removes the registry's data. + t.Cleanup(func() { + for range dr.puts.Load() { + select { + case <-imports.imported: + case <-time.After(10 * time.Second): + return + } + } + }) + return dr +} + +// deltaTestDisk is a disk of eight 64 KiB chunks, each a distinct byte. +func deltaTestDisk() []byte { + disk := make([]byte, 8*64<<10) + for i := range 8 { + for j := range 64 << 10 { + disk[i*64<<10+j] = byte(i + 1) + } + } + return disk +} + +func deltaTestBundle(t *testing.T, disk []byte, aux string) string { + t.Helper() + bundle := machinePublisherBundle(t) + require.NoError(t, os.WriteFile(filepath.Join(bundle, "disk.img"), disk, 0600)) + require.NoError(t, os.WriteFile(filepath.Join(bundle, "aux.img"), []byte(aux), 0600)) + return bundle +} + +// pullManifest fetches a manifest from the registry with a pull-scoped token. +func (dr *deltaTestRegistry) pullManifest(t *testing.T, repo, digest string) v1.Manifest { + t.Helper() + token, err := dr.tokens.GenerateToken("reader", []builds.RepoPermission{{Repo: repo, Scope: "pull"}}, time.Minute) + require.NoError(t, err) + request, err := http.NewRequest(http.MethodGet, dr.server.URL+"/v2/"+repo+"/manifests/"+digest, nil) + require.NoError(t, err) + request.Header.Set("Authorization", "Bearer "+token) + request.Header.Set("Accept", string(types.OCIManifestSchema1)) + response, err := http.DefaultClient.Do(request) + require.NoError(t, err) + defer response.Body.Close() + require.Equal(t, http.StatusOK, response.StatusCode) + var manifest v1.Manifest + require.NoError(t, json.NewDecoder(response.Body).Decode(&manifest)) + return manifest +} + +func TestStreamingMachinePublisherDerivedReusesStoredBaseLayers(t *testing.T) { + dr := newDeltaTestRegistry(t) + publisher := &builds.StreamingMachinePublisher{RegistryURL: dr.server.URL, Tokens: dr.tokens} + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + baseBundle := deltaTestBundle(t, deltaTestDisk(), "base aux") + base, err := publisher.Publish(ctx, "base-x", baseBundle) + require.NoError(t, err) + baseManifest := dr.pullManifest(t, "builds/base-x", base.Digest) + + derived := deltaTestDisk() + for j := range 64 << 10 { + derived[3*64<<10+j] = 0xAA + } + derivedBundle := deltaTestBundle(t, derived, "derived aux") + dr.uploads.Store(0) + result, err := publisher.PublishDerived(ctx, "derived-x", derivedBundle, builds.MachineDeltaBase{ + Repo: "builds/base-x", + Digest: base.Digest, + DiskPath: filepath.Join(baseBundle, "disk.img"), + }) + require.NoError(t, err) + require.Equal(t, int32(2), dr.uploads.Load(), "only the config and the delta layer are uploaded; the base layer is already stored") + + derivedManifest := dr.pullManifest(t, "builds/derived-x", result.Digest) + require.Len(t, derivedManifest.Layers, 2) + require.Equal(t, baseManifest.Layers[0].Digest, derivedManifest.Layers[0].Digest) + require.Equal(t, images.MacOSDeltaMediaType, derivedManifest.Layers[1].MediaType) +} + +func TestStreamingMachinePublisherDerivedRejectsMissingBase(t *testing.T) { + dr := newDeltaTestRegistry(t) + publisher := &builds.StreamingMachinePublisher{RegistryURL: dr.server.URL, Tokens: dr.tokens} + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + bundle := deltaTestBundle(t, deltaTestDisk(), "aux") + missing := fmt.Sprintf("sha256:%064x", 1) + _, err := publisher.PublishDerived(ctx, "derived-y", bundle, builds.MachineDeltaBase{ + Repo: "builds/never-published", + Digest: missing, + DiskPath: filepath.Join(bundle, "disk.img"), + }) + require.Error(t, err) +} + +func TestStreamingMachinePublisherDerivedRejectsNonBaseRepo(t *testing.T) { + dr := newDeltaTestRegistry(t) + publisher := &builds.StreamingMachinePublisher{RegistryURL: dr.server.URL, Tokens: dr.tokens} + bundle := deltaTestBundle(t, deltaTestDisk(), "aux") + _, err := publisher.PublishDerived(context.Background(), "derived-z", bundle, builds.MachineDeltaBase{ + Repo: "library/alpine", + Digest: fmt.Sprintf("sha256:%064x", 2), + DiskPath: filepath.Join(bundle, "disk.img"), + }) + require.ErrorContains(t, err, "under builds/") +} diff --git a/lib/builds/macos_registry.go b/lib/builds/macos_registry.go new file mode 100644 index 000000000..078255fef --- /dev/null +++ b/lib/builds/macos_registry.go @@ -0,0 +1,99 @@ +package builds + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// machineRegistryRequest sends one authenticated registry request. The caller +// closes the response body. +func machineRegistryRequest(ctx context.Context, client *http.Client, token, method, target, media string, body io.Reader, size int64) (*http.Response, error) { + request, err := http.NewRequestWithContext(ctx, method, target, body) + if err != nil { + return nil, err + } + request.ContentLength = size + request.Header.Set("Authorization", "Bearer "+token) + if media != "" { + request.Header.Set("Content-Type", media) + } + request.Header.Set("Accept", string(types.OCIManifestSchema1)) + response, err := client.Do(request) + if err != nil { + return nil, fmt.Errorf("machine registry request failed") + } + return response, nil +} + +func machineHash(data []byte) v1.Hash { + sum := sha256.Sum256(data) + return v1.Hash{Algorithm: "sha256", Hex: hex.EncodeToString(sum[:])} +} + +// machineFetch GETs a registry object and checks it against digest. A size of 0 +// marks metadata: the body is buffered up to maxMachineMetadataBytes and returned. +// Any other size must match exactly and is only hashed, never kept, so layers of +// tens of gigabytes are not held in memory. +func machineFetch(ctx context.Context, client *http.Client, token, target, digest string, size int64) ([]byte, error) { + response, err := machineRegistryRequest(ctx, client, token, http.MethodGet, target, "", nil, 0) + if err != nil { + return nil, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("registry returned status %d", response.StatusCode) + } + metadata := size == 0 + limit := size + 1 + if metadata { + limit = maxMachineMetadataBytes + 1 + } + sum := sha256.New() + var kept bytes.Buffer + out := io.Writer(sum) + if metadata { + out = io.MultiWriter(sum, &kept) + } + n, err := io.Copy(out, io.LimitReader(response.Body, limit)) + if err != nil { + return nil, err + } + if n >= limit || (!metadata && n != size) { + return nil, fmt.Errorf("object does not match digest %s", digest) + } + if (v1.Hash{Algorithm: "sha256", Hex: hex.EncodeToString(sum.Sum(nil))}).String() != digest { + return nil, fmt.Errorf("object does not match digest %s", digest) + } + if !metadata { + return nil, nil + } + return kept.Bytes(), nil +} + +// machineBlobStored reports whether the registry already holds a blob of the +// descriptor's digest and size. The store is shared by digest across repositories. +func machineBlobStored(ctx context.Context, client *http.Client, token, base string, descriptor v1.Descriptor) (bool, error) { + head, err := machineRegistryRequest(ctx, client, token, http.MethodHead, base+"/blobs/"+descriptor.Digest.String(), "", nil, 0) + if err != nil { + return false, err + } + head.Body.Close() + switch head.StatusCode { + case http.StatusOK: + if head.ContentLength != descriptor.Size { + return false, fmt.Errorf("stored blob %s has size %d, want %d", descriptor.Digest, head.ContentLength, descriptor.Size) + } + return true, nil + case http.StatusNotFound: + return false, nil + } + return false, fmt.Errorf("blob check returned status %d", head.StatusCode) +} diff --git a/lib/builds/macos_registry_test.go b/lib/builds/macos_registry_test.go new file mode 100644 index 000000000..77ccbac79 --- /dev/null +++ b/lib/builds/macos_registry_test.go @@ -0,0 +1,54 @@ +package builds + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestMachineFetchVerifiesSizeAndDigest(t *testing.T) { + body := []byte("synthetic object") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write(body) + })) + defer server.Close() + digest := machineHash(body).String() + oversize := strings.Repeat("x", maxMachineMetadataBytes+1) + bigServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte(oversize)) + })) + defer bigServer.Close() + + for _, tc := range []struct { + name string + target string + digest string + size int64 + ok bool + }{ + {"metadata matches digest", server.URL, digest, 0, true}, + {"sized object matches", server.URL, digest, int64(len(body)), true}, + {"wrong digest", server.URL, machineHash([]byte("other")).String(), 0, false}, + {"size shorter than body", server.URL, digest, int64(len(body)) - 1, false}, + {"size longer than body", server.URL, digest, int64(len(body)) + 1, false}, + {"metadata over cap", bigServer.URL, machineHash([]byte(oversize)).String(), 0, false}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := machineFetch(context.Background(), http.DefaultClient, "token", tc.target, tc.digest, tc.size) + if !tc.ok { + require.Error(t, err) + return + } + require.NoError(t, err) + if tc.size == 0 { + require.Equal(t, body, got, "metadata is returned") + } else { + require.Nil(t, got, "sized objects are verified, never buffered") + } + }) + } +} diff --git a/lib/builds/manager.go b/lib/builds/manager.go index 806ae5622..3584484bb 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -87,6 +87,12 @@ type Manager interface { // Config holds configuration for the build manager type Config struct { + // MachineBuild is the opt-in native macOS build backend. + MachineBuild *MachineBuildBackend + + // MachineBuildOnly disables Linux builder preparation and recovery. + MachineBuildOnly bool + // MaxConcurrentBuilds is the maximum number of concurrent builds MaxConcurrentBuilds int @@ -149,6 +155,7 @@ type manager struct { logger *slog.Logger metrics *Metrics createMu sync.Mutex + machineBuilds sync.Map // build ID -> *machineBuildControl builderReady atomic.Bool // pendingRecovered is set once RecoverPendingBuilds has re-enqueued // persisted pending builds; until then BuilderHasBuilds also scans @@ -204,6 +211,10 @@ func NewManager( // Start starts the build manager's background services func (m *manager) Start(ctx context.Context) error { + if m.config.MachineBuildOnly { + m.RecoverPendingBuilds() + return nil + } go func() { for { m.ensureBuilderImage(ctx) @@ -444,7 +455,12 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc // Apply defaults to build policy policy := req.BuildPolicy - if policy == nil { + if machineBuild(&req) { + var err error + if policy, err = m.admitMachineBuild(&req, sourceData); err != nil { + return nil, err + } + } else if policy == nil { defaultPolicy := DefaultBuildPolicy() policy = &defaultPolicy } else { @@ -481,6 +497,15 @@ func (m *manager) CreateBuild(ctx context.Context, req CreateBuildRequest, sourc CreatedAt: time.Now(), } + if machineBuild(&req) { + // Stage verified input before making the job visible to recovery. + if err := m.storeMachineSource(ctx, meta, sourceData); err != nil { + deleteBuild(m.paths, id) + return nil, err + } + return m.enqueueMachineBuild(meta, req, policy), nil + } + // Write initial metadata if err := writeMetadata(m.paths, meta); err != nil { return nil, fmt.Errorf("write metadata: %w", err) @@ -678,18 +703,29 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques start := time.Now() m.logger.Info("starting build", "id", id) - // Update status to building - m.updateStatus(id, StatusBuilding, nil) - - // Create timeout context + // Register machine cancellation before exposing the running status. Ordinary + // Linux builder cancellation retains its existing instance-delete behavior. buildCtx, cancel := context.WithTimeout(ctx, time.Duration(policy.TimeoutSeconds)*time.Second) defer cancel() + if machineBuild(&req) { + m.machineBuilds.Store(id, &machineBuildControl{cancel: cancel}) + defer m.machineBuilds.Delete(id) + defer func() { + meta, err := readMetadata(m.paths, id) + if err == nil && (meta.Status == StatusReady || meta.Status == StatusFailed || meta.Status == StatusCancelled) { + m.removeMachineSource(id) + } + }() + } + m.updateStatus(id, StatusBuilding, nil) // Mirror base images to the local registry before launching the VM. // BuildKit is configured with our registry as a mirror for docker.io, // so pre-cached images will be served locally without pulling from Docker Hub. - if err := m.mirrorBaseImagesForBuild(buildCtx, id, req); err != nil { - m.logger.Warn("failed to mirror base images", "id", id, "error", err) + if !machineBuild(&req) { + if err := m.mirrorBaseImagesForBuild(buildCtx, id, req); err != nil { + m.logger.Warn("failed to mirror base images", "id", id, "error", err) + } } // Run the build in a builder VM @@ -754,6 +790,9 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques // If image_name is set, re-tag the image so it's accessible by that name. // The builder pushed to builds/{id} but the user wants it as image_name. imageRef := buildRepo + if machineBuild(&req) && result.ImageRef != "" { + imageRef = result.ImageRef + } if req.ImageName != "" { ref, err := images.ParseNormalizedRef(req.ImageName) if err != nil { @@ -798,6 +837,9 @@ func (m *manager) runBuild(ctx context.Context, id string, req CreateBuildReques // executeBuild runs the build in a builder VM func (m *manager) executeBuild(ctx context.Context, id string, req CreateBuildRequest, policy *BuildPolicy) (*BuildResult, error) { + if machineBuild(&req) { + return m.executeMachineBuild(ctx, id, req, policy) + } if !m.builderReady.Load() { return nil, fmt.Errorf("builder image is being prepared, please retry shortly") } @@ -1106,6 +1148,11 @@ func (m *manager) updateStatus(id string, status string, err error) { // updateBuildComplete updates the build with final results func (m *manager) updateBuildComplete(id string, status string, digest *string, errMsg *string, provenance *BuildProvenance, durationMS *int64) { + if active, ok := m.machineBuilds.Load(id); ok { + control := active.(*machineBuildControl) + control.mu.Lock() + defer control.mu.Unlock() + } meta, readErr := readMetadata(m.paths, id) if readErr != nil { m.logger.Error("read metadata for completion", "id", id, "error", readErr) @@ -1246,12 +1293,18 @@ func (m *manager) CancelBuild(ctx context.Context, id string) error { case StatusQueued: // Remove from queue if m.queue.Cancel(id) { + if machineBuild(meta.Request) { + return m.cancelQueuedMachineBuild(meta) + } m.updateStatus(id, StatusCancelled, nil) return nil } return ErrBuildInProgress // Was already picked up case StatusBuilding, StatusPushing: + if machineBuild(meta.Request) { + return m.cancelMachineBuild(id) + } // Can't cancel a running build easily // Would need to terminate the builder instance if meta.BuilderInstance != nil { @@ -1449,15 +1502,36 @@ func (m *manager) StreamBuildEvents(ctx context.Context, id string, follow bool) // RecoverPendingBuilds recovers builds that were interrupted on restart func (m *manager) RecoverPendingBuilds() { - pending, err := listPendingBuilds(m.paths) + all, err := listAllBuilds(m.paths) if err != nil { - m.logger.Error("list pending builds for recovery", "error", err) + m.logger.Error("list builds for machine cleanup", "error", err) return } + for _, meta := range all { + if !machineBuild(meta.Request) { + continue + } + switch meta.Status { + case StatusReady, StatusFailed: + m.removeMachineSource(meta.ID) + case StatusCancelled: + if m.config.MachineBuild != nil { + if err := m.reapPriorMachineBuilder(context.Background(), meta.ID); err != nil { + m.logger.Error("failed to reap cancelled machine build", "id", meta.ID, "error", err) + continue + } + } + m.removeMachineSource(meta.ID) + } + } + pending := pendingBuilds(all) for _, meta := range pending { meta := meta // Shadow loop variable for closure capture m.logger.Info("recovering build", "id", meta.ID, "status", meta.Status) + if m.skipMachineBuildRecovery(meta) { + continue + } // Re-enqueue the build if meta.Request != nil { @@ -1485,11 +1559,16 @@ func (m *manager) RecoverPendingBuilds() { if len(pending) > 0 { m.logger.Info("recovered pending builds", "count", len(pending)) } - m.pendingRecovered.Store(true) + if !m.config.MachineBuildOnly { + m.pendingRecovered.Store(true) + } } // refreshBuildToken regenerates the registry token for a build and updates the config file func (m *manager) refreshBuildToken(buildID string, req *CreateBuildRequest) error { + if machineBuild(req) { + return nil // Machine builds have no builder config or registry token. + } // Read existing build config config, err := readBuildConfig(m.paths, buildID) if err != nil { diff --git a/lib/builds/source.go b/lib/builds/source.go new file mode 100644 index 000000000..b2075607f --- /dev/null +++ b/lib/builds/source.go @@ -0,0 +1,75 @@ +package builds + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "os" +) + +const maxBuildSourceBytes = 64 << 20 + +type buildContextReader struct { + ctx context.Context + r io.Reader +} + +func (r buildContextReader) Read(p []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.r.Read(p) +} + +// stageBuildSource streams compressed source once into a private, exclusive job +// file. Both backends consume this file and use its hash for input verification. +func stageBuildSource(ctx context.Context, source io.Reader, path string, limit int64) (hash string, err error) { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return "", err + } + defer func() { + if closeErr := file.Close(); err == nil { + err = closeErr + } + }() + h := sha256.New() + n, err := io.Copy(io.MultiWriter(file, h), io.LimitReader(buildContextReader{ctx, source}, limit+1)) + if err != nil { + return "", err + } + if n > limit { + return "", fmt.Errorf("build source too large") + } + if err = file.Sync(); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +// verifyBuildSource rejects missing/replaced source on recovery before a machine +// is launched. It never writes a second source copy. +func verifyBuildSource(ctx context.Context, path, expected string) error { + info, err := os.Lstat(path) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() > maxBuildSourceBytes { + return fmt.Errorf("invalid staged source") + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + h := sha256.New() + if _, err = io.Copy(h, io.LimitReader(buildContextReader{ctx, file}, maxBuildSourceBytes+1)); err != nil { + return err + } + if hex.EncodeToString(h.Sum(nil)) != expected { + return ErrSourceHashMismatch + } + return nil +} diff --git a/lib/builds/source_test.go b/lib/builds/source_test.go new file mode 100644 index 000000000..1bb7e8a84 --- /dev/null +++ b/lib/builds/source_test.go @@ -0,0 +1,36 @@ +package builds + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestSharedBuildSourceStagingAndRecoveryVerification(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "source.tar.gz") + hash, err := stageBuildSource(ctx, strings.NewReader("source"), path, maxBuildSourceBytes) + require.NoError(t, err) + info, err := os.Stat(path) + require.NoError(t, err) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) + require.NoError(t, verifyBuildSource(ctx, path, hash)) + _, err = stageBuildSource(ctx, strings.NewReader("replacement"), path, maxBuildSourceBytes) + require.Error(t, err, "exclusive staging must preserve existing source") + require.NoError(t, verifyBuildSource(ctx, path, hash)) + canceled, cancel := context.WithCancel(ctx) + cancel() + require.ErrorIs(t, verifyBuildSource(canceled, path, hash), context.Canceled) + require.NoError(t, os.WriteFile(path, []byte("changed"), 0600)) + require.ErrorIs(t, verifyBuildSource(ctx, path, hash), ErrSourceHashMismatch) + require.NoError(t, os.Remove(path)) + require.Error(t, verifyBuildSource(ctx, path, hash)) + target := filepath.Join(filepath.Dir(path), "target") + require.NoError(t, os.WriteFile(target, []byte("source"), 0600)) + require.NoError(t, os.Symlink(target, path)) + require.ErrorContains(t, verifyBuildSource(ctx, path, hash), "invalid staged source") +} diff --git a/lib/builds/storage.go b/lib/builds/storage.go index ce555aac9..1add7a3c1 100644 --- a/lib/builds/storage.go +++ b/lib/builds/storage.go @@ -144,7 +144,10 @@ func listPendingBuilds(p *paths.Paths) ([]*buildMetadata, error) { if err != nil { return nil, err } + return pendingBuilds(all), nil +} +func pendingBuilds(all []*buildMetadata) []*buildMetadata { var pending []*buildMetadata for _, meta := range all { switch meta.Status { @@ -152,13 +155,10 @@ func listPendingBuilds(p *paths.Paths) ([]*buildMetadata, error) { pending = append(pending, meta) } } - - // Sort by created_at ascending (oldest first for FIFO recovery) sort.Slice(pending, func(i, j int) bool { return pending[i].CreatedAt.Before(pending[j].CreatedAt) }) - - return pending, nil + return pending } // deleteBuild removes a build's data from disk diff --git a/lib/builds/types.go b/lib/builds/types.go index 4ad6008e9..571259328 100644 --- a/lib/builds/types.go +++ b/lib/builds/types.go @@ -43,6 +43,10 @@ type CreateBuildRequest struct { // The Dockerfile specifies the runtime (e.g., FROM node:20-alpine). Dockerfile string `json:"dockerfile,omitempty"` + // MachineBaseImage selects the installed-machine backend. The HTTP multipart + // field machine_base_image maps here; it is enabled only with macos_builds. + MachineBaseImage string `json:"machine_base_image,omitempty"` + // BaseImageDigest optionally pins the base image by digest for reproducibility BaseImageDigest string `json:"base_image_digest,omitempty"` @@ -219,6 +223,9 @@ type BuildResult struct { // ImageDigest is the digest of the pushed image (only on success) ImageDigest string `json:"image_digest,omitempty"` + // ImageRef is the pullable output reference for machine builds. + ImageRef string `json:"image_ref,omitempty"` + // Error is the error message (only on failure) Error string `json:"error,omitempty"` diff --git a/lib/images/macos_bundle.go b/lib/images/macos_bundle.go index 5e41bb8e7..73b8a4e0a 100644 --- a/lib/images/macos_bundle.go +++ b/lib/images/macos_bundle.go @@ -2,10 +2,12 @@ package images import ( "bytes" + "context" "encoding/json" "fmt" "io" "os" + "os/exec" "path/filepath" ) @@ -19,6 +21,19 @@ type macOSMachinePayload struct { Platform *MacOSImage } +// RequireClosedFile fails unless no process holds path open. lsof exits 1 when +// nothing holds the file; any other failure leaves the storage unverified. +func RequireClosedFile(ctx context.Context, path string) error { + err := exec.CommandContext(ctx, "/usr/sbin/lsof", "-t", path).Run() + if err == nil { + return fmt.Errorf("storage is open: stop its VM first") + } + if exit, ok := err.(*exec.ExitError); !ok || exit.ExitCode() != 1 { + return fmt.Errorf("cannot verify storage is closed: %w", err) + } + return nil +} + // ValidateMacOSBundle validates a fixed-layout export, rejecting unknown metadata // fields so an accidental credential field cannot become part of an image. func ValidateMacOSBundle(root string) (*MacOSImage, error) { diff --git a/lib/images/macos_delta.go b/lib/images/macos_delta.go new file mode 100644 index 000000000..17255ac55 --- /dev/null +++ b/lib/images/macos_delta.go @@ -0,0 +1,261 @@ +package images + +import ( + "archive/tar" + "bufio" + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// MacOSDeltaMediaType marks a machine layer that patches the boot disk unpacked +// from the layers beneath it. Its tar holds, in this order, delta.json, the changed +// chunks of disk.img, and aux.img and config.json in full. +const MacOSDeltaMediaType types.MediaType = "application/vnd.hypeman.machine.disk-delta.v1.tar+gzip" + +// MacOSDeltaChunkSize is the granularity at which a delta records changed bytes. +const MacOSDeltaChunkSize = 64 << 10 + +const ( + macOSDeltaVersion = 1 + macOSDeltaManifestEntry = "delta.json" + macOSDeltaPatchEntry = "disk.img.patch" + macOSDeltaDisk = "disk.img" + macOSDeltaAux = "aux.img" + macOSDeltaConfig = "config.json" + maxMacOSDeltaManifestBytes = 1 << 20 + macOSDeltaEntries = 4 +) + +// MacOSDelta lists the chunks of a boot disk that differ from the disk beneath it. +// Size is the disk's logical size, which the base disk must match. +type MacOSDelta struct { + Version int `json:"version"` + ChunkSize int64 `json:"chunk_size"` + Size int64 `json:"size"` + Chunks []int64 `json:"chunks"` +} + +// Validate checks a delta before any chunk is applied: a supported version and +// chunk size, a whole number of chunks, and changed chunks in ascending order. +func (d *MacOSDelta) Validate() error { + if d.Version != macOSDeltaVersion || d.ChunkSize != MacOSDeltaChunkSize { + return fmt.Errorf("unsupported machine disk delta version %d with chunk size %d", d.Version, d.ChunkSize) + } + if d.Size <= 0 || d.Size%d.ChunkSize != 0 { + return fmt.Errorf("machine disk delta size %d is not a positive multiple of %d", d.Size, d.ChunkSize) + } + count := d.Size / d.ChunkSize + previous := int64(-1) + for _, index := range d.Chunks { + if index <= previous || index >= count { + return fmt.Errorf("machine disk delta chunk %d is out of order or out of range", index) + } + previous = index + } + return nil +} + +// ComputeMacOSDelta compares a derived boot disk with its base chunk by chunk. Both +// files must be the same size and a whole number of chunks. +func ComputeMacOSDelta(basePath, derivedPath string) (MacOSDelta, error) { + base, err := os.Open(basePath) + if err != nil { + return MacOSDelta{}, err + } + defer base.Close() + derived, err := os.Open(derivedPath) + if err != nil { + return MacOSDelta{}, err + } + defer derived.Close() + baseInfo, err := base.Stat() + if err != nil { + return MacOSDelta{}, err + } + derivedInfo, err := derived.Stat() + if err != nil { + return MacOSDelta{}, err + } + if baseInfo.Size() != derivedInfo.Size() { + return MacOSDelta{}, fmt.Errorf("derived disk is %d bytes, base disk is %d", derivedInfo.Size(), baseInfo.Size()) + } + delta := MacOSDelta{Version: macOSDeltaVersion, ChunkSize: MacOSDeltaChunkSize, Size: derivedInfo.Size(), Chunks: []int64{}} + if err := delta.Validate(); err != nil { + return MacOSDelta{}, err + } + baseReader := bufio.NewReaderSize(base, 4<<20) + derivedReader := bufio.NewReaderSize(derived, 4<<20) + baseChunk := make([]byte, MacOSDeltaChunkSize) + derivedChunk := make([]byte, MacOSDeltaChunkSize) + for index := int64(0); index*MacOSDeltaChunkSize < delta.Size; index++ { + if _, err := io.ReadFull(baseReader, baseChunk); err != nil { + return MacOSDelta{}, fmt.Errorf("read base disk: %w", err) + } + if _, err := io.ReadFull(derivedReader, derivedChunk); err != nil { + return MacOSDelta{}, fmt.Errorf("read derived disk: %w", err) + } + if !bytes.Equal(baseChunk, derivedChunk) { + delta.Chunks = append(delta.Chunks, index) + } + } + return delta, nil +} + +// WriteMacOSDeltaTar writes a delta layer's tar. The changed chunks are read from +// derived, which must be the disk the delta was computed against. aux.img and +// config.json are copied in full from bundle. +func WriteMacOSDeltaTar(w io.Writer, delta MacOSDelta, derived, bundle string) error { + if err := delta.Validate(); err != nil { + return err + } + manifest, err := json.Marshal(delta) + if err != nil { + return err + } + disk, err := os.Open(derived) + if err != nil { + return err + } + defer disk.Close() + archive := tar.NewWriter(w) + if err := archive.WriteHeader(&tar.Header{Name: macOSDeltaManifestEntry, Mode: 0600, Size: int64(len(manifest)), Typeflag: tar.TypeReg}); err != nil { + return err + } + if _, err := archive.Write(manifest); err != nil { + return err + } + patchSize := int64(len(delta.Chunks)) * delta.ChunkSize + if err := archive.WriteHeader(&tar.Header{Name: macOSDeltaPatchEntry, Mode: 0600, Size: patchSize, Typeflag: tar.TypeReg}); err != nil { + return err + } + chunk := make([]byte, delta.ChunkSize) + for _, index := range delta.Chunks { + if _, err := disk.ReadAt(chunk, index*delta.ChunkSize); err != nil { + return fmt.Errorf("read changed chunk %d: %w", index, err) + } + if _, err := archive.Write(chunk); err != nil { + return err + } + } + for _, name := range []string{macOSDeltaAux, macOSDeltaConfig} { + if err := writeMacOSDeltaFile(archive, filepath.Join(bundle, name), name); err != nil { + return err + } + } + return archive.Close() +} + +func writeMacOSDeltaFile(archive *tar.Writer, path, name string) error { + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + info, err := file.Stat() + if err != nil { + return err + } + if err := archive.WriteHeader(&tar.Header{Name: name, Mode: 0600, Size: info.Size(), Typeflag: tar.TypeReg}); err != nil { + return err + } + _, err = io.Copy(archive, file) + return err +} + +// macOSDeltaApplier applies the entries of a delta layer to the boot disk already +// unpacked into targetDir. The entries must arrive in the order WriteMacOSDeltaTar +// writes them, so a layer cannot name files outside the format. +type macOSDeltaApplier struct { + targetDir string + step int + manifest *MacOSDelta +} + +func (a *macOSDeltaApplier) entry(name string, size int64, archive io.Reader) error { + switch a.step { + case 0: + if name != macOSDeltaManifestEntry { + return fmt.Errorf("delta layer must start with %s, got %q", macOSDeltaManifestEntry, name) + } + if size > maxMacOSDeltaManifestBytes { + return fmt.Errorf("machine disk delta manifest is %d bytes", size) + } + decoder := json.NewDecoder(io.LimitReader(archive, size)) + decoder.DisallowUnknownFields() + var delta MacOSDelta + if err := decoder.Decode(&delta); err != nil { + return fmt.Errorf("decode machine disk delta: %w", err) + } + if err := delta.Validate(); err != nil { + return err + } + a.manifest = &delta + case 1: + if name != macOSDeltaPatchEntry { + return fmt.Errorf("delta layer entry %q must be %s", name, macOSDeltaPatchEntry) + } + if size != int64(len(a.manifest.Chunks))*a.manifest.ChunkSize { + return fmt.Errorf("machine disk patch is %d bytes, want %d", size, int64(len(a.manifest.Chunks))*a.manifest.ChunkSize) + } + if err := a.applyChunks(archive); err != nil { + return err + } + case 2, 3: + want := macOSDeltaAux + if a.step == 3 { + want = macOSDeltaConfig + } + if name != want { + return fmt.Errorf("delta layer entry %q must be %s", name, want) + } + if err := writeSparseFile(filepath.Join(a.targetDir, name), archive, size); err != nil { + return err + } + default: + return fmt.Errorf("delta layer has more than %d entries", macOSDeltaEntries) + } + a.step++ + return nil +} + +// applyChunks writes each changed chunk over the base disk in place. The base must +// be a regular file of the delta's size, so the offsets land where the diff computed. +func (a *macOSDeltaApplier) applyChunks(r io.Reader) error { + path := filepath.Join(a.targetDir, macOSDeltaDisk) + info, err := os.Lstat(path) + if err != nil { + return fmt.Errorf("delta has no base disk: %w", err) + } + if !info.Mode().IsRegular() || info.Size() != a.manifest.Size { + return fmt.Errorf("base disk does not match the delta's %d-byte size", a.manifest.Size) + } + file, err := os.OpenFile(path, os.O_WRONLY, 0) + if err != nil { + return err + } + buf := make([]byte, a.manifest.ChunkSize) + for _, index := range a.manifest.Chunks { + if _, err = io.ReadFull(r, buf); err != nil { + break + } + if _, err = file.WriteAt(buf, index*a.manifest.ChunkSize); err != nil { + break + } + } + return errors.Join(err, file.Close()) +} + +// complete reports an error if the layer ended before its last entry. +func (a *macOSDeltaApplier) complete() error { + if a.step != macOSDeltaEntries { + return fmt.Errorf("delta layer has %d of %d entries", a.step, macOSDeltaEntries) + } + return nil +} diff --git a/lib/images/macos_delta_test.go b/lib/images/macos_delta_test.go new file mode 100644 index 000000000..4e0227e85 --- /dev/null +++ b/lib/images/macos_delta_test.go @@ -0,0 +1,203 @@ +package images + +import ( + "archive/tar" + "bytes" + "context" + "io" + "os" + "path/filepath" + "testing" + + gcr "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/stretchr/testify/require" +) + +type machineTestFile struct { + name string + data []byte +} + +// machineFilesLayer builds a layer of regular files with the given media type. +func machineFilesLayer(t *testing.T, mediaType types.MediaType, files ...machineTestFile) gcr.Layer { + t.Helper() + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, f := range files { + require.NoError(t, tw.WriteHeader(&tar.Header{Name: f.name, Mode: 0600, Size: int64(len(f.data)), Typeflag: tar.TypeReg})) + _, err := tw.Write(f.data) + require.NoError(t, err) + } + require.NoError(t, tw.Close()) + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(buf.Bytes())), nil }, tarball.WithMediaType(mediaType)) + require.NoError(t, err) + return layer +} + +// deltaTestDisk returns a disk of n chunks, each filled with a distinct byte. +func deltaTestDisk(n int) []byte { + disk := make([]byte, n*MacOSDeltaChunkSize) + for i := range n { + for j := range MacOSDeltaChunkSize { + disk[i*MacOSDeltaChunkSize+j] = byte(i + 1) + } + } + return disk +} + +func writeDeltaTestFile(t *testing.T, path string, data []byte) { + t.Helper() + require.NoError(t, os.WriteFile(path, data, 0600)) +} + +// deltaTestLayer builds a delta layer from a manifest and a bundle directory. +func deltaTestLayer(t *testing.T, delta MacOSDelta, derived, bundle string) gcr.Layer { + t.Helper() + var buf bytes.Buffer + require.NoError(t, WriteMacOSDeltaTar(&buf, delta, derived, bundle)) + layer, err := tarball.LayerFromOpener(func() (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(buf.Bytes())), nil }, tarball.WithMediaType(MacOSDeltaMediaType)) + require.NoError(t, err) + return layer +} + +func TestMacOSDeltaRoundTripAppliesOnlyChangedChunks(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + base := deltaTestDisk(8) + derived := append([]byte(nil), base...) + for j := range MacOSDeltaChunkSize { + derived[2*MacOSDeltaChunkSize+j] = 0xAA + derived[5*MacOSDeltaChunkSize+j] = 0 + } + basePath := filepath.Join(dir, "base-disk.img") + derivedPath := filepath.Join(dir, "derived-disk.img") + writeDeltaTestFile(t, basePath, base) + writeDeltaTestFile(t, derivedPath, derived) + bundle := filepath.Join(dir, "bundle") + require.NoError(t, os.Mkdir(bundle, 0700)) + writeDeltaTestFile(t, filepath.Join(bundle, "aux.img"), []byte("derived aux")) + writeDeltaTestFile(t, filepath.Join(bundle, "config.json"), []byte(`{"derived":true}`)) + + delta, err := ComputeMacOSDelta(basePath, derivedPath) + require.NoError(t, err) + require.Equal(t, []int64{2, 5}, delta.Chunks) + + baseLayer := machineFilesLayer(t, types.OCILayer, + machineTestFile{name: "disk.img", data: base}, + machineTestFile{name: "aux.img", data: []byte("base aux")}, + machineTestFile{name: "config.json", data: []byte(`{"base":true}`)}, + ) + deltaLayer := deltaTestLayer(t, delta, derivedPath, bundle) + target := t.TempDir() + require.NoError(t, unpackMachineLayers2(ctx, target, baseLayer, deltaLayer)) + + got, err := os.ReadFile(filepath.Join(target, "disk.img")) + require.NoError(t, err) + require.Equal(t, derived, got) + aux, err := os.ReadFile(filepath.Join(target, "aux.img")) + require.NoError(t, err) + require.Equal(t, "derived aux", string(aux)) + config, err := os.ReadFile(filepath.Join(target, "config.json")) + require.NoError(t, err) + require.Equal(t, `{"derived":true}`, string(config)) +} + +func TestMacOSDeltaRejectsBaseOfDifferentSize(t *testing.T) { + dir := t.TempDir() + bundle := writeDeltaBundle(t, dir) + derivedPath := filepath.Join(dir, "derived-disk.img") + writeDeltaTestFile(t, derivedPath, deltaTestDisk(8)) + delta := MacOSDelta{Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: 8 * MacOSDeltaChunkSize, Chunks: []int64{1}} + baseLayer := machineFilesLayer(t, types.OCILayer, machineTestFile{name: "disk.img", data: deltaTestDisk(4)}) + err := unpackMachineLayers2(context.Background(), t.TempDir(), baseLayer, deltaTestLayer(t, delta, derivedPath, bundle)) + require.ErrorContains(t, err, "base disk does not match") +} + +func TestMacOSDeltaRejectsLayerWithoutBase(t *testing.T) { + dir := t.TempDir() + bundle := writeDeltaBundle(t, dir) + derivedPath := filepath.Join(dir, "derived-disk.img") + writeDeltaTestFile(t, derivedPath, deltaTestDisk(8)) + delta := MacOSDelta{Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: 8 * MacOSDeltaChunkSize, Chunks: []int64{1}} + err := unpackMachineLayers2(context.Background(), t.TempDir(), deltaTestLayer(t, delta, derivedPath, bundle)) + require.ErrorContains(t, err, "delta has no base disk") +} + +func TestMacOSDeltaRejectsIncompleteAndMisorderedLayers(t *testing.T) { + ctx := context.Background() + manifest := []byte(`{"version":1,"chunk_size":65536,"size":524288,"chunks":[]}`) + cases := map[string][]machineTestFile{ + "missing entries": {{name: "delta.json", data: manifest}}, + "wrong first entry": { + {name: "aux.img", data: []byte("x")}, + }, + "patch skipped": { + {name: "delta.json", data: manifest}, + {name: "aux.img", data: []byte("x")}, + }, + "extra entry": { + {name: "delta.json", data: manifest}, + {name: "disk.img.patch"}, + {name: "aux.img", data: []byte("x")}, + {name: "config.json", data: []byte("{}")}, + {name: "extra", data: []byte("x")}, + }, + } + for name, files := range cases { + t.Run(name, func(t *testing.T) { + target := t.TempDir() + writeDeltaTestFile(t, filepath.Join(target, "disk.img"), deltaTestDisk(8)) + layer := machineFilesLayer(t, MacOSDeltaMediaType, files...) + diffID, err := layer.DiffID() + require.NoError(t, err) + require.Error(t, unpackMachineLayer(ctx, layer, diffID, target, unlimitedMachineBudget())) + }) + } +} + +func TestMacOSDeltaValidateRejectsBadManifests(t *testing.T) { + size := int64(8 * MacOSDeltaChunkSize) + cases := map[string]MacOSDelta{ + "version": {Version: 2, ChunkSize: MacOSDeltaChunkSize, Size: size}, + "chunk size": {Version: 1, ChunkSize: 4096, Size: size}, + "partial size": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size + 1}, + "unordered": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{3, 1}}, + "duplicate": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{3, 3}}, + "out of range": {Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{8}}, + } + for name, delta := range cases { + t.Run(name, func(t *testing.T) { + require.Error(t, delta.Validate()) + }) + } + valid := MacOSDelta{Version: 1, ChunkSize: MacOSDeltaChunkSize, Size: size, Chunks: []int64{0, 7}} + require.NoError(t, valid.Validate()) +} + +// writeDeltaBundle writes the aux and config files a delta layer copies in full. +func writeDeltaBundle(t *testing.T, dir string) string { + t.Helper() + bundle := filepath.Join(dir, "bundle") + require.NoError(t, os.Mkdir(bundle, 0700)) + writeDeltaTestFile(t, filepath.Join(bundle, "aux.img"), []byte("aux")) + writeDeltaTestFile(t, filepath.Join(bundle, "config.json"), []byte("{}")) + return bundle +} + +// unpackMachineLayers2 unpacks layers in order into one target directory with an +// unlimited budget, as the bundle unpack does. +func unpackMachineLayers2(ctx context.Context, target string, layers ...gcr.Layer) error { + budget := unlimitedMachineBudget() + for _, l := range layers { + diffID, err := l.DiffID() + if err != nil { + return err + } + if err := unpackMachineLayer(ctx, l, diffID, target, budget); err != nil { + return err + } + } + return nil +} diff --git a/lib/images/macos_import_darwin.go b/lib/images/macos_import_darwin.go index d0de8ee4d..0fa90c9e6 100644 --- a/lib/images/macos_import_darwin.go +++ b/lib/images/macos_import_darwin.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "os" - "os/exec" "path/filepath" "time" @@ -44,11 +43,8 @@ func ImportMacOSImage(ctx context.Context, p *paths.Paths, name, source string) if !info.Mode().IsRegular() { return nil, fmt.Errorf("bundle file must be regular: %s", file) } - if e = exec.CommandContext(ctx, "lsof", "-t", file).Run(); e == nil { - return nil, fmt.Errorf("bundle storage is open: stop its VM first") - } - if exit, ok := e.(*exec.ExitError); !ok || exit.ExitCode() != 1 { - return nil, fmt.Errorf("cannot verify storage is closed: %v", e) + if e = RequireClosedFile(ctx, file); e != nil { + return nil, e } } root := p.ImageRepositoryDir(ref.Repository()) diff --git a/lib/images/macos_unpack.go b/lib/images/macos_unpack.go index ad08493ec..7b71ec481 100644 --- a/lib/images/macos_unpack.go +++ b/lib/images/macos_unpack.go @@ -70,7 +70,8 @@ func (c *ociClient) unpackMachineLayers(ctx context.Context, layoutTag, targetDi } // unpackMachineLayer extracts one layer, spending its decompressed bytes from the -// bundle's remaining budget. +// bundle's remaining budget. A delta layer patches the boot disk unpacked beneath it +// instead of writing files. func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targetDir string, remaining *int64) error { rc, err := l.Uncompressed() if err != nil { @@ -90,6 +91,12 @@ func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targe return err } archive := tar.NewReader(stream) + mediaType, err := l.MediaType() + if err != nil { + return err + } + delta := mediaType == MacOSDeltaMediaType + applier := &macOSDeltaApplier{targetDir: targetDir} for { header, err := archive.Next() if errors.Is(err, io.EOF) { @@ -102,6 +109,15 @@ func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targe if !filepath.IsLocal(name) || strings.HasPrefix(filepath.Base(name), ".wh.") { return fmt.Errorf("machine layer entry %q is not a plain local path", header.Name) } + if delta { + if header.Typeflag != tar.TypeReg || isGNUSparse(header) { + return fmt.Errorf("delta layer entry %q must be a regular file", header.Name) + } + if err := applier.entry(name, header.Size, archive); err != nil { + return overLimit(err) + } + continue + } target := filepath.Join(targetDir, name) switch header.Typeflag { case tar.TypeDir: @@ -127,6 +143,11 @@ func unpackMachineLayer(ctx context.Context, l gcr.Layer, diffID gcr.Hash, targe return fmt.Errorf("machine layer entry %q has unsupported type %q", header.Name, header.Typeflag) } } + if delta { + if err := applier.complete(); err != nil { + return overLimit(err) + } + } // Hash the trailing tar padding too, so the digest covers the whole stream. if _, err := io.Copy(io.Discard, stream); err != nil { return overLimit(err) diff --git a/lib/instances/create.go b/lib/instances/create.go index 9477b2202..8af4a9147 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -99,6 +99,9 @@ func (m *manager) createInstance( if err := validateExpiresAt(req.ExpiresAt, m.nowUTC()); err != nil { return nil, err } + if m.macOSOnly && strings.TrimSpace(req.Platform) == "" { + req.Platform = "darwin/arm64" + } if req.GPU != nil && req.GPU.Profile != "" && !devices.Capabilities().SupportsVGPU { return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, devices.ErrVGPUNotSupportedOnMacOS) } @@ -382,6 +385,11 @@ func (m *manager) createInstance( HealthCheck: cloneHealthCheckPolicy(req.HealthCheck), RestartPolicy: cloneRestartPolicy(req.RestartPolicy), } + if req.FreshMacOSIdentity { + if err := m.freshMacOSIdentity(stored); err != nil { + return nil, err + } + } // 12. Ensure directories log.DebugContext(ctx, "creating instance directories", "instance_id", id) diff --git a/lib/instances/create_image.go b/lib/instances/create_image.go index cd605597f..825196d8b 100644 --- a/lib/instances/create_image.go +++ b/lib/instances/create_image.go @@ -20,8 +20,22 @@ type createImageResolver interface { func resolveImageForCreate(ctx context.Context, imageManager createImageResolver, imageName, platform string, log *slog.Logger) (*images.Image, error) { if p, err := images.ParsePlatform(platform); err == nil && p.OS == "darwin" { img, err := imageManager.GetImage(ctx, imageName) + if err != nil && !errors.Is(err, images.ErrNotFound) { + return nil, fmt.Errorf("get macOS image: %w", err) + } + if err != nil || img.MacOS == nil { + img, err = imageManager.CreateImage(ctx, images.CreateImageRequest{Name: imageName, Platform: platform}) + if err != nil { + return nil, fmt.Errorf("pull macOS image: %w", err) + } + } + pinned, err := pinnedImageName(imageName, img.Digest) if err != nil { - return nil, fmt.Errorf("get imported macOS image: %w", err) + return nil, err + } + img, err = awaitReady(ctx, imageManager, img, pinned, log) + if err != nil { + return nil, err } if img.MacOS == nil { return nil, fmt.Errorf("%w: image is not a macOS disk bundle", images.ErrInvalidPlatform) diff --git a/lib/instances/create_image_test.go b/lib/instances/create_image_test.go index d964bafe2..fea162437 100644 --- a/lib/instances/create_image_test.go +++ b/lib/instances/create_image_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "log/slog" + "strings" "testing" "github.com/kernel/hypeman/lib/images" @@ -30,6 +31,55 @@ func (f createImageResolverFake) WaitForReady(ctx context.Context, name string) return f.waitReady(ctx, name) } +func TestResolveImageForCreatePullsUncachedMacOSImage(t *testing.T) { + t.Parallel() + + const imageName = "registry.example/macos/app:latest" + digest := "sha256:" + strings.Repeat("a", 64) + pinnedName := "registry.example/macos/app@" + digest + createPlatform := "" + waitName := "" + resolver := createImageResolverFake{ + getImage: func(_ context.Context, name string) (*images.Image, error) { + if name == imageName { + return nil, images.ErrNotFound + } + if name != pinnedName { + t.Fatalf("unexpected image lookup %q", name) + } + return &images.Image{ + Name: imageName, Digest: digest, Platform: "darwin/arm64", + Status: images.StatusReady, MacOS: &images.MacOSImage{}, + }, nil + }, + createImage: func(_ context.Context, req images.CreateImageRequest) (*images.Image, error) { + createPlatform = req.Platform + return &images.Image{ + Name: req.Name, Digest: digest, Platform: "darwin/arm64", + Status: images.StatusPending, MacOS: &images.MacOSImage{}, + }, nil + }, + waitReady: func(_ context.Context, name string) error { + waitName = name + return nil + }, + } + + img, err := resolveImageForCreate(context.Background(), resolver, imageName, "darwin/arm64", slog.Default()) + if err != nil { + t.Fatalf("resolve image: %v", err) + } + if createPlatform != "darwin/arm64" { + t.Fatalf("expected Darwin pull, got %q", createPlatform) + } + if waitName != pinnedName { + t.Fatalf("expected readiness wait on %q, got %q", pinnedName, waitName) + } + if img.MacOS == nil || img.Platform != "darwin/arm64" { + t.Fatalf("expected ready macOS image, got %+v", img) + } +} + func TestResolveImageForCreateWithPlatformForcesPlatformResolve(t *testing.T) { t.Parallel() diff --git a/lib/instances/delete.go b/lib/instances/delete.go index d9ea69a40..5cc1ce936 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -114,7 +114,7 @@ func (m *manager) deleteInstanceWithOptions( attribute.String("operation", "graceful_guest_shutdown"), attribute.Int("stop_timeout_seconds", stopTimeout), ) - gracefulShutdown = m.tryGracefulGuestShutdown(gracefulCtx, &inst, stopTimeout) + gracefulShutdown = m.tryGuestShutdown(gracefulCtx, &inst, stopTimeout, false) if gracefulShutdown { gracefulSpanEnd(nil) } else { diff --git a/lib/instances/fork.go b/lib/instances/fork.go index b7a10a72e..6fec8d6b1 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -551,7 +551,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe inst, err := m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false) return returnWithReadiness(inst, err, false) case StateStopped: - inst, err := m.stopInstance(ctx, forkID) + inst, err := m.stopInstance(ctx, forkID, false) return returnWithReadiness(inst, err, false) } } diff --git a/lib/instances/macos.go b/lib/instances/macos.go index 9e8e4d901..909972aef 100644 --- a/lib/instances/macos.go +++ b/lib/instances/macos.go @@ -24,20 +24,17 @@ import ( // macOSAuxStorageFile is the instance's copy of the image's auxiliary storage. const macOSAuxStorageFile = "mac-aux.img" -// requireMacOSImageName refuses a create on a macOS-only server unless the image -// is an already imported macOS image, so a Linux image is never pulled there. +// requireMacOSImageName checks lookup errors before resolving the reference as +// darwin/arm64. The resolved image is validated by checkMacOSCreate. func (m *manager) requireMacOSImageName(ctx context.Context, name string) error { if !m.macOSOnly { return nil } - cached, err := m.imageManager.GetImage(ctx, name) - if err != nil && !errors.Is(err, images.ErrNotFound) { - return err - } - if err != nil || cached.MacOS == nil { - return fmt.Errorf("%w: macOS-only server requires an already imported macOS image", ErrInvalidRequest) + _, err := m.imageManager.GetImage(ctx, name) + if errors.Is(err, images.ErrNotFound) { + return nil } - return nil + return err } // checkMacOSCreate applies macOS machine defaults to a create from a macOS image. diff --git a/lib/instances/macos_agent_startup.go b/lib/instances/macos_agent_startup.go index 05451f57f..74025e0cb 100644 --- a/lib/instances/macos_agent_startup.go +++ b/lib/instances/macos_agent_startup.go @@ -83,7 +83,7 @@ func (m *manager) stopOverdueMacOSAgentStartup(ctx context.Context, id string, l reason += ": " + last.(string) } log.WarnContext(ctx, "stopping macOS guest whose agent did not start", "instance_id", id, "reason", reason) - if _, err := m.stopInstance(ctx, id); err != nil { + if _, err := m.stopInstance(ctx, id, false); err != nil { return err } meta, err = m.loadMetadata(id) diff --git a/lib/instances/macos_build_export.go b/lib/instances/macos_build_export.go new file mode 100644 index 000000000..13539cd15 --- /dev/null +++ b/lib/instances/macos_build_export.go @@ -0,0 +1,5 @@ +package instances + +// MacOSBuildExportReceipt describes one stop/export operation, not persisted state. +// A failed export may still confirm that instance cleanup is safe. +type MacOSBuildExportReceipt struct{ Graceful, VMMExited bool } diff --git a/lib/instances/macos_build_export_darwin.go b/lib/instances/macos_build_export_darwin.go new file mode 100644 index 000000000..c37a45424 --- /dev/null +++ b/lib/instances/macos_build_export_darwin.go @@ -0,0 +1,133 @@ +//go:build darwin + +package instances + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/kernel/hypeman/lib/forkvm" + "github.com/kernel/hypeman/lib/images" +) + +// StopMacOSBuildInstance performs only the strict graceful stop used by build +// export. It copies no storage and never falls back to a forced hypervisor stop. +// Success requires the shutdown acknowledgement and confirmed VMM exit. +func (m *manager) StopMacOSBuildInstance(ctx context.Context, id string) (receipt MacOSBuildExportReceipt, err error) { + if err = ctx.Err(); err != nil { + return + } + if _, bounded := ctx.Deadline(); !bounded { + return receipt, fmt.Errorf("machine stop requires a deadline") + } + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + current, err := m.currentInstanceWithoutHydration(ctx, id) + if err != nil { + return receipt, err + } + if current.MacOS == nil || !current.GuestAgentEnabled() || (current.State != StateRunning && current.State != StateInitializing) { + return receipt, fmt.Errorf("strict stop requires an active macOS guest with a system agent") + } + if err = m.markRestartManualStopLocked(ctx, id); err != nil { + return + } + stopped, err := m.stopInstance(ctx, id, true) + if err != nil { + return receipt, err + } + receipt = MacOSBuildExportReceipt{Graceful: true, VMMExited: true} + m.notifyLifecycleEvent(ctx, LifecycleEventStop, stopped) + return receipt, nil +} + +// ExportStoppedMacOSBuildInstance copies closed storage for an instance that is +// still Stopped when the copy starts. The instance lock excludes Start during the +// check and copy; a concurrently restarted instance fails export instead. +func (m *manager) ExportStoppedMacOSBuildInstance(ctx context.Context, id, destination string) error { + if err := ctx.Err(); err != nil { + return err + } + if _, bounded := ctx.Deadline(); !bounded { + return fmt.Errorf("machine export requires a deadline") + } + if !filepath.IsAbs(destination) { + return fmt.Errorf("export destination must be absolute") + } + lock := m.getInstanceLock(id) + lock.Lock() + defer lock.Unlock() + current, err := m.currentInstanceWithoutHydration(ctx, id) + if err != nil { + return err + } + if current.State != StateStopped || current.MacOS == nil { + return fmt.Errorf("export requires a stopped macOS instance") + } + if rel, e := filepath.Rel(current.DataDir, destination); e != nil || filepath.IsLocal(rel) { + return fmt.Errorf("export must be outside instance storage") + } + return writeStoppedMacOSBuildBundle(ctx, current, m.paths.InstanceMacOSBootDisk(id), destination) +} + +func writeStoppedMacOSBuildBundle(ctx context.Context, inst *Instance, disk, destination string) error { + if inst.State != StateStopped || inst.MacOS == nil { + return fmt.Errorf("machine must be stopped") + } + aux := filepath.Join(inst.DataDir, "mac-aux.img") + var diskInfo os.FileInfo + for _, source := range []string{disk, aux} { + info, err := os.Lstat(source) + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() == 0 { + return fmt.Errorf("machine storage must be nonempty regular files") + } + if diskInfo != nil && os.SameFile(diskInfo, info) { + return fmt.Errorf("machine storage files must be distinct") + } + diskInfo = info + if err = images.RequireClosedFile(ctx, source); err != nil { + return err + } + } + for _, file := range []struct{ source, name string }{{disk, "disk.img"}, {aux, "aux.img"}} { + if err := ctx.Err(); err != nil { + return err + } + path := filepath.Join(destination, file.name) + if err := forkvm.CopyRegularFile(file.source, path); err != nil { + return err + } + if err := os.Chmod(path, 0600); err != nil { + return err + } + } + data, err := json.Marshal(inst.MacOS) + if err != nil { + return err + } + path := filepath.Join(destination, "config.json") + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) + if err != nil { + return err + } + _, writeErr := file.Write(data) + closeErr := file.Close() + if writeErr != nil { + return writeErr + } + if closeErr != nil { + return closeErr + } + if err = ctx.Err(); err != nil { + return err + } + _, err = images.ValidateMacOSBundle(destination) + return err +} diff --git a/lib/instances/macos_build_export_darwin_test.go b/lib/instances/macos_build_export_darwin_test.go new file mode 100644 index 000000000..0d357bed1 --- /dev/null +++ b/lib/instances/macos_build_export_darwin_test.go @@ -0,0 +1,53 @@ +//go:build darwin + +package instances + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func TestMacOSBuildBundleRequiresClosedDistinctStorage(t *testing.T) { + root := t.TempDir() + disk := filepath.Join(root, paths.MacOSBootDiskFile) + aux := filepath.Join(root, "mac-aux.img") + require.NoError(t, os.WriteFile(disk, []byte("synthetic disk"), 0600)) + require.NoError(t, os.WriteFile(aux, []byte("synthetic aux"), 0600)) + inst := &Instance{State: StateStopped, StoredMetadata: StoredMetadata{DataDir: root, MacOS: &images.MacOSImage{HardwareModel: []byte{1}, MachineIdentifier: []byte{2}, MAC: "02:00:00:00:00:01", CPUs: 4, Memory: 8 << 30, GuestAgent: true}}} + file, e := os.Open(disk) + require.NoError(t, e) + require.Error(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, t.TempDir())) + require.NoError(t, file.Close()) + out := t.TempDir() + require.NoError(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, out)) + _, e = images.ValidateMacOSBundle(out) + require.NoError(t, e) + for _, name := range []string{"disk.img", "aux.img", "config.json"} { + info, e := os.Stat(filepath.Join(out, name)) + require.NoError(t, e) + require.Equal(t, os.FileMode(0600), info.Mode().Perm()) + } + require.NoError(t, os.Remove(aux)) + require.NoError(t, os.Link(disk, aux)) + require.Error(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, t.TempDir())) + inst.State = StateRunning + require.Error(t, writeStoppedMacOSBuildBundle(context.Background(), inst, disk, t.TempDir())) +} + +func TestMacOSBuildStopAndExportRequireBoundedInputs(t *testing.T) { + m := &manager{} + _, e := m.StopMacOSBuildInstance(context.Background(), "unused") + require.Error(t, e) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, e = m.StopMacOSBuildInstance(ctx, "unused") + require.ErrorIs(t, e, context.Canceled) + require.Error(t, m.ExportStoppedMacOSBuildInstance(context.Background(), "unused", t.TempDir())) + require.Error(t, m.ExportStoppedMacOSBuildInstance(ctx, "unused", t.TempDir())) +} diff --git a/lib/instances/macos_identity.go b/lib/instances/macos_identity.go index 7e97f93cd..6c9ef6a7a 100644 --- a/lib/instances/macos_identity.go +++ b/lib/instances/macos_identity.go @@ -186,7 +186,7 @@ func (m *manager) rekeyMacOSCloneOnStart(ctx context.Context, stored *StoredMeta // Teardown runs even if saving the pending flag fails. Otherwise the clone would keep // running, networked, with its source's SSH host keys. saveErr := m.saveMetadata(&metadata{StoredMetadata: *stored}) - _, stopErr := m.stopInstance(context.WithoutCancel(ctx), stored.Id) + _, stopErr := m.stopInstance(context.WithoutCancel(ctx), stored.Id, false) if saveErr != nil || stopErr != nil { return fmt.Errorf("macOS clone rekey failed and teardown needs attention: %w", errors.Join(err, saveErr, stopErr)) } @@ -224,6 +224,29 @@ func (m *manager) clearMacOSStopReceipt(stored *StoredMetadata) error { return nil } +// freshMacOSIdentity gives a new macOS instance its own machine identifier and MAC +// instead of its image's, so it can run beside the image's other instances. The +// guest's SSH host keys are rekeyed on first start. +func (m *manager) freshMacOSIdentity(stored *StoredMetadata) error { + if stored.MacOS == nil { + return nil + } + if err := prepareMacOSClone(stored); err != nil { + return err + } + used, err := m.storedMACs(stored.Id) + if err != nil { + return fmt.Errorf("list MACs for macOS identity: %w", err) + } + mac, err := network.NewUniqueMAC(used) + if err != nil { + return fmt.Errorf("allocate macOS MAC: %w", err) + } + stored.MacOS.MAC = mac + stored.MAC = mac + return nil +} + // assignMacOSCloneMAC gives a networked clone a MAC no stored instance uses. It runs // under the fork admission lock, so concurrent forks cannot pick the same MAC. func (m *manager) assignMacOSCloneMAC(clone *StoredMetadata) error { diff --git a/lib/instances/macos_identity_test.go b/lib/instances/macos_identity_test.go index 2978e7da1..0d3f98506 100644 --- a/lib/instances/macos_identity_test.go +++ b/lib/instances/macos_identity_test.go @@ -4,10 +4,12 @@ package instances import ( "net" + "os" "testing" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/require" ) @@ -42,6 +44,21 @@ func TestPrepareMacOSCloneRebindsIdentityWithoutAliasingSource(t *testing.T) { require.False(t, source.IdentityRekeyPending) } +func TestFreshMacOSIdentityRebindsIdentifierAndMAC(t *testing.T) { + p := paths.New(t.TempDir()) + require.NoError(t, os.MkdirAll(p.GuestsDir(), 0700)) + m := &manager{paths: p} + stored := testMacOSSource() + stored.Id = "builder" + require.NoError(t, m.freshMacOSIdentity(stored)) + require.True(t, stored.IdentityRekeyPending) + require.NotEqual(t, []byte("source-identifier"), stored.MacOS.MachineIdentifier) + require.NotEqual(t, "a6:fd:95:db:a2:62", stored.MacOS.MAC) + require.Equal(t, stored.MacOS.MAC, stored.MAC) + _, err := net.ParseMAC(stored.MAC) + require.NoError(t, err) +} + func TestPrepareMacOSCloneRefusesWithoutGuestAgent(t *testing.T) { for _, disable := range []func(*StoredMetadata){ func(s *StoredMetadata) { s.MacOS.GuestAgent = false }, diff --git a/lib/instances/manager.go b/lib/instances/manager.go index b64d1c1c9..aa234f69a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -736,7 +736,7 @@ func (m *manager) StopInstance(ctx context.Context, id string) (*Instance, error if err := m.markRestartManualStopLocked(ctx, id); err != nil { return nil, err } - inst, err := m.stopInstance(ctx, id) + inst, err := m.stopInstance(ctx, id, false) if err == nil { m.notifyLifecycleEvent(ctx, LifecycleEventStop, inst) } diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 0bf65d7af..88775bf14 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -255,7 +255,7 @@ func TestGracefulShutdownWaitsForSocketOwnerInsteadOfExitedStoredPID(t *testing. }} m := &manager{} - assert.False(t, m.tryGracefulGuestShutdown(context.Background(), inst, 1), + assert.False(t, m.tryGuestShutdown(context.Background(), inst, 1, false), "stop and delete must fall back to the hardened kill path while the socket owner is alive") assert.True(t, ProcessExists(owner.Process.Pid)) } @@ -297,7 +297,7 @@ func TestGracefulShutdownFallbackKillsCapturedPIDAfterListenerCloses(t *testing. VsockSocket: filepath.Join(t.TempDir(), "missing-vsock.sock"), }} - assert.False(t, m.tryGracefulGuestShutdown(context.Background(), inst, 0)) + assert.False(t, m.tryGuestShutdown(context.Background(), inst, 0, false)) require.NoError(t, m.killHypervisor(context.Background(), inst)) assert.False(t, ProcessExists(owner.Process.Pid)) } @@ -752,7 +752,7 @@ func TestMacOSGracefulShutdownWithoutLiveVMMIsNotGraceful(t *testing.T) { sent = true return nil }} - assert.False(t, m.tryGracefulGuestShutdown(context.Background(), inst, 1), + assert.False(t, m.tryGuestShutdown(context.Background(), inst, 1, false), "a macOS stop with no VMM to signal must not produce a graceful receipt") assert.False(t, sent) } @@ -800,7 +800,7 @@ func TestMacOSGracefulShutdownRequiresGuestStopReceipt(t *testing.T) { return err }} - assert.Equal(t, guestStopped, m.tryGracefulGuestShutdown(context.Background(), inst, 5)) + assert.Equal(t, guestStopped, m.tryGuestShutdown(context.Background(), inst, 5, false)) assert.NoFileExists(t, receipt, "the receipt is consumed") }) } diff --git a/lib/instances/restart_policy.go b/lib/instances/restart_policy.go index 227b6c76c..056bb076b 100644 --- a/lib/instances/restart_policy.go +++ b/lib/instances/restart_policy.go @@ -173,7 +173,7 @@ func (m *manager) HandleHealthCheckUnhealthy(ctx context.Context, id string) err return err } - stopped, err := m.stopInstance(ctx, id) + stopped, err := m.stopInstance(ctx, id, false) if err != nil { _ = m.updateRestartStatusLocked(id, restartStatusAfterFailedAttempt(policy, nextStatus, reason, now)) return err diff --git a/lib/instances/stop.go b/lib/instances/stop.go index 2d8ee7a39..4e4f23340 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -39,9 +39,10 @@ func hasVFIODevices(stored *StoredMetadata) bool { return storedVGPUDevicePath(stored) != "" || len(stored.Devices) > 0 } -// tryGracefulGuestShutdown asks guest init to shut down and waits for the -// hypervisor process to exit. Returns true if the process exited in time. -func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, stopTimeout int) bool { +// tryGuestShutdown asks guest init to shut down and waits for the hypervisor +// process to exit. Returns true if the process exited in time. With strict, only +// an acknowledged shutdown within the context deadline counts. +func (m *manager) tryGuestShutdown(ctx context.Context, inst *Instance, stopTimeout int, strict bool) bool { log := logger.FromContext(ctx) if !inst.GuestAgentEnabled() { @@ -67,7 +68,7 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, if pid == 0 { // Nothing was signalled. Linux treats an already-gone VMM as stopped, but // macOS only counts a stop as graceful after a shutdown attempt. - return inst.MacOS == nil + return !strict && inst.MacOS == nil } inst.HypervisorProcessIdentity.Set(pid) if inst.MacOS != nil { @@ -104,6 +105,14 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, // If we couldn't signal the guest, don't burn the full graceful timeout. waitTimeout = shutdownFailureFallbackWait } + if strict { + if ctx.Err() != nil { + return false + } + if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < waitTimeout { + waitTimeout = max(0, time.Until(deadline)) + } + } if WaitForProcessExit(pid, waitTimeout) { // The shim exits after a forced stop too, so macOS also needs its receipt that @@ -113,7 +122,7 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, return false } log.DebugContext(ctx, "VM shut down gracefully", "instance_id", inst.Id) - return true + return shutdownSent || !strict } log.WarnContext(ctx, "graceful shutdown timed out, falling back to hypervisor shutdown", "instance_id", inst.Id) @@ -124,10 +133,9 @@ func (m *manager) tryGracefulGuestShutdown(ctx context.Context, inst *Instance, // Flow: send Shutdown RPC -> wait for VM to power off -> // fall back to hypervisor shutdown -> final SIGKILL if still alive. // Multi-hop orchestration: Running/Initializing → Shutdown → Stopped -func (m *manager) stopInstance( - ctx context.Context, - id string, -) (_ *Instance, retErr error) { +// stopInstance stops an instance. With strict, a failed graceful shutdown is an +// error instead of falling back to a forced hypervisor stop. +func (m *manager) stopInstance(ctx context.Context, id string, strict bool) (_ *Instance, retErr error) { start := time.Now() log := logger.FromContext(ctx) log.InfoContext(ctx, "stopping instance", "instance_id", id) @@ -180,7 +188,7 @@ func (m *manager) stopInstance( attribute.String("operation", "graceful_guest_shutdown"), attribute.Int("stop_timeout_seconds", stopTimeout), ) - gracefulShutdown := m.tryGracefulGuestShutdown(gracefulCtx, &inst, stopTimeout) + gracefulShutdown := m.tryGuestShutdown(gracefulCtx, &inst, stopTimeout, strict) if gracefulShutdown { gracefulSpanEnd(nil) } else { @@ -189,6 +197,10 @@ func (m *manager) stopInstance( // 5. Fallback hypervisor shutdown if guest graceful shutdown didn't work if !gracefulShutdown { + // Export callers must never turn a forced stop into a graceful receipt. + if strict { + return nil, errGracefulShutdownFailed + } log.DebugContext(ctx, "shutting down hypervisor (fallback)", "instance_id", id) shutdownCtx, shutdownSpanEnd := m.startLifecycleStep(ctx, "shutdown_hypervisor", attribute.String("instance_id", id), diff --git a/lib/instances/types.go b/lib/instances/types.go index bd8989885..7c50cb359 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -267,6 +267,7 @@ type CreateInstanceRequest struct { Name string // Required Image string // Required: OCI reference Platform string // Optional: target platform as os/arch[/variant]; drives image resolution and auto-pull. Empty means host platform. + FreshMacOSIdentity bool // Internal: give a macOS instance its own machine identifier and MAC instead of the image's. Size int64 // Base memory in bytes (default: 1GB) HotplugSize int64 // Hotplug memory in bytes (default: 0, set explicitly to enable) OverlaySize int64 // Overlay disk size in bytes (default: 10GB) diff --git a/lib/middleware/oapi_auth.go b/lib/middleware/oapi_auth.go index 148967567..9e8a64d25 100644 --- a/lib/middleware/oapi_auth.go +++ b/lib/middleware/oapi_auth.go @@ -203,6 +203,7 @@ func extractRepoFromPath(path string) string { } // isWriteOperation returns true if the HTTP method implies a write operation + func isWriteOperation(method string) bool { return method == http.MethodPut || method == http.MethodPost || method == http.MethodPatch || method == http.MethodDelete } diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 4c604c755..b6b825e65 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -274,6 +274,12 @@ const ( SnapshotTargetStateStopped SnapshotTargetState = "Stopped" ) +// Defines values for CreateBuildMultipartBodyMachineNetworkMode. +const ( + Egress CreateBuildMultipartBodyMachineNetworkMode = "egress" + Isolated CreateBuildMultipartBodyMachineNetworkMode = "isolated" +) + // Defines values for GetInstanceLogsParamsSource. const ( App GetInstanceLogsParamsSource = "app" @@ -2002,6 +2008,22 @@ type CreateBuildMultipartBody struct { // Admin builds can populate the shared global cache that all tenant builds read from. IsAdminBuild *string `json:"is_admin_build,omitempty"` + // MachineBaseImage Experimental installed darwin/arm64 base pinned by sha256 digest. + // Requires server macos_only and macos_builds. The source must contain + // build.sh at its root, which runs as root in the guest. Machine source + // is limited to 64 MiB compressed; only machine_network_mode, + // timeout_seconds, memory_mb, cpus and tags are accepted alongside + // source and this selector. CPU/memory zero inherit the base. + // Linux/cache/secrets/admin options are rejected. The builder and its + // output get their own machine identifier and MAC, so the output can + // run beside the base. + MachineBaseImage *string `json:"machine_base_image,omitempty"` + + // MachineNetworkMode Native machine builds only. Defaults to isolated. Explicit egress enables + // unrestricted VZ NAT for checksum-verified artifact downloads; it does not + // enforce domain allowlists. Linux builds do not use this selector. + MachineNetworkMode *CreateBuildMultipartBodyMachineNetworkMode `json:"machine_network_mode,omitempty"` + // MemoryMb Memory limit for builder VM in MB (default 2048) MemoryMb *int `json:"memory_mb,omitempty"` @@ -2021,6 +2043,9 @@ type CreateBuildMultipartBody struct { TimeoutSeconds *int `json:"timeout_seconds,omitempty"` } +// CreateBuildMultipartBodyMachineNetworkMode defines parameters for CreateBuild. +type CreateBuildMultipartBodyMachineNetworkMode string + // GetBuildEventsParams defines parameters for GetBuildEvents. type GetBuildEventsParams struct { // Follow Continue streaming new events after initial output @@ -19326,397 +19351,403 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9+3IbN7Y3+ioonv1VpD0kRV18Uyr1Hcd2HO1EsbZle749oQ8FdoMkRt1AB0DTZlKu", - "On/NA0ydJ5wnObUWgL4RTbZkS7InrqmayGw0GpeFhXX9rT96kUwzKZgwunf8R09HC5ZS/POxMTRavJFJ", - "nrKX7LecaQM/Z0pmTBnOsFEqc2EmGTUL+FfMdKR4ZrgUvePeGTUL8m7BFCNL7IXohcyTmEwZwfdY3Ov3", - "2HuaZgnrHff2UmH2Ympor98zqwx+0kZxMe996PcUo7EUycp+ZkbzxPSOZzTRrN/47Cl0Tagm8MoA3yn6", - "m0qZMCp6H7DH33KuWNw7/rU6jbdFYzn9O4sMfPxxbuS5oSKers5kwqPV+mR/5iJ/j18jNDcypYZHRNt3", - "SIYvkSnVLCZSEBoZvmSEi6nMRUxePTkjkRSCRdCZHgs51UwtWUxmSqbELBhZSG2wjVE0uiSGThM2HIte", - "v7EfTMCTePsq/XXBzIKpwGC5Jq4XMpOKmAXXhAt4GrFhdcOMytn6yvZ7PE7YxPCUydysL9SP8h1JpJjj", - "tHy/JM21IQu6ZOR3piT5LacJn624mLcv0pTNpGLkx1XGUipIltCIacIN4cJIPxu7RiWN3UtDxMXnQio2", - "iZk2XFDof5JJZfT66F/gHzQhlbY4NGxPzIIaT+VCGnLJWFafKH1HL+vL+OvBQf/RaDR62+9xw1J7rOh7", - "nuZp7/j+vXuH9/q9lAv77/1i9FwYNmeq96H4hSpFV5XpaJmriE0iHqtNM4kSzoQhT06evrzmBHr7oyH+", - "b+9hr9/bf3Qw3L//EP+9f79XndbawtdH/mHz0Ts31OR6nQfZ0zRxhDKpEMn6rH/J0ylTRM5IlCvFhElW", - "BI8UizsQXW3ao9BWRFLM+DxXLF7/tj9yteVcUE2osExj0OAXZWedzl0ETCyW78REsZRyAWu8NoiX/hGB", - "E0rcIYIhRVIYJZMEmIIxLM2M9qeoD2xcEJplCY+Q9dQO1VE60r1+T+RJQqdrIyx3myV8zrFBp6XhurJJ", - "/l1iJGHCMFWc8C5LU2OLbR8ulzu4GyVf7PTJhUziSS4MT9a/+hp+dmta/RLXZMGSmMjZrE/4jFACvcDP", - "lsbry34wOrg/GB0NRvdf7T84Hj06Ht37W6/fm0mVUtM77sXUsAHscpe9QaatuYjCuyOaJJLChaRYZHen", - "uLBqGzhlkUwZga43jvzo40feet08KTcVGhLXsLxZA1s9bFwZ3ag7odoUTAj3i5vVhAbG9IqnTBuaZsCH", - "YAyVxWzjQr7D5j74ld+4wPsftcCCvTcTt0LB+YTog73PWAQ3ovTcpBAwoD9H3gXLuhUaV4xqKayYBJfp", - "r71c6DyDq5vFkyyhBvrt9XuWDCYp1xpeLX6IubZ8pN/zRD4R0kxULoRtKJh5J9VltaXrZcKzXr+3oHqy", - "nGd5r7/p2qoTNX6CJTTT2J/bcTVhSknVs6LxajKTym9S7211CTd0tbZCurhiAyvU6/dqC1Cwcz8XP+5i", - "V4OD6/fcXxNltQrszk5mfeDVvtaHWwxtM2O3t4gVov02E/eyrnOAmNO5kNrwSHdi8yg8wPamMg6wzqdF", - "d4THTBg+40w5uZoRBfdDyojvhEAnhAuS68Y5KET/CVsyYfRkeTQxUba+KA3Fprp5FdmkvBErt3Kx/cVJ", - "2UKk9bkHFacl5Xgmn7Ilj9i67Oa2ZhIrvmQqwL4LAcCyQtuO7MBZBxYipGC7tZUSSx5z2oUdxDimCQ9Q", - "z9mTE2Ifk5OnZGfB3tc/cvBg+rDX3qWgaYAWfsxTKgZwIGBYvn9sW+3756NQz1ymaT6ZK5ln6z2fvDg9", - "fU3wIREo4VZ7fHgQklSziE9oHCumdXj+/mF1bKPRaHRMD45Ho+EoNMolE7FUrUtqH4eXdH8Usw1ddlpS", - "1//akv7y5uTpyWPyRKpMKtTZth6c6vJU51Ulm/quhOj/+5wn8TrVT+Fn1rJOTGmuDROGfG+bEcWsKkfe", - "LaRmJKLRgpGpVVpQPcf+upB88WF/e4VG8JTrTCKfJ29OS2kOVUP2nkW5qX33WxJzbbiInFBVmVyHEYH5", - "KWFw/4YkC1wB4tpwKYjxotO1hYBIMbrlc9Ci08fWz39uyWuS6rbefRPCBUl5knDNIiliXf0GF+b+Uftk", - "KqfYXptrn3oGP5OUaU3njOygWQrVDcvhCddkRnnC4t1uEnbbZP4up5V7rXbmkAwGdBrtHxwGGVpK52wS", - "87mzKzZpEH4H+Rj6MYSnrRNBCaPbPPCTis3Wv/cD3if4EcVmTDERffTnMiWXTFCnUv0Hfrf3f+2VBtc9", - "Z23dw8U8K5t/6Pd+y1nOJpnU3I5wjU24J0BGuNQE3wiPGR/Fu50oShuqNp8PbPEJTmIpbG5dG2f6AXmL", - "zre+8graNBk6sqNCwKlwgVa+/QwkrYDIIoVhIrA6P8s5SbiwahATxu0FCnurjH2XyPlu75OtQ7H864cf", - "xn0N5mV/aOkNnvULrSCR8+pqLhhVZspqi9lyr7qOytG1Lv9Z7fg0LlCq2WQzBznjQrAYbe7uYNuWIFsH", - "dR88RZfcTJZwA4fOHA7rJ26Ia9HaVSKjyxlP2GRB9cKOmMYxtwbXs9pMAiJkzZlB0UjgO0TRBpXq8x8f", - "H9y7T9wHAmvorL/QYH0mlbehe9uWGKqmNEmCtNFOble/o9cpJEwBpcG37e4pKNATpuV0PbebTnnP9cL+", - "hbwbRoV3X6/fi4C8Evh7jXD7vfcD6HWwpApXHbqvDOu//Zcqv31ffrTy61nx/cqPL91QKj/94EdV+e1J", - "OUC/KkytnwenpuGkw1Kl5MIMuMAtAAHcWZ6mdvMKLctZMogUFQmPqS7sfJNg9WRNpCI7L394cnh4+Gi3", - "u3DF9eVE89/ZZD7dKDe7QTtBGV4j8Brhgsz5nE5XhukheUKFkAackdGCijmLCZ0ZeMuNtaaA3xttc8K0", - "CklMtYlI5vcj+ujh+/fUPLrP3+lHv6dTNf/7Ie21WRdz3bK+aIGjbjehKVFUBDbxehd1St+D5m+pJOT8", - "PLXuKlI0cnugm0MYklKhn/H3sOaG7NdWOri4Ye2vcGEJKQa54L/luNtZQlfrOqBhNB0gRYSW1/INe4D0", - "9uOj4fzYd/xM31Fu4OQUPlM35T6RSQz3zowrbXrdvWFXEgyY+lQiUu2UhSWmdYpoLmDrpV6Ms/WsJHzG", - "olWUsHXm7tl3pnJnbo0Z3Dn4Z2FA7MrGmQpwYv/jWfGJ2s9Py+/Vfn9mP/6h33tCMzrlCfecuc6nnU9+", - "4gx/23ap2tlT++pL9+aHfm/GqMkVC6zlOcYHEGsnHiRsyRLiWgPxHpPHZydE52qGLnOkVtsWzPZSMz0W", - "is2pihOmkdTfLXi0KMyVYHrQhIoyLIA8Tt7RlSaZYpoJczwW48JCrse9Phlb5cv9banE/m1DQ3wjMVdM", - "u3+ArWHc60NfiZzrcW9IfpTaDEBddrLUMRk7e4we98iO/ZNkVGuzUDKfL8YipYLOWcqE6RMM0kAzMHCm", - "ZLVLqIjJuKekZsbQAUvzBHk/dPY4yxI2Fuc84ZEUJKXRi3P37jtuFuSlfalyeeKM4bLuk8yZnjVN2VhQ", - "awaFnVyRTMkpIwnNRbRgmjAxkypiu0NyxtTAL7HfW0IVg82wRlySCzgijJaboX99S5gwykU6dGcuMJOr", - "0B8sfaFE66u8eWLf+FB4R67y8i/uFWBVbsZBHUWtSmt6hZ7tjViY+v11VPME9McCVxR3lUOrd4LUtmyW", - "0DmSSuUQWWLUYxHJNEVFz32FaLZkiibFDsEdJ9GeAHeDu5HGIkpkHg8Wq4ypJddS9eGGYBEGI6g++Y2l", - "eR8/Cn8NUh4puUyJFGORABnv0TS+f7Q7JI9JwjUQh58/t8TtSAw5AdoDOIx23KPeJj/u2YlxTeDiH5Jn", - "wihuaY5oS3BTe5M2iKvr5lWY1drdhvtzlc7O7RtrHg77syPo/hqLrdBNSYAFGVe4aOjO2sCA12Vwv66b", - "nVFufMVuRVS4napR5zHhhtAsY1RpwsVYlNQkYqRS91KmGC6H5sZtXcoM2UmpQfYCVtuxQA7xjSa13d8d", - "EpQbbXxYyZHgFaJYsrIDYmPhx/yOJwkaDkmOEQzI4exBQ8tYJYICDxMVBe9LSm+Xc345v2htArUjYr/9", - "389OXxO90oalZMoFVatdwnXJay1prjvmwiKj2z3U3FFUq886lhj1BO8SKeoCZPO8bnUhOBG0pIttFPaj", - "DIVYUhUtQmF02pAnZ68JPOaGRUDEtfFSld4PupKkbukOvkpRiLXrXesOmc7WOUs4UjjibZM9Ke6RsIhU", - "uOPXvV3wZsWPyxIWAa+y8TRWf2PEOZeJTJGlC7Y+m73WJfKd621f12RH6j2Y8G7lZoEzrXIxJC8EQTGC", - "1KSIscCmXERJHjNNKvyclOZbL1/UwqAKsh+Lf/2//5+VJWKy5LSQNsgbrgzEkPxutdiZoikDple/0qIF", - "iy7HoimGEOhVS0L9pVIsM9fVC0Xx+cIQId8Nx+LEjqkY8I6WM/OOKpZnoGaSwcCNeuDkrF0bxIUdcdMI", - "0fy1sTeVxbliLGHNnVdsaH+dwraR6i+l2FKn1Tk17B0NKMTPgfQGS64xZA1pwrUlJ2dDgo/RlRAt7FPg", - "oiDFkh1LFVzMxwKfOJF4lziJ1pKZc00OyYuUm4L4hSyuF3fTjQUEGE4ZE+hFTJY2ALkk1RVrbgBEc+6D", - "g3e4Hzoacxj7xMjJPA/aXf1dF3PFIkPenA6MHLw5JUbR2YxHQEcZU27U9oJpDjrIzyHGIGlZaf+iD6FI", - "hqBkKB7PQciB5XKiv/3NXk4ZUzCuV4/P4ILD1f+WjHuCGvtOyekH4MnhMYvJL49fwdUavTjfdavmVFPb", - "cQ+unZARvN/T+VQw0zZ++xRjcMNbgaG025ivXaK1LdpG3h8n0cDFbzv4ZoNMUnIwkE5AiC1IcEh+rsmv", - "9oUicAVE0HWJvujBH+JvSTHa71CgISmjQhPqZP3KiAjXY+Glj5rAAZI2cSsKr6pcWC5l78NBhfuiWGLl", - "EZTIiiD9Jfz/YKlldOlCPKyWWXBaQVM83dWlW5OsuOkk5Gw1AVQUlsCS+q/XBE/QEbSgmV5Ig1q1/8ee", - "YtpIxXZBOXfxUvA8o7lm5C8kZalUq7Hw7fv2nCXU2BiG4t0ZMKYe2YkSKVjVjmADCOhYaCOzjMXOMfEt", - "mbnDTYsgQqkKW7VtNBbuE3rPSgCaFLPADSg9MMmq3OTKTHbR1ICzKaYFc85TO+yFNFmSzwdunj2yk/Al", - "c9OGCfLfme0DvChSioGLh4Smpe0EHxFD1ZwZZ3rW2D/SzLg3Fjt4bgn+GxwraS54hLc5tptn+aBq4uiR", - "nednr/fKmKWxqDzGV8CoN+BykPCUA3vbgR/Iyd4LouCGxp+5mNvRY2M7nWKS2N5NcUje0CR35xSpHdZ3", - "LLz0jyMdGJWbxSxPjsnydyeE1fej2MlCKto/7DtLU4IWplZJ5s0p0XTJPEHiW6BSawPHiMZLpgzXzhp7", - "SXYcOQ2cnwvpThPBWEyEHItqX3CQkUxhiE2Rn1hDjCaNRRqLUmxrRJL7q827DAtThBd8ultsNms0LT6G", - "j9darqAinxdKfeMiyXi7VxVskXhQaFQuU0V/ZeRFxgS0imWUp0wYwsVM+qb1KLLR8KAtMq3l8+fVa6X4", - "vF5IZcicG6IYiHFSOG427g1irgwcf53PZvw9apE5WoGsTIM96T5wqHEvF5dCvhPjXn2YD+7Tg1k8Pdy6", - "G37Y/doaBvcBmZ6zSrfm6232oT3Z6DADUZMYCbpzVfN3FP4J3GVlShJcemoAZyXhLK7Qdt9G686ZYMC5", - "YhsDaoiX0FpcPoOgKHvjXqUreWJadtRGzrZu6JY5zBM5xQvPT6QMOh2Sk/ra9QnFBygw11e5yJaoBIaS", - "HTacD+F2ySI+gMjQAT0YjEaDUZPak6OBDXnPqDFMwQD/n1/p4PfHg7+NBo/eln9OhoO3f/mPoELeMVrV", - "+7XdPHf8WeoTP9hqCGtzoNvCW6/tWasOv/3wommhdacjxfAY0GTrGCDi4EmleTutPzlZj3qzKxVLsEMP", - "udxL+FRRtdoTcy7eH4M4p0193Ta33WRgCSTHWMHINyBUE2dh+XVvSRWnwry1YyTjqnEA5DeCtwbc+09x", - "RGQw8P049uUj72sOgCH5RRpCnUcaVLpjbLT+3bmiaUrR2rmkCY/xcDg3m+Yxs8ZZgc/I0kpJiplcCXI0", - "Gvknk8qgQG6Q2k9l3HOGUxUtiDMIKYKmEJshE7EMPmlkTBtpSpWlWD9pcMz+slf8teP/HL79y+7//o9P", - "SupIaBto3JozrsjPGnHe0IjsJPIdUxHVjCQMZqv7JOZzbrQV3mIMjUKD2rckssEeNnRRKsKEN1Fjuzo5", - "p6sBzfjAWV6su/tnJuZm0Tu+fxhc3B33x+Dtf/qfWhZW5UlIX3spc7S84uNq4rUfQyfni1/dPEGnS8rF", - "iX1tf12c/Lgd9hPZtNNWqWvdarhjiqygLQNZz8CHiKM0JEQsmVJwFC1fe3L6FFSYS+YYGmiNZJyPRocR", - "NsA/mfsFhDhQZfC33brEY7sLCTu/9li0kBgGmSTyKkbKfpOpt0UJbvSRhVa75P7rMYXoRLducQx2dw3J", - "VMlLptAlA0vMmSaXbGXNBfOaQZOJJVlS5V2DrzAtAZv4R+gNYqgegxKLyfmQy8qUY4tOZ7NpUIwmJbdM", - "Kbi7UK/UkcwYMFTXDKdGLphYXpCUZlatV8wazVNqmOKgLpaGxZTFHHj0WDA8GCSjcOajSCowscLeog22", - "XIVvNLmwauIFdn/BBZD1hT2YDVPpH70Xr199/+L1L08nL86e/fL4ZPLTs/+Bn+1LveNf/+hZMI0i8Op7", - "RhVT5D/+wPl+sMGzGNMHqe4LqZy26130GK9CMz6UGROUDyOZ9vq9/6z+8+2Htz7c02buLXvHwYF9CIqZ", - "zvYZiMH3CVCaSFVGnlLEBUEW5c0OFXNDw5RvJb8rHQlUUbicTDMdzE1p2i1KOXR/NDr9fs9Fj9zz/9gd", - "Eue6xeEDD5LKicd6AeRT4HKgPy1JZOQyNGZtKfj+UyEGj77WTHJhtjKnsuk6jxoMyqdXYEV7Uy72NGzD", - "ILrauiPdXDtQ+ZlYciUFqucoLk0Tputn5ZcXT59Nnv3ypncMF0GcYx5hr987e/HyVe+4dzgajXohAmXv", - "M66YDgY+Pp5qmeSGEWxUxpgOyWmuMbjTMY9ZjnZQlM80SXMwLCUrwt5HSa6BT6E0YEwSyj9+ODjYf7X/", - "8Hg0Oh6N/tY5YBVIfwvzfn722mao43lnNDGLCbrmtr34I7Z9gk0/9L1tElX7wCoVe+qtlXB63TtkZ1GX", - "ruzkCBLkuHf4/Ht7oPafj3sVQnSpwEUvtuPG6j3/PrQuFWvUuqRXPLMZbiL2VL8ORmPt9bl2ZqKqJ3wg", - "KELsQMuYqphMJVXxt/VYGnwTDPnfaILWCpo4P5EVvfEdK0tKplFX9jE+pbFV9xtLgJaf1JoSqSAMPaVL", - "DixnkKZc+vfqbMkZU2o21GHFwRSw41XChXr9HszM/cdPsNfvLX+v51RvNwe60JhOuuIWJZAmGRdsgxb4", - "mQj7YFNOJI0H+59Y1q/EuzVgaOyD+nYXyqkn8DV0pykV8Tsem8UEMuJhyIEb0j0hRePimnwPM6HJv/7x", - "zzenpUFk//k0c3fm/sG9j7wzG7ckdB3MtSkmkmfhabzOwpN4c/qvf/zTz+RuJ2FFymuJ527/n9kemgA1", - "DrfLRuO3+F0LOazMz5UtvvQtGAahK1cumUroqnKduDH19kfI0xujUty6G917FRPy5ssFevPiWvV6ATY/", - "4yyxMVo29Bb1C+09RNZ3iKcFeNI32mcgMTKV0tgRwNowuoaNEr6VPs4kNXQ3RySFoVww5cZb3Bj2SrEa", - "S3r/aEievc+Y4iAw0YTEVL3jwga9+Dedzk2oAECghAs24Knz4NolsZNEngwdg6PNBxxpG3C0brVHY3VM", - "3P1oX5aqYRC7ScuSYsiYJ1mBpLfp+Ly0rUulP0CSAYr8nurCM9uFDgsy3D84dX8eWIKsEl9xuOxZ46og", - "Pvupb0EMBdXZ4Ee1hbWbMgJfYlQbckSe8+8JF5DKnzByyr8n1N1OXJA3f7P7gLxOd5Kl9CXPbJzHhM6L", - "tNRNyH/nlzxz+jy+4aMM7B0U59AznqHhWFhkH2AbSDsQke99twaSCLQNXIDbFu+zdRkNrAPljWabWzet", - "ykWfTHMDKr80jDjji4MYgLFg4ykjeRlE2lDA3QTXAyNwWS6ZEiyZWAVbd1wZ+xJxL7UuDk51RrVDIlMm", - "z+rr9fSn03Oy83QlaMoj8pPt9VTGOUQDWp6wW1+9/lhkCuFd4CNwFrj7rpwRmZuBnA2MYswPMcXOCj+M", - "A7lYPj977WBSNAQpvWSwsEzELuDVCztFxKv4xlhXeK3b6vcbi94GhePd+x1P9rlrXh7t7hbJfs+YJARu", - "l1i2BklENo8CHFcF9fnUvj6cwOeyxGKw3GBIXmvgBiMIe5EKoxbIVJpFVbvEa0lXNaASQrNsBvpnWMUs", - "tdnG+T5aBP3WUZbXSfeg34ppuLQBEyDO6K6cawmNK4wLAZcCnOugI6MKgs24JJ/1HbPwslW7kh1guWfU", - "1AE5uprCbc8IBrs1OtRZtHFJulu0KzbWNdu2NwD+0TFpYnP/J8IPZKNFv7QBfsS3zm0nzSVyfff9zK6x", - "SifFmjT8AJ9meR7ris20E26rlXmsPKPJzgWYWR0dg2H1ok8u/rP2A/BTL0+BuvCO2NVAHi3gp2r/ei2y", - "eYsd99rRzVRffz8e61bULbLcJ0ZRoW2c/YJmbEh+xIuRGJaCtAg3Fa+GFgr57lsirY7iXx0LGJqux1p6", - "a77mcwGhZxg0Dwwnjq3J31nsfPyNDsEdezvcmsRuR+dAi5CXuWh7cuFN7xd1/rtumF+3WzlL/ZrBAn+3", - "hgo05Zi9NDfweau9OJ+0zI0FMXNTbwh4dfP/tsAgN5bCFHmN/T8v2EUTQ3oZsFjA5OSs9NdUHEdt/hkn", - "/IV9R5dshVvu/UR0zVNUdRGFHTkupN2JMpVX0XZorxKuK/4l6ylyziEMPq4f0aDbZNtWwHp1Xv665h/M", - "wxn4yZYU45R5j0/quRBMzn4PjY6a4eKjJeHYpnVd9K3pg6F9kQgglsTH5je7x7BkDGK8cL8MXW8XRZLg", - "p8AcL7M/qltLKjvrRWnsxotihsX9umwA6NR6+6RseOqixe2p2DvFPSPzAJcdRV6XNZM6xevj7EDPKp1t", - "CAjr2sV6lqxd38qYHT1ZYGefRGL3A/1fNcTjJvB/3FDDLXJM/ZMXNEkuyI5rtEsU+zuiwrq9ggi7YvsB", - "5daRQBFu9ua0PxaWC1wsjMkm8H96Aqf4otmZe9ef8BKU/eEIRdejo8N6zoYdcKPburk8iIbTvjUQd3VD", - "8VstRnmPfo+PixsIEG/IjqFzODoO9gjuid2PNdhzoVmUK7b9rD9G8chB7+CYEsrF4MdXr86IYnOuMdU4", - "mNRgQ+NbMNqcsaAMV/PTNbJutT04PLp3/8HDR8P4Ug1ZpIa5HoAiM9gf0pT+LgV9p0Hs2ktXNMuOl/tb", - "+bzdgmJ87Szfa7btpCBTYMk+ILmLlvykfKV0GF5yEXft4Cdouz3o1SvxN+12yRQb5Nlc0Zh9YqfLteOa", - "cDXbd3VLJZTNAc25NmCIKCGAdzbHMdcXaymTQUwNvUoYsx3uesRyurJdWZU81F9rePh5Mya8htsTrIfw", - "sUFmfiyhbWnDFLaGBBZPjAyk73rZ4eQprLlv2wk8FSGQjZwsZ1xuS/KHodVTd/F1b96ALgZZxJ2lzns1", - "eGH1t8a6N6fV2JrhWAxQCjsmT4sPFN0WXVJUMWhsvfw7UtXyh9ERMAXoE/LmdEheFaP9RnungB0TKTJH", - "c3QoolA0sCJZdQA5Jtpw03zdmSUtHvMuhhBJ92xYVG1BA2hhQEN72ZQ35mPzunGj3F1ShZ7paBDehE72", - "0t5HmxDKahEh9waj/cH+vVf7o6tGhNwA5HSor8d13uJi/6vc58nrk6cHzuy5e21Qsk8OSh1mZU/LpAWy", - "k2tWSciVKpiqUMkIaElFuHaGwQ0lDJTwm5va2pV4BS1vAm07hAeGTfrXwMNuMsytoKuVya0bdFYZmh/K", - "U1IR5O2OZhEPZl5DyOD3itFLrCKwfm8jDsXE3mfheEMEC5muEBwLjVxKSjPTpMCQKY3f+0cPjh4e3j96", - "OBoF8JzXCV5GfIIe2U4DgBCchK4KaMMdDGOIyTSR0zqh3zu8//DB6NH+QddxWMd9t3Uo9G//FtlxK/IX", - "X+nNP6kN6uDgwf3Dw8PR/fsHR51GZTvrNijXti6SPDh8cLT/8OCo0yqE9LpnHl+7KcCHij08tvWR4F8D", - "nbGIz3hEEJeOwAuIHLAAN773QtfP5JTGviRG+O4wlCd6Y3Sf/Zhrae2taZ4YniXMPsMN6eSSwJk/xZ6C", - "NcaEKEqQXK0nh0q+Ne7Lz6VoguJMzKb5fO4g/4qlO3XZ/KXwxFkSH5NCZdvI53A3y4G9baMDN4eO1PCz", - "fFeA71WIwF5dFqlCMVLQid202qx8uhAXWR4kidal/CFXFoQDOyV0KnNjPfiuZkz5EYtnI9CjnYu4m7nj", - "B6kut+Z3wE1clMbpaDDwif4zBAJopPhXhb7C027jEdxzTV7aN6yhsPw5y+t1Afv4JWdQFB6dIC7sxq6b", - "rtJlWG5Bm7kP6rPfK3nnLUU0DmY2COzTathqzrAklNkqsQCl2Pipc2zeOV0MXtxqSOmw7oK9u41Fx4TH", - "AZDtAEwoN7PimwKnC1tD2YhIl24A+Jww87gCL9U4aececcMt+XAsXL5V8ZMLCoAX7TrYQAKp+JzXP1y3", - "s95i6PJVKNMT17Wps/piW8ygLtyAIR5gAadnNhLYRi9W8W7dnvT6vfOidpZjTPWl8T8GVqRMMFiHHjp7", - "fdUA5EzJGQ/hAWHUkXvqFDUfmvvz0eh8sP/fNroTyA8lNi5spBL4EYaN+kfYvttF9Pzs9VnbmIriU6Q6", - "urU5bUA5KiNryqyYYnFcrjnXvudKoV4qiKGXDCxMeOJqzSx8F4uH5ImNtUG5hs/Qom18MwuvAgzIfnvJ", - "jmtQORATJljRqXWlxjlQVRXnDZqBKKC91xMx4p6fva7jNTwKid8IuDLNYWCTNGAP/AGeE9ugAIw4/b4u", - "gh8chboOM+6zGgGhpj+jEUQHdKaQgA2xMY1tiIzPz16/dAWX2rCqgZyKokwWpnpIfilKksHy6hKPahgw", - "LjZI0IaeQ4OJTmQogOUHxRjBZ2UpVKAbdHpbdOMi0IDGiG4lhS/ICLH/Dmvwt5wqKgwXLCZvftDfkpHz", - "sr/5gdj8I8ezuK7GXdaVywehLW3NIzxbrDTY9uyy2KIoXFQNm8gFOmsmZ+WLzgQc0E/C5fg8xyE7UIQR", - "F/D85eDkxZu9NGbLfm1M8NBG6T4/e71bYctLD2ZRtK1z42WbhclvVVEdclJWpGweBMTRdqbPJnXA3ozI", - "lEXUB3U3Ny8qCjhPMVwDDWXg5jZMEEWdnZqKWifIuiDwPk+SoeOZfsgELdMzZN0V7Fd7uzmmplhGUfkq", - "JK/3Bj/KiM6jiLG4EfHSHHWF3o5JLlKq9IImzckdFwAL0YIqGhmmyDfvv+mFKzAhK+16N1Sos7icOpNl", - "5SoK0GPl3LUd8Tc/1I+nlQ5sWrWzqXPBDcLEITb4TjFIi2Y+JKd05UwvGV6zjl7KHaOzmQVLLe4RxRJG", - "NVZO1+TND9tjOY00NGmbwyt46PjUDkwIThEMs0+yGhuA36soZdXP3g9eGeV8Ap4p/GDTzTEkJyLM9mzA", - "EOIMWsgwrflcgDqoXcwQ3Kd+Jdf2rn53Bi1JTVzGXn3hatPpBy6AEMWEbiubB7quGa1XcpWXdTYlL7dX", - "D7WdtH/3iU9VbbjCnPrpzy1mtPrgJWJfdRzDGlQ1y6iihiUrq48U1FmvBsHW41AgW+AKubLPoPkHWwEo", - "V2xiFoppiBmrGSgO++tVpDVmJSyZK5xn51Rx1xlJUqou8ZR59Zvkwq5APS/2cBs41sKY7AqTgqAKa5Mz", - "TC1p0szY0mtxOU8ZZGlNmXnHmPBToZrQalR8M+NdtxQvg1weprisr2HvMPDdc5spQeaKRozYt3wN+oKt", - "wdHrupS9w5ZhwYWjdcv+7m/aX/fqLE+67XFoWPvbdthEV9ngV0/OfBGuosi5X+aD9VWGQhL2yLk3tmzt", - "gd5cDs5/SlgU7TWZfcqwPpzLNatmsfuoRKx/JzBjr5I5XmEOuprr7L6Dp8AuVd+e87eddMLmcQ+F30Cm", - "UahYPD5wWXpzBBvE3ASVo+bFYysp2Tu5cj+7TCqQtLgAXOZ63HeUq6TX7w1mblbHe3vA7xPEWz063H+4", - "tzn4e2PUvwtynMR8k1XItXLBch4CwIJV4qTrJLFHs6yD3Rzf3HY/IHta2wRbIB7utoqS5VNcwF/TCF58", - "TyPjS1qiIb0WKEGrxxZR0euCDHSY2ipVoO0+qp7PUTByBcJQ6+S/t0b7MDMbiQw0Yo2UjXW0RP57UDqV", - "KlS3UCrj0pWnzCvtxX3oA4E9/nfNpT56WJ3l/Xv3Du9t40PIbBrH3J27wFRt6zogA6sstzu/rgP4j67L", - "HO7JZrLCddlCU6+eBEhq84LKjIkrree9o8ODq61n14mc+GDOBl8K4VkBgBXKRGVmccoMjamhdSaDFmjg", - "MlgYi7IUEwdn325mLS1RT1WAqutW9ftUMTMtBT5fejz7lAo+QyXJtqx+WS/owb37x7YMccxmR/fuD4fD", - "q8L2PCuedduKPRtaW0HwGerFx+3DDaDzdJnLH72zx69+BEaWa2UvrT095eK48u/in+UD/MP+c8pFGNWn", - "S+VqPlurWF3b3ix3sOksPoaZCJeM5SWlDlF9LS4kzClAKLEgRmYttvrmwDALGuf1midXAMXckMoP4gpY", - "5FvX5tpFqEVhCzWV4tNVu0KHQtQQ+rkp8sNbnLGN+6a1dBQ1utfjYa5VZV1vLES7VqcwY6IoPZsk9q9I", - "CkQlD9WhrV2R/lmHMoZ4jbh6hcUnqz8WX6/8+KQ6kMrvvgxi5SdXkPbtFQPZNgqkf12XQ7dzIS+ObjnM", - "YfN/cSt0Lf7tQDSDGRx3fBdeJ9iz/vUX8//67f/oswd/3//t5zdv/mf5/L+e/sL/501y9uKjAJo246fe", - "KQjqJ8M9xQjHGvhpV1I6Bc/MOj2B+teywu6JtTiYaIHlismUHUMI9s/cMGWLXzZSlqFeA0NNCd8iUmCV", - "QodmsQsvn9k4BHj5Dy8Gf2j2ETvYCuU2pABK0vk0linlYncsxsL1RfxENOoF8FdMIprZCpZcgGEWArsV", - "lht1juHy433yB82yD7tQEoUawt7bUgAZxYqNs8I3IXxYhxuVDV53zVmBt2zziMeiuK0LAFEbHTAsq41w", - "ljQz/VoWZbP+5jSnh6MQ1CpmocFGYkUtVEEKygYyKtLjyMPR7ro+t0XHKGhoA/m58BmbpgxpxgFSbElt", - "fspiHiFf8dm9tmUlsdpSmjPiZUq+X+HevLQppzGhuVkwYVxgZJ9EUl5y1sct7RNwzmK8Fr5po3AWMhtM", - "V4OFzAoYlsIXbuNY6kr2/xm4iQ7eMMVn7ktBNA0gkYDQiUfGzQzblNaFtYm9ssWIhEanvGtuq1ppW8jI", - "JmCYXAlfa4dhxV0EGyqoj4Vk8m9JlHC0OukF+vAWiBZqoJsQ4GdvVCS60WkUs1nz3/VApIN791GD9f8+", - "POicZ26XbhOV5UlAp0096+vAsS2bxAFY8WDiDeFbQgfhBrScmKCdwkj87znxHZUnrmAk1tNnM1+1K92S", - "6ErO624wOdEdAwcqNKG56Tq79VNYy3Xs0IONKcLXkg5wRs9wHuTVz+fEMJV61IudCHYHT4nFrhlwrXNX", - "RfDxk9Nnu8NeECWu5tLCrdqYC1kfdAAhxZ6D1tCp0kZDU9YnJ08xx91dK6UuhklJP0hFEnsrlpfRMcL5", - "1M091NYfPXnqBNBkVQYqWbFl3Nv1PWbN6+2YvPSfJbQYSpGtXNKW77K8TLBbF7ZqM6bWem8kt6N/zKl/", - "7j7G/Cgszmp5MSLdtt5f3W2ObsXhYdNCduULqfJiu/2rsvefGhP+08voh1eT0Z0XepItqA5R96Lq1cRG", - "a/tedWTX2b1qj/SrfXeiefBs/dWXqKpcQ4bBPVd7PQSCfG+wv/9q/+jq5rurwnnXkfoqGJ8Fond3KO6b", - "gLQOAFRzM2lNCSHw2CWAeLvIm1PIrgTkNXjYsI7sHz7oYpTAr3ZNpqg8BJEah1RwKY+8VyQBWADES54k", - "VoDRfC5oQh6RnfOT5z+d/PzzLhmQFy9Om1ux6Y3g/lwD2RtvAVxHGxwVAF+rAXmQIuP31auf4UmUMEya", - "snL45fXxvreaFjvgf/vBPT97DW8sqJ74cOv2DGNaZumz91wbvY672Clr4WPwxu2rpWGsyyRtHy6ocito", - "+Y81VO0giufuDaCN+5STteW8AyDuu0zt/fxAwDfCdn8s9razM9wQ9HbrlRaCrW7gmNxru92uD6J9I8Op", - "wUCF2FZVwvG4C9dGre73eCDn/LGL4yMnZ2WZutIZ4btvzOnRwXD//kOstLw/6sLYUxpt+Pbp4yfdPz46", - "sLfMMZ0eR/ExKuzX9Vk5wrYqCE3eQWbu2CuJ4569MCvabeXY2jbdci7WwcGvhwXeFONaYLlRnPWBS3qS", - "biyU1CGpuAl1mOYONDblScI1i6SIdV1GXlBNdGaxkm3BoEKCHwscYJ8UddtRSiE0ilRemh6ddO3k/Txz", - "dI9FizMpNHNVS35iK01Sjk7Q4vMY+qhJkbsWj8WO8nmORUIj1iuO4QfMGuoTpwr1MWoYCyPBC2OhF7kB", - "JrYLCSgCyhsrZ5UlU44eo12ic6vS4nhxNVbAMDWPmRoLaBZAY/6jUE+O7wMyQr9XaHJQ3GU0ClHTjTo/", - "hw52HXvAFBgqXDk9jHZXuSC5iJkiL6VmxlDCLDk0Q+Su6Dj9SIxz/3o38cq9XspV4YO5DRK9kzD10RjP", - "ONQW/RyjQ6+hnN+7vojeKVPQy68uR9C9NblKBINLyQCNbwq3nTXIsdiZITVzuR5F7s1rWzi4PnUXemwk", - "ONXVirw5Pa2FPbgMjW4TRy7Rsg8yu9I2HGyxkWwdzVXcyxVE7O0o2B+PTN2UVCoS4ifHoa56GD10gaXQ", - "mmGrpjhvsq+BVhnMgeLC7hMQzYYJNmpHW6CXMjCy2qte6ZlDZXWitM+pcIj0BWhL6dTXK703zfVeFvE9", - "+1zvIaLOQ0TUOQpCHsRsOcnzkGoEj9x3yOvXAHuDvyAitE2QqX2e0vv7D0cPHw0eTvfvD47i0f6A7h/e", - "Hxzco6PZYfTgcP/gcENSUYcc2OuntQY15kAccxG1PvHR86Gg5rbchYZs4uKx33ERy3e16y8YIFv9ugu+", - "3fb59dD6zkMIpgQlVBtrvmjhZKd4ybMI+sZ2Ps+6qBMXNnTefzXa/1jrDw6u5Y54pXJh3arYoHQhpJUB", - "VzerPs7rsXwckE982bZa1Y93X7TR8b1Hx/c+dtF88sa2MTbJ6RY3ty0izCOnN7JDfJJwxY7kDZQ9JxNZ", - "q75tvur1e7mo/o3CQCOWunjcKYmr7cD2w2xk07XSAnlwUtNXMFLFImfGxyCpeH0Ea74UwBogAj0BWxip", - "2OIskCD64U4qugt0g24xZ6Kzics2GQMzI7l2+XxcACNG/yN04vARjslzbIuPaGrVOjcIW7i76nqj8crG", - "y8D58p+2StbmIZ87/QrfAWWLwL9w2rAMzmS7uQsrnR2TXyS+U2h7QjZtv7Y5qlnrzRttyY4D2fd4N/gx", - "J2oekx8K8bIQUJ1AuqOZ+3PiGFYJM7VbA/twOw6BhJWdqwBX9Ht2RXv9nl8o+DMAdfG6pPq181clxVAg", - "GaMJnuUyTT83PHHg+jgTrg2PtEsegc1tE3tcbiaLJ1Z5aotJtamvtg0pXvJS1ZtTsoP4qX8hzrAN/9ot", - "4ldrd93Bo6NH9x8cPLrfCSWtHOB20fgJIhOsD26rnBxl+cTZRtqmDkX0cM0ja1UoYl/A21WSeaYksB6Y", - "ue+w+vFHw0dVcLhY5sCHiyE5JEmLRY0bFsQ/LHhRSxzkbzxZ8tlM/PZ7dHnwd8XT/ff39cF0vwX02n4o", - "bHY7qQYXrNmo2XRgS3uG8buQoJRuhbh7yTTOgJwzQ5B+BoRGqN4U+dSO5HJdW/EgYR0dHh4+fHDvoBNd", - "udFVDs4EjXCBS9mNoGxJsCXZeXl+TvYqBGf79EgwWQDmoHHOiCvQPqoLpMP90WGISlou7pJqXN/LtHXJ", - "3zj10U3KLTombxeq5dopD6724eHowdG9h/e6HWNnHp6o95s5jGtnv+fLZ1R3fgelyVePzwj0rmY0glr2", - "j609eNRaUTHXjPzy+JWFdhKSZEwNyhvR91RPc3WxZlean7nS/LCIjC3+sGWKtUiVB/che2u/G+pjKHzF", - "4ZnWjn6dCwaOb4C8AvsaWIp1Jt5vu3dCgpkl1ZcsSihPH0c+WaNxj9kiDxNlm5Wb0OWKcbr82hXY4d1O", - "ZrLC+GVTffAdIhXJRVFaaLjduftJfLTt/N9eNNvvh1CCj4DVc/BktlzjNZYyU2zJZa4/QUfS2OzbWSKl", - "utK7barPS6bBg+dBsd6cfqOJozWiDcvq2pijxg0gbtec3JXOc41EwkTetliddqPL1m+acL/l1PY3QXPU", - "uEErdGIMnCsX28NIn9AkyrGYFi32E2aFFwWCC2RZsrJZAkkipSDRggp0tyhXuk/MCSWAwjAMxtTCk8ks", - "GJ8h35FEWtB3KPbj6kzZQcBrIAyBZWmnWs3RklKj/PO91DIZV0moTo330nCl2jA6U5FUD+tJjazgkdtX", - "ajbVRM41qpcG8x+GzTIYGVU2rYEKWzdtmVqtNBC6HRhig5mHblR8CAfTqspOeMGUdbuSNFJSV4DF3pzq", - "YdfsuSIfentoaH2wHUjXukYDVxlecbpzecXQ/RjIDPqYGxJpGLMPNwRdejNnSkWOlacqhOxM+sPOgZVg", - "154UgFlXHKw2EywnA6bAwgdW5O8XliXfJngvetZ2neVyEczXenuNqsJdtQ2wnacGVzS8Wv2CBkNkvI5Q", - "txEUr4QKbMJiXQWssqw/wjX2yisYhGQHs2cqbKmCJbnbJdwmrPzCd9b0XleJGDweXTEaN0MynlGzOBEz", - "uc4JruKD9TZtF/eaMeVhEmMmOIu9Vlo4Y53RjGtXhy7OmVs5/GwNSA+DJShqTs7aBslCdV7f/GAXQ7Md", - "w+ZqM/hd17DDTnIdTs19pXJcKyQgIJ4ySbdTOCnXk7BjbL1jxeZ5QhVCB3Ycsl6lCReXXXrXq3QqEx4R", - "eKHpYZ9JSCGbwCP9Hc5lt9Ps4IVJW2myczs4l2FoN6Tx3XIK38Esdxv5zWjT2bPv78H7neLQgjHnP/CE", - "OZTL14K/rxB6vRzD0cGoLe++pdNaxv06zOtVObcj2eCJz3UgS3GjlOOLqrHY1cjA1lgcDo9ah1vJgzB7", - "Z+L1fEP1FJSPAzl5Yvl1A+KETBlmEPmprXXXhS12mUqwtAxU2/u7nNZNq10DiAMFC3dECbah2CyYKYA7", - "utG0bVusrUlld6+CZoFsFSaKL10RJGJbaccyUquNn7xcq3K4YG7JuJ+jrXjYoYKPjyQpgBDcV7sjIjTr", - "UQYinxHvRpsVVnbGclqrSpFVKFOmFCLQg4QjhZ8NArhIZdfaI2hBXveKKJZSLsaCi8LcirBsjAi2ZKqS", - "bysVKFlzSPb4a0XFg99ZmpmVKwaBZvhvNAFnnB/jWFQHCZ2DGX1IHgtrWVR5ZmrlYlnstD4gFF8kXzGj", - "sH4qNwsyU0wvqnMP1cwFGe+dVHFrMbIV8U1gGhS9tcTISyaqrKzoJqga2o4m9q31eEBb8BqfOv2T1GpQ", - "k2aN6c3fyzVTYSGxmFLRpFMQTOWoVJQTPJAwAFdb1P1lWXyBp9IBPaXs/r99l+VPZ0Xn9d8azSoIKR4z", - "/LE12wZNsFGWr69Owye3NegFE+o24eKseyXIjg/G9gWa9G5LoaRO92S3nL5m2oEfzZ5mUf3rRw/vPbjf", - "sVLVR7n9LA7Yp3byLdMNzr2WnTrt4kF6eO/ho0eHR/ceHVzJw+IzVFr2py1Lpbo/ACNi4LAm//rHP9+c", - "1qnl4B5Gc4+uNKg8ax/S66zDgN6c/usf//SjuvaAQoxmHe6/JQKgNd4nqe6kDzmoOwO7Ock26PePa0YC", - "WrAZssMQ1BvCd+y6DcrBNABFuknBNKMRN6sAI6fvfIEL16SB2t2h98ZgQyKv7dvhqwLn0vm0aAExLK7B", - "f1ovc4MWHnYueKfzaZtH+0Xzq9iOlF6LarREh2AFSxFhA3cxn3dU10LDY2b9Dv0iV209b8e22Izf20yq", - "gKEgJG+tryhUtaAhT7qXqtvf2M6K37Jm1mmu+NsN57D9CF7J6hu4kQNG3yjLu3ZUyu1wAV7vrcm0Wopy", - "Y63PWt3K8ta9+nc75CGvFz0pbrCrf6+SenmVFxu0ZunRjcEtedl3v0YSLdRUyaoJGNBkwgaVMgjYmujc", - "egThzDvA+kAuaHQpZ7M6au69dpR1BBDCtDH/FWoMaCZ9sPo7m0UTotuiBY179/S4R6SClMx03Gu4rYKJ", - "mCl9P3EfqKPEjDbBnpeJ9I1Baj+DaSKjS1tlEYv3D8mIpIwKTXKBh7/hVdsfbfYO9XtZZW/8OqJyt7aU", - "bjPJlC3okmNlFudTmddCOtl7bjSGnmI/xySWFjeqVmLazRCa2TTJ43LSeOlQsXIdQ4fQTgofG1u2RQPf", - "DAtbi9+Zkn0HewAc+8WL074NYMAgRjuwWqSkn6gdQa/fKz/RKNRQ/h6ORAZnBY67Cfyfrq9jNbsdPauK", - "aWa0QwIvyaFBBJC+JEyzIkDaTZGrJ6itX0m5wLBBF/4BC+q/bgmExCzCE6nXz2Kd0K9B3I0MBLfSoRSE", - "wxAJ46HYUMnnpXMINwdgjQ2V6vC2n2qEuPUSTrSRWa9+qifsfcRY3IQODTfpGnXv3gxG3f9MHdpQUbjd", - "tcbI6fXZDW8uVQzH2rba1ewAIcUATl2xpfZwWuQ6j3pTJ7Tq6lfBMSYhoNZQgw7jF+z95rX+hb03iLQe", - "5wmL19e2OJKWVbnLaNuKXztHsu1AS8W2lue8gbKVNnL9WoUr7avsLmpXulY3Uq9ybXfOmfFtzx0Zte5Q", - "vWRMzaXluiC+ST3GxpJSn7gLnuynuw0SPFqErSIO3rdjtifYW8D5PuPvNxCPbWAV4zpASnmQPDUQi1S6", - "k9L35OgBFhHTjbELPl+YZFUPwDkKoDJ9VFFXxQwT3lDYZefL3fQvrke7ue2s9h4Sjs8rIENrxVGcSDrZ", - "hMD9pGjmrfMZXaEVp9VJ+ODwaDQ6PBhdC4LbD+sKy/WkfMWVQK3305acV3mPsKrHzHKgag9FuvZ6XW1b", - "+K5cTaKNYjQ9xhSejEaMJGyGcHtFaux2z2Lz05sH7wQql49b0L/fKLdv3gdfL75TfErMK/+ypg+cax3N", - "qPp8i0O0hc1Ea+B8gey9w8Ho/qv9w+N794/3928CNrtYpLa8kQe/7797kBzQ2VHycPXgt/3Fg/lBehjU", - "wy65iLvS6k/QtjXKprwk66hINZZGdtwcMqaaBdPrViNAc064YANd5FptT3jcwAus/33r+b+and/OYKPs", - "cF6fZFWEoIboIGXdDZKXm8xG30VzNidPN8/iWrlMzYGE6a05FGjVcTBY62K/95EYD7noeA29rjTsfBFt", - "zK/bdhWFPOx40oO73LLiIfJuQDxUZr3pAl+/5AK207lU3CzSzbdF0awAJMe46d+1ievIUUNyMhdSsfrP", - "RZhcVYmCl3v9XvL7Uf3MuN+7Y4g5LOOCAN1WV6WCDmFkCVuyZPMqYJNS8VA2kp0qhgvx3f5g/xHGISS/", - "H303GjyqRxz07WpVl2/ft679OuqyhtVigr4I1f6jK0Vc+/XcREE/8VApvPJedijHjsbL2vT+6vCpu7UN", - "Lh+v7XEDE6hVAP1YSc9dbpOq0BSDRTaEcl8x1OqG9lglMjJlcy50F7vt4agw3N4Dwy157KDGbUF4WXyn", - "2j3QSpVOeJqymFODo1JmQwbDQUdbXFOXuFqVE/9WQFobhsW1R9vBFrYlXG27Jocfkdn7UdpvN413Ew4I", - "2tW8iopoZNiwT/gMbND1/n3Za6t0YGIkxqsde8i3kmQdD9ClHOjtJH0yl4aUyfgd7W25aLcLFuOHGp35", - "RvQNSxAHnwRapYAi45vY18lTkikZ51GZP5rgoEvsEJWLYVchf+vQbtS+gSneYOTYbt9oM2h0s0+27XfD", - "NgkE277V+6PtW30jRpF+L8/i7TzMNurGwa6EAb8lBTFgoqkve0MSrEzmbQeO/rK6gus6r7UlRyAS5Zl3", - "sABNrVNSwN2CLoZQXO9TljDDQp0QmcRllgTXJRfdzlL37z9ctLk40SO1PpCfGMsINRZJCb+XgjcvNDBf", - "wLS4S3ZGHjdcW4fXwBY+cqtVH9yDrZJY61ZVTbhtxRgsl2/YvC3yymXF/O1nUxPN1rFUPMOvCWkv24sJ", - "uDd92Fur/fgmzHJ3KaS9cK6HBgCshycucNb998tYYBDr6sR7FHLPh8jiFZ1j1b0tfpRWUglVqEQnByWG", - "ujJZWKoT/kmwHqn20C9eQMDXfUavK1tIuCF0Tps1KNoKXFpP9PYYovaSMa/ofCO6bzMdtmp+fzz4mzW3", - "k8nweO+7v/zfg7f/GTS7NwwImqlBzGYYcXXJVgNbzwmMFcM6ti0Wk+hpQ+fuzDCaojENceMdV6qO996o", - "4J6rX2i6NgUMVasUY9rfOqG//Ed7oFdlGV/jhbH17H50rZWbqElrpL+Xd1Km5j6o3mfU7Q7HApP0LtlK", - "k0qJt+mqdmK/0cUrlVB8cmHJfcjE8oJMOdbM1GMB6j2NIpaBWuWqBnEci5LIhhWjSbUfV2rOnxfnmbWB", - "FZA+vQaM/OL1q+9fvP7l6eTF2bNfHp9Mfnr2P73jnmDvBvYL8QBo7+jefVfuvbqS+8GSI1evnDEkpy5f", - "wcU8zHLU7BH6DHykkIKWrAh7HyW5hrAJXHpjkuvXyFjPWr5+zYmPBFU2JgmFZzjwbwD0hjHbe9VFF3Ht", - "iZFrrJPvrDxckHXRxRLOuIdXipFekghWKIGtCK92ubH1Rb+/6HaXWLjZwGHH1F84bAEr6s9cI3CHD4Kp", - "NCY7mELji/naJzYM9Qrwt4+LDoMhmJ+4ZtPo0aeoq/p6YyHVpUwGoOe1FJ8ImtXtWgRTCLArm5rRa/O+", - "zKcBZcbZuOd8TgMOl5Bj5ZPUP/UD2po6trb/rYXgwgkdT5uVOeyxtEvVqCTRsJZoM2jP90hBvG/BMkaw", - "ZpvEyytBhrXu91Jh9lyd4hBwSCwRH35T2nZ5yjzg5ABf2p6NvFHBrMysMpL2vTmVeWhbNi0Q5OMDj1Ss", - "shH4AouvuWQuQakDHA3SWx2dzL2M4jn63TXZKUxgfgmKtOt1v8Dmihan9H3xBWgB2kTTEYvzKCtq7T//", - "HqsPvHS7BOzTdYHDaGi5Yaz9OhVtWhNPVeubUaWq9Xnb9sGD53jVBu7XdrYaxFl+o0aaIXr8K+XmB6lQ", - "L24Hf7lxyH68/GOmEFavCcjfCc2epyyeyNxsPv/QDKRHe+UXlWbLSsXeBkCRiKNaXnMbL/DwJOUY1lca", - "loNFueJmBUYjp5lMMR/UlwfGhcQP4c/lh7Ek64cPaD2fBTJnnjPBFI+w4K2FGhSoMQFsX1n30JbAXIPA", - "RRHoxZMTZ3fxKMqoR3ODpOcDUB+fnYAznilre+iNhofDER7mjAma8d5x73C4Pxz1UKta4BT3plCw3yWS", - "O4260OBPYicJfe8bwZuKpszgG78GIBEwANM1RxWEzitKZEa5clpklsi4IBgOb2MFB3+hHttbuW+XvbPx", - "GPOtMQ2IZS/c5r5FQRnPDk7zYDRyWPHGXb+YxGQzJ/b+7sJoy+92kurcEgUKGqypeV62LJb+Q793NNq/", - "0pg2DQXPbujDrwV12cwMtfN7o9HNf/RE2PxEl23u4sKqJw4JqXrWfn0Le6bzNKVq5ResulqZ1G2CMQO7", - "lmtr9TijSQSsAssuAaANs8/BJEptCLfKBVar9i8ChdZPge3bb3KB1vS9jFefbAlr3/A2ig91dgbH5cMa", - "PX862inIeH0j3SMPWm6p9hYI6HtalFq/s5NyNHp08x8F22/CI8hW9QTsArO5xtinBCHYPQiTVOS3XBpK", - "iryGL+hIO5l1WpBbv7yK9v7g8Qd7vBMW8gecMZVSYbNEbJsth37tOFvfTHmcN95qnvBPnvbcTeXRiOxF", - "hYJc/YhWr62mMLh+HR31jtu+aacX3yHhH93CCXeTLart3uWRwxqn4Ff+ko6T8zVOSyEkKMs9Z+ZzofnR", - "bV5Zri7Dn/AUfSkE/JwVEl65W2uXwl6mcmEV4KAE+LLM3HTvfVMX/l6VTyrhQujXgK6xQoixDmYar4bE", - "r6lV+sHoqy1Ev1racLP6+TqD4X0uJ+zgNk4YzrjwFH29pr5eU5tOuaUWPwU8mJVT3sEGcSULxJ/P/nBl", - "68NX20N320Mny4Ng72xrwIMcEheaG8mYEb2AeqfgbLbATz4Ix1A1nP9OqIoWfMkQ3Q/r3uWJ4RlVGGKT", - "EvASWB96q2Fio1mi6G4Puhv4gMxygZuAHppNMARm0gbEWYZiciHAWEu1j5opcRXXKrTbsx80sBcdllcj", - "ebeQmhXAhsJUbnPM89ZWO8Zuh2PxyiHewgJiVLnnNZoliNu7wf4jBaFj4V741rMQHxGnaVpyLqoQPJFb", - "iE67LWtTxZFOdCRDoEOvmKDCDHTGIj6D6EOc1iVbucDWYIedSlnBgP04oS6MD9Q62A0D12HoUhil+Gnx", - "jDhKqvtvhDSuyHfp5PJYSlRB7YNghZJ5Iqc0mdj1uWQBn+BzbOEWpXS4lN4kIWNmq/JnK7OQwv6dT3Nh", - "cvv3VMl3minwOY0FZqS4tYZohUJAJO+wNl6aSWVsnAN+c88Oce+PS7b6MByLx3HKhX1D21dooiW4KeA9", - "Iz14iOVeLfRgT1PYD/4k10amVQhYT3d2mDI3WW5cao1mph+CPx0LI8kfHuTyw94f5Rc/oLOY0RjopNLE", - "Tgll67ZR6wmF2U+wacDdznABxj24SG2Yx1xRYXBMJUonmVe3dKcoE4FFaJsrHFFBMpnZEhtIVAsKJFfr", - "A0EraJIQg0fJvwuCO+5ky3wcBmE6bQUgtIhxjWPEBTn9vnKYRkcPw+dJs0ixUETJf52/+IXgrQx7YJuV", - "4Vq4RlyAwABRNFg7xPG0ZwCTZB1VWJ9x3OPxuFe4c+NdHGuuXbjMYIA+xe9gaN/Zz/R5/N1wCF1Zd+Ux", - "+fUP28sxnKUstYCo496HPqk8mHOzyKfFs7fhBW3DcTuvMQKyY6+5XeQklCPkTuXGt1ckmO6kuwUAQJCU", - "HKgauDLlgqrVpozKwNK7FZQzG8lYWYw/xhi5OO4dj33s4rjXH/eYWOJvLsBx3PsQXgHntWwvBoj3WeHc", - "LIjo/mi0ux0S3K1vwGfZwTHwiXXAVq2oqGQKO+jwaP9c/oF/a/3z+0JjE/DxEqPJKv7e+H7Xcj18/RY2", - "/xcQe4B1pAwjSgbVmmxwU2AtQbzasOjAtTwjFVWiqiIHfCMNfYCKiCVeH9hugbp9rwZQUcSS5LZPzl2d", - "m8JvV9RS+KIcdbhZ5fne6Fe4Y4ob3dZtV/Mn3A39fnGG/YBZ3xn12dLHYIcrySBKkNPxiW0MlpNzHNPg", - "nAlDnuGvQ/dfr5Qi6uVFIucXx9amAEURSMKFS1SoRFCD3OLWEl+yQEHFe/afRRm/HSvi/Osf/8RBcTH/", - "1z/+6dD3//WPf+Jx37MAeFhP/GLBqDJTRs3FMYF0vAFNIL7dTQYr/rIlUytyOHLOCHwEQxIsKqK9ZQ6I", - "mWPxkplciUpCh62op12HzocB8+EiZ9oBLUFDPnPlfmzQZcCg5M+yXcpbPdH9AGAlzqAyAbgVPQ0g2h+3", - "RdWditwL23LtnGvW3Gb86FoU4Xb+Yth7Y6l3YAd4RQaDSxw6d/jATZrsnJ8/2x0SVAMtVWBJJ1Rqym6c", - "fjP8ypO28yTLUeoMBVfZ8qaIZnTKE+5toS31aOwRTGm0AOCoIvC5QIP3XRz7kQKPgRhQF6HZx6Zj8eJ8", - "D22/hkUmV6zvOIFyGLBlwTrpEnDwC8i/uMGwtYFrOxYzRqEPcvLUMoEKTHqRsVl0LBBqBYNvuanVxuuP", - "hcX6ddjScPBAkk3wJfz+nBr2jq76pCL52vo1CTWgqes+NB4LmwPp1mCAYDKkMswh8jM7pIEPMXbJZIrN", - "wGqLE3El3vHbO2DwcKHX/TLh1X/OpsHaYRHpxHGY3xxVVGkNldjTi3O/G7t9oiWJEo7UEFExFnOMUPLw", - "ylLUdrXIdFtQFQ8iCZdAFW4LQqMTFs/beOyTSl+9G5Rkat8JHKcfm+T6pQkXi/UJwCG2EIKbXYpPXZtu", - "PkXX45/JqWgX6CpeRWtaZpbf2NX96mHs4GEMr5v3NoZcfk89RubNhSLbT9xRJLKnvfU1t08qS3YXpkay", - "48GH0F0jFTl7ckJoHCum9e6/tyESZmqptJT/4H4EVnwXMTFuLFLZvfD2ljqBfCns4KUbNaF+Xs0KyNX7", - "ba9WHqn1pisqJZVX3s3fHo2PXuUaKYXekta+3iRbo4C5jiCnskotAxSNElaIL8U5rVLRNquyjS8urpyN", - "4pJtBcEb7kDenn3ZfToXzbvhFpji0wZDvENGWM8Br9Y1/5Ko+XWxi25em8zPnxdpjm5PCrptU3SIzL8k", - "dTFuLBtwQYvA0nqBPmfG4q7cpJ7uvhCYONi53am2A13ZWRfTsq8SCyCDE0JLzGbd98Q26ab62v7+TJov", - "Ls9VJBa35F9FlA7KbrlWmxTcE1ek++b02xrE3C3H0zgCCywyPHC2acRrgrqXVK9EtPs1pOaTUzQudkWJ", - "VX7eLC4s2fi81LNuS66DgutlRXiuC72WCzJLAL7WOgFiPsMgQlOtsI6jPLiFURaVzBU1zMVOfokJyWd5", - "Yr0bkRRLpioQktUrde8PjKbdrip55rXxdn398ucBE5GMC+dJu0zqnnxihcnSfy3J+PZP3ReYaMu9eNAm", - "MH7E/tsod1Jgiv6vgx8cquj/OviBJhkX7H8dPk6oYdrs3hixjG7rprttBeYLJj7QX3h90dZY056h82qC", - "bJ0yPa7uVSizhofrjd1Fhs8no75PL+Q1QYQ7SXi3RvegSd02lMqJr5dgt1QqD6vVwEr+9xb6zmsEfev2", - "utrnucYRoGD3BeK3SOXhYKnD0+aeuB1vEnPFtN5mjChadbRH+PZ/KpOEnfSVjBLFun61S3SxS1SXa6Np", - "wja8WeOE/cYded8LYgutNj76igN2Cw4dR5EVHLCah7tEAltIbfDR5wEK8e+UicOLo1C9zzq6TEtOsfFe", - "82fq5Gkfd7gPe3rytEwGvyUHqh/HrVsE3HdvXxp7nE75PJe5ruaZp9REC6YdBkPC6jfDl2arKOWGVmvF", - "Z0ylo9u8027dGPGV7m/ITNLcUMu8bRTENmXEt+qqjLj2oIxYfGqHU+GqSfV9pcHdluwUj1DdlYxrQN4f", - "+l3GFVKSoDqHJqUeQ1C1OR6L/w2K0a+G0fTtdz4hPh+NDu7j70ws337nc+LFqScVJozirjDs41+eYujJ", - "HLMLsHZsCb/RHAdJc20s6fkiMf92mlsZfdNddfNU+FV166S6VZZrs+pmG96w7lYvNHXrypunt9CC22d/", - "UvXtT+5TrqmWOp/NeMSZwHJdaJXUa0HUVsX86k6+nn2UChfEUYnArEkindXIgmttkdCLGtCfNMSx31q1", - "QxJqDEszA7BPEYPqwTZ1UC9yE8t3whfxwAn6enC8nE/oevddTXwn4czdcFH7rppuUcjytlVd9+EvNHVW", - "Zq4mu1MuS9GmXbu8W+K9WZ2yw1V7+1rll0xiVn1bX7oMNIRAUTrrf0pzm2dcvFniWQ7Jq1c/+5xiUE+U", - "L3FopK9r6Gtfj0W1ruGQPCsLRtoGvgdQH1jsMAjgfV8pMGY0TrhgmITBdCj9t16N9E6PxaeXgMOlVm/b", - "997lWNoNvUMJ+M5Ywa3ImkWwgXGe4uJoVqqwFqfFy5t4ar4ofuUYUIDxhGS9PZobOXAoBXsLaTE1w7DC", - "ZwmNEFUYmlnASwcMYxFuq10h2ouSScKUBTLNcuPFLcAUdoPjwsiiVLeTzC6g+0kuDE8u+jYGEtommkCZ", - "eAt1Mxa1jzmZD8EbEJgER6hYZkfcqDsMg+ZgaoRWiLNQ/SShyTu60mPh4B7s61isXrHIYv4myZD8KBFp", - "x9bfrjBeW/z2Gz0WFzxO2MQB5VwQroleSGWYYDFJ5ZLp+ncZVQlnCifxhMLKaZLSFUJpWlRhuz4yYxau", - "sgbHI+HfVMQcy6jCl4spH48FJQejEUkZFdqBa2g6wwvH9UFwELUBfUsoORo9cm819g3h3v3y78BpUoot", - "ZUSnUDEXqNiWJ9/FDUxdWWNbJx+2b8aVtvtV2DddvcraxnLtq/TGfZKLEj4Ebf0V7xpsl8mVwHk69yTj", - "qrgGHWLSlEUU1lPI+ncQRFdGUa5CFyRsdaW+9r+j4FiZ3jkuVRicI0GTQQQ5ABKRHxd4piUepd1vW6iq", - "JKo/x0UTPCRSEUoqdF1aNFiUI2vcQdDZi7JYrPDF3y92v/VnB46vYwT++Iv5l3Q/IRHJ2ax2ALdfTfYA", - "b0qKWyfhP+s5feKYXo3FxZzOhdSGR54ZekzvqtL8VSHspBBuXtkgNc+kumyPhP5BqsuuGpiP2PyyFLHq", - "DD9DRwQMD4N07t4fgdZwq6wA0dy6ktakr+KUotDFjS5ilUkixZypqlX+1t0GVa1uxyJtwmWqrLO7wEUD", - "JWTifrSFxivx5uhiiFyvf9pgNSQ22OwyZm26sqpHiSR3NV4Jp6oK+GB1QW3jD/peHEK68hu2g9L7+nYF", - "mSoAjW5Fai0+7mFJA1CtY/Fa2+IOF9b1dEEKHgwCrS3YQt4teLSAfvA37N+iutIsuyig9HePyXM8yJVt", - "sx/fsWVagNa0TJhFY12m6cXxeqntN6en+BK2Wdii2hfHxJfXLu4PDa2qMKwwi4RqQ35x4LI7hTKOO3oB", - "NszK/HYdQGtZ6mAsQmCtgHVqO+QzclHBbb1oARX0/PZnOdefjauoLFBj52Ikcaoj0iYTca8tyIMnYcfP", - "/mgUqpvQET7WDuOG0WPXBvOznBfFcWqkTLOsK/m6YSIVL9N0Aw2TnZKDEG1imZu/aBMzpfBlR91txE12", - "aGT/YeglEKpDHvUHe3csWpbKzjC8VMBUe/0eE3naO/7V/WuZpr1+z40H3ntnsrRS7eIKQv4WON5mhx/6", - "oR2qYO7+icXzO7sSXSGl4qjEktnoSJR4nAYFUK54XnBLrwP9W7+hKti/jevO2QDa1YiXtsGf3sXpDY23", - "Jr47jaYhPgXp485OMYpZDVv45yTJV8Zbt5sJWaAq48ge3cbIEHqZRInUrOYn+3KAG529sCF6t9vb/BoP", - "YHhx7kvwdQkEOnevnvs3PwMTxraQGz9m4qd767E36yP4kkEo9NpsgP010P62BeV89oT06bZkbapdKOQr", - "bV7dWNuJMEHdWmcR7oXYlmSluZEpNTzCcoDRQkpdIfsCmt8W7nQ2+IIy0UJljQUuEeMCSPXCWfMvnIh5", - "7CyPhFYfuW8M8XWXvhF+wz8q3/ihYlwpOH7fa1BYmUZDyTTF2YxkNNcM5MwcLGKrKGG+/iMDv3pEM5Mr", - "hqVtGUm54GmeVmsuwI4tKeJDXeynF30yzQ1JwMqmioc+ZimSacpEbOWksVgwuuRMwaIk1DARrQaaYUn8", - "JSNQ9iGRNEZLTRZTdJhhIqViQIFYwCJlhsbUUBR0LuDET2wu2EVRJd9aRwR7X1ID+I5z8a2tpgPdXviB", - "XhCG5SK4XhTVlCMaMxEFyyicf95s7NOb9M+ZaU70jgKsrsVL7zLiqmq69sP5PIKxvljIjw5sfoPQq9uV", - "6noSjSejf88jbefq53hHfrpiiTed4s/DQVcQ3WfjpLt7L5xUJM7t5yqnEsn8z2pHLBhKDRECEnTtNl7X", - "v1ZUZy2W+Uo8b+8P/+fJNayLnwkn7Lcq9m11AMtJfw4s163qtXjuHZlVnS2papO7OxbsBnV34pNUFS73", - "pRhba8h7Bd+uciejKGpfUnxl20227eJGrsu2vW12LTKiwsi5GGCobZiDOzNuK6t2poN/06SexuwqLPPO", - "WWTpubh1GE3PGjO6AlPJnyHWeoP/KJJKWRQRatiXBf1dsRpWsyzQNlcWGO37pNc3p6e7bVxCmY08Qpkv", - "mENUMpvgtTReN+C+WDKleOzhTZ+cPi19sioXQ/Ii5YYYSS4Zy8rEIEzOBNTOAk+lMewmcEq/x4RRq0xy", - "YbaOomx6M4Mpf5AOVuYzFCVdLYmv7vDO7nC07H957Ay5DKHFBDZrpoaarTWuuZhJlVq5jE5lDr0DD4Jl", - "gv20wRYznjC90oalNrgToCDguGFdIld73r1nd7mPoc1wcmzWYcZUyrXmUuixcCk3GVPwbXgd+q/EqQUd", - "AoYW/PXMMsnPIwYSBmPD/qhpWzVEvsKa1r3j3h7Nsr2YGtoSZ+eG9xFD+gGDGolepVNIoISoyEtNdhJ+", - "adUTstQkgT92N0ZFTvC9T11Z//onC1b6RMxksG6ppdmCmP9UyXGOrXnH5GfB1u5Iz/zoKMLnrHpwPS+c", - "yVYWq7fyWMVoMsC6/B6LieSGJ/x3y3ahE64Nj2wWGS328c1pweCHY3HKjOKR9fdGEH+OIBWo6e5lSkZ7", - "gON3GGUcAf0OGQ4OmW/74xS/+OTsNbZLWSrVqj8WVLiOXz0+s57mGY1Yc6CCGfAkk5O9F1ti1s9xmf6N", - "4yftBDcCUgQ3/Kt78uqwMa1nSLccUZltUttk9qcP8HXS5Fcbx5dp4wAaLmezU2C1eYC1sD1jKZM8hX/Y", - "P062QdUZGi3eYNPPRvK2w9n6GT/BL+JQujnFzNZ4vhMHjF2wLzV+FhbOTwGFmFpkYvAWeGz+jNT96V0J", - "1XX8DF2vbkV9/fTP5mzd9s3nxuBBU6rr8aUcc0tpfiZGbraEvaO83RL2fSKjS+3QdaomTKIYRch8+LGE", - "OHfuShQTMNmXOFQqi33GdH8sGsZQC+KkCSWGqZQLmuzhnG0nCNbuLWp0KTnm3EeYMzPQYOrOZJIgIjsi", - "GsJsoMXMd1DxLmtXYbLapuoYNRJgm2TKPID9bkh1+yvl5gep6mj0nwtffFVZf0R5pBxt/1sA+Nu/+FGA", - "/Kf0PYZtx7lzbvsR7TyX5Y/WLNUnuDfj3uFIj3t9Mu4dpOMe7MATiuZcasg9knKRG0C2fGptbZhVfX9E", - "NIukiLXH0ffWxMORbpm5I8uWhN37+N5tij2OqnApX7qPhNgDtCPwPiYQkZ3qgXNnMu7joYuJzI11Pbhz", - "5VrFzKB5ZPfW/caVM/JVt+/Cyf/qjm+NR+EuA7usbL3l7FmuF6zd5PazLZqVmymsQ1FkWy/I3+VU9yHr", - "wFrmATFvje/B22f2A7dROwI+dZW6EW7uX4tGdCgaUa5VGH/TBnvCleypw4JwsveZVAZ+d7gFjoZQk0Aw", - "ECyF+eLJyVhEwIosWqRiqUTu5CDu7S38+K/n5NmTl33yFAs+kx/z6e6QvIBMG+xVO3/RWFhJzDKviAKw", - "up1BHLqe7diRem4ycB0+cCUp/uDTeXrwZAS8PH6vfMB6v7dgNEaJ5I/ez9J+LAAk/fJnOECwf+7NYtt7", - "G4UPsOGr1eDxzDC13u2py9kSBQyKu6Q9qqAT3PDD+EHtwfTKb1rZwKKdHB70AuAnH77W8bj5QuG347HD", - "77rSjpDN96XWCMaDWDDHEAusXtdFJYy2jGXHyzYqGPjJk6efRKm4US/1Jt5VKxfw73q6cKZfrKMpq+0T", - "EHFRQWerp9cnKi8swrV9jUQ0oxE3qz6hSeLuKHcTFNExg0L8nSpGL8FuPwQUMvdln1wMvtq+d9SSmOtL", - "24PzxQ4JhMfpfFoMjuBBs15jXHNI0TWSRDSJ8oQaRthsxiLMC8aSPLrFl1sMpXeDZ6f8SBAWphJhn39x", - "ZQvDNIG7V5JFk+L27FbvKRYllKftePJOUMPgRww1mEKnUkDsQmIpjkZKak1cVwOW8DmHshoYKQGVNMDM", - "RVM2FllChWCK5NpG6MPQB5liWueK+Q7g7nIU1SclVmSmpHGhCYmUSo+Fp/A3p0Qblm0gs5e251Oc8w3J", - "trZz96U7MlI3xtBuCnFNCGyIpRS74EBHYBq6g7B4O6C7lhK/lIP/SvE5xL5mSlLLZG1ckj3Wfjntoa9l", - "T7eWMD0vWnUrYVr0WsmQrGQPbsT6m5Tw6XHvahGIgY9f8lY4SPfoahnNP8FLHb9dz5wND8I9+shZhqqx", - "/jsWPj2vJCx2NWCVFP6lmZMqI68d1VrS73aIr85ZvjeZddsZy+vOILy+ZOQuWkvlbVN4Pz9CGN0u4sRt", - "1837smmrhrxV001b4Ae2F0j4LCjwZioj3DHiyjUqI3xWGABAO3eIxRI8qHeV01/zPUv1tbjBjaby2woH", - "CA3XlspvuZ4LXt2oKL1xbbqpSa7HP5MEbxfoKvK7X/avWn8HlaGyWNtc0EDwLM3Myr3kfZVl0Jnmv7Nh", - "iyO4iFu9OVfwNUI6Px15eDptDei8Q/SyO3Tk3EnMqKsGyTWE+ZVV93xi9xeGd1g9c7WLZQ9unQFV0YIv", - "WbvRvX6C3RJlig0ymaFzJbYL5tbD32WGquH8d+K6d/iv7l9YThQLGbCYxFyxyEC0iDASOYL9xjeaKCmN", - "ey7Vqj1KxB6RH5RMH7vZbLkP3ZlyxrAyzjBdDWJq6GDpuc0GE9pHRHf6eEpgeIQL8vx7ssPeG2WLlpAZ", - "aD6Qbe6XlL2PGIs10uRudcD7o/AQoePJfNpllBvKz7zAP2hColwbmfq9P3lKdrCc3ZwJ2AsQ9WcoyWZK", - "LnnM4toYIXjfrup+y4Je1e4KQkVRi9ArF3ZwdyLDdLmQ5r/zrM4WipCYKRcUB7e1wEv9TFlAAfge5cIH", - "4Lg98qP4eoU5zW/HKztAiVha1S2ikdLCTe9+vea+5Guumgzl77TabefDczYbr7vlR3VMW7qJIhRF7tzt", - "mq3ffD4pPVx/kdk8znS+LBTSNrP550WCo9u7H27bXP7mC04BBWP5srFstgO1DBPMzxjTHbMlS2SWYol7", - "bNvr93KV9I57C2Oy4709jP1eSG2Ojx49OOx9ePvh/x8AJoyqeVL+AQA=", + "H4sIAAAAAAAC/+y9+3IbN7Y3+ioonv1VpD0kRV0s20qlvuPYjqMdK9a2bM+3J8yhwG6QxKgb6ABoyUzK", + "VeeveYCp84TzJKfWAtA3osmWbEn2xDVVE5mNOxYWFtblt/7oRTLNpGDC6N7RHz0dLVhK8c8nxtBo8U4m", + "ecpes99ypg38nCmZMWU4w0KpzIWZZNQs4F8x05HimeFS9I56p9QsyNWCKUYusRWiFzJPYjJlBOuxuNfv", + "sfc0zRLWO+rtpMLsxNTQXr9nlhn8pI3iYt770O8pRmMpkqXtZkbzxPSOZjTRrN/o9gSaJlQTqDLAOkV7", + "UykTRkXvA7b4W84Vi3tHv1Sn8WtRWE7/ziIDnT/JjTwzVMTT5alMeLRcnexLLvL32BuhuZEpNTwi2tYh", + "GVYiU6pZTKQgNDL8khEupjIXMXnz9JREUggWQWN6LORUM3XJYjJTMiVmwchCaoNljKLRBTF0mrDhWPT6", + "jf1gAr7Em1fprwtmFkwFBss1ca2QmVTELLgmXMDXiA2rG2ZUzlZXtt/jccImhqdM5mZ1oX6UVySRYo7T", + "8u2SNNeGLOglI78zJclvOU34bMnFvH2RpmwmFSM/LjOWUkGyhEZME24IF0b62dg1KmnsQRoiLj4XUrFJ", + "zLThgkL7k0wqo1dH/wr/oAmplMWhYXliFtR4KhfSkAvGsvpE6RW9qC/jL3t7/cej0ejXfo8bltpjRd/z", + "NE97R4cPHuw/6PdSLuy/d4vRc2HYnKneh+IXqhRdVqajZa4iNol4rNbNJEo4E4Y8PX72+oYT6O2Ohvi/", + "nUe9fm/38d5w9/AR/nv3sFed1srC10f+Yf3ROzPU5HqVB9nTNHGEMqkQyeqsf87TKVNEzkiUK8WESZYE", + "jxSLOxBdbdqj0FZEUsz4PFcsXu3bH7naci6oJlRYpjFo8IuysU7nLgImFssrMVEspVzAGq8M4rX/ROCE", + "EneIYEiRFEbJJAGmYAxLM6P9KeoDGxeEZlnCI2Q9tUN1kI50r98TeZLQ6coIy91mCZ9zLNBpabiubJKv", + "S4wkTBimihPeZWlqbLGt43K5g7tR8sVOXS5kEk9yYXiy2utb+NmtabUnrsmCJTGRs1mf8BmhBFqBny2N", + "15d9b7R3OBgdDEaHb3YfHo0eH40e/K3X782kSqnpHfViatgAdrnL3iDT1lxE4d0RTRJJ4UJSLLK7U1xY", + "tQ2cskimjEDTa0d+8PEjb71unpabCgWJK1jerIGtHjaujG7UnVBtCiaE+8XNckIDY3rDU6YNTTPgQzCG", + "ymK2cSHfYHMf/MqvXeDdj1pgwd6biVuh4HxC9MHeZyyCG1F6blIIGNCeI++CZd0JjStGtRRWTILL9Jde", + "LnSewdXN4kmWUAPt9vo9SwaTlGsNVYsfYq4tH+n3PJFPhDQTlQthCwpmrqS6qJZ0rUx41uv3FlRPLudZ", + "3uuvu7bqRI1dsIRmGttzO64mTCmpelY0Xk5mUvlN6v1aXcI1Ta2skC6u2MAK9fq92gIU7NzPxY+72NXg", + "4Po999dE2VcFNmcnszrwalurwy2Gtp6x21vECtF+m4mrrOscIOZ0LqQ2PNKd2DwKD7C9qYwDrPNZ0Rzh", + "MROGzzhTTq5mRMH9kDLiGyHQCOGC5LpxDgrRf8IumTB6cnkwMVG2uiiNh0118yqySXkjVm7lYvuLk7KB", + "SOtzDz6cLinHM/mMXfKIrcpubmsmseKXTAXYdyEAWFZoy5EtOOvAQoQUbLu2UuKSx5x2YQcxjmnCA9Rz", + "+vSY2M/k+BnZWrD39U72Hk4f9dqbFDQN0MKPeUrFAA4EDMu3j2Wrbb88CLXMZZrmk7mSebba8vGrk5O3", + "BD8SgRJutcVHeyFJNYv4hMaxYlqH5+8/Vsc2Go1GR3TvaDQajkKjvGQilqp1Se3n8JLujmK2pslOS+ra", + "X1nSn98dPzt+Qp5KlUmFb7aNB6e6PNV5Vcmmvish+v8+50m8SvVT+Jm1rBNTmmvDhCHf22JEMfuUI1cL", + "qRmJaLRgZGofLfg8x/a6kHzRsb+9QiN4xnUmkc+TdyelNIdPQ/aeRbmp9fstibk2XEROqKpMrsOIQP2U", + "MLh/Q5IFrgBxZbgUxHjR6cZCQKQY3dAdlOjU2er5zy15TVLd1rovQrggKU8SrlkkRayrfXBhDg/aJ1M5", + "xfbaXOnqOfxMUqY1nTOyhWopfG5YDk+4JjPKExZvd5Ow2ybzdzmt3Gu1M4dkMKDTaHdvP8jQUjpnk5jP", + "nV6xSYPwO8jH0I4hPG2dCEoY3eaBXSo2W+3vB7xPsBPFZkwxEX10d5mSl0xQ96T6D+y393/tlArXHadt", + "3cHFPC2Lf+j3fstZziaZ1NyOcIVNuC9ARrjUBGuEx4yf4u1OFKUNVevPB5b4BCexFDY3ro1T/YC8Recb", + "q7yBMk2GjuyoEHAqXKCVbz8HSSsgskhhmAiszks5JwkX9hnEhHF7gcLeMmPfJXK+3ftk61As/+rhh3Hf", + "gHnZH1pag2/94lWQyHl1NReMKjNltcVsuVddQ+XoWpf/tHZ8Ghco1WyynoOcciFYjDp3d7BtSZCtg28f", + "PEUX3Ewu4QYOnTkc1k/cEFeitalERhcznrDJguqFHTGNY24Vrqe1mQREyJoxg6KSwDeIog0+qs9+fLL3", + "4JC4DgJr6LS/UGB1JpXa0LwtSwxVU5okQdpoJ7fr39GrFBKmgFLh23b3FBToCdNyup7bTfd4z/XC/oW8", + "G0aFd1+v34uAvBL4e4Vw+733A2h1cEkVrjo0XxnWf/ueKr99X3Za+fW06L/y42s3lMpPP/hRVX57Wg7Q", + "rwpTq+fBPdNw0mGpUnJhBlzgFoAA7jRPU7t5xSvLaTKIFBUJj6ku7HydYPV0RaQiW69/eLq/v/94u7tw", + "xfXFRPPf2WQ+XSs3u0E7QRmqEahGuCBzPqfTpWF6SJ5SIaQBY2S0oGLOYkJnBmq5sdYe4A9Gm4wwrUIS", + "U20ikvn9gD5+9P49NY8P+ZV+/Hs6VfO/79Nem3Yx1y3rixo46nYTihJFRWATb3ZRp/Q9vPwtlYSMnyfW", + "XEWKQm4PdHMIQ1I+6Gf8Pay5Ibu1lQ4ubvj1V5iwhBSDXPDfctztLKHL1TegYTQdIEWEltfyDXuA9Obj", + "o+H82Dp+pleUGzg5hc3UTblPZBLDvTPjSpted2vYtQQDpj6ViFQ7ZWGJaZUimgvYeqkX42w9KwmfsWgZ", + "JWyVuXv2nancqVtjBncO/lkoELuycaYCnNj/eFp0Ufv5Wdlf7ffntvMP/d5TmtEpT7jnzHU+7WzyE6f4", + "27RL1cae2aqvXc0P/d6MUZMrFljLM/QPIFZPPEjYJUuIKw3Ee0SenB4TnasZmsyRWm1ZUNtLzfRYKDan", + "Kk6YRlK/WvBoUagrQfWgCRWlWwB5klzRpSaZYpoJczQW40JDrse9Phnbx5f721KJ/du6hvhCYq6Ydv8A", + "XcO414e2EjnX496Q/Ci1GcBz2clSR2Ts9DF63CNb9k+SUa3NQsl8vhiLlAo6ZykTpk/QSQPVwMCZkuU2", + "oSIm456SmhlDByzNE+T90NiTLEvYWJzxhEdSkJRGr85c3StuFuS1rVS5PHHGcFn3SeZUz5qmbCyoVYPC", + "Ti5JpuSUkYTmIlowTZiYSRWx7SE5ZWrgl9jvLaGKwWZYJS7JBRwRRsvN0L/8Spgwynk6dGcuMJPr0B8s", + "ffGI1tepeWxrfCisI9ep/LOrAqzKzTj4RlHLUpteoWd7Ixaqfn8d1SwB/bHAFcVd5VDqSpDals0SOkdS", + "qRwiS4x6LCKZpvjQc70QzS6ZokmxQ3DHSdQnwN3gbqSxiBKZx4PFMmPqkmup+nBDsAidEVSf/MbSvI+d", + "wl+DlEdKXqZEirFIgIx3aBofHmwPyROScA3E4efPLXE7EkNOgPoADqMd96jXyY97dmJcE7j4h+S5MIpb", + "miPaEtzU3qQN4uq6eRVmtXK34f5cp7EzW2PFwmF/dgTdX2GxFbopCbAg4woXDd1Zaxjwqgzu13W9McqN", + "r9itiAq3UzXqPCLcEJpljCpNuBiLkppEjFTqKmWK4XJobtzWpcyQrZQaZC+gtR0L5BDfaFLb/e0hQbnR", + "+oeVHAmqEMWSpR0QGws/5iueJKg4JDl6MCCHswcNNWMVDwo8TFQUvC8prV3O+OXsorUJ1I6I7fu/n5+8", + "JXqpDUvJlAuqltuE65LXWtJcNcyFRUa3e/hyR1GtPutYotcT1CVS1AXI5nndaEJwImhJF5so7EcZcrGk", + "KlqE3Oi0IU9P3xL4zA2LgIhr46UqPQyakqRuaQ56pSjE2vWuNYdMZ+OcJRwpHPGmyR4X90hYRCrM8avW", + "LqhZseOyhEXAq6w/jX2/MeKMy0SmyNIFW53NTusS+cb1pt412ZJ6Bya8XblZ4EyrXAzJK0FQjCA1KWIs", + "sCgXUZLHTJMKPyel+tbLFzU3qILsx+Jf/+//Z2WJmFxyWkgb5B1XBnxIfrev2JmiKQOmV7/SogWLLsai", + "KYYQaFVLQv2lUiwz19ULRfH5whAhr4ZjcWzHVAx4S8uZuaKK5Rk8M8lg4EY9cHLWtnXiwoa4abho/tLY", + "m8riXNOXsGbOKza0v0phm0j151JsqdPqnBp2RQMP4hdAeoNLrtFlDWnClSXHp0OCn9GUEC3sV+CiIMWS", + "LUsVXMzHAr84kXibOInWkpkzTQ7Jq5SbgviFLK4Xd9ONBTgYThkTaEVMLq0DckmqS9bcAPDm3AUD73A3", + "dDTmMPaJkZN5HtS7+rsu5opFhrw7GRg5eHdCjKKzGY+AjjKm3KjtBdMcdJCfg49B0rLSvqJ3oUiG8MhQ", + "PJ6DkAPL5UR/+5u9nDKmYFxvnpzCBYer/y0Z9wQ1tk7J6QdgyeExi8nPT97A1Rq9Ott2q+aeprbhHlw7", + "ISV4v6fzqWCmbfz2K/rghrcCXWk3MV+7RCtbtIm8P06igYvfNvDNGpmk5GAgnYAQW5DgkLysya+2QuG4", + "AiLoqkRftOAP8bekGO13KNCQlFGhCXWyfmVEhOux8NJHTeAASZu4FYWqKheWS9n7cFDhviiWWHkEJbLC", + "Sf8S/n9wqWV04Vw87Cuz4LSCpni6q0u3Illx00nI2agCqDxYAkvqe68JnvBG0IJmeiENvqr9P3YU00Yq", + "tg2Pc+cvBd8zmmtG/kJSlkq1HAtfvm/PWUKN9WEo6s6AMfXIVpRIwap6BOtAQMdCG5llLHaGiW/JzB1u", + "WjgRSlXoqm2hsXBd6B0rAWhSzAI3oLTAJMtykysz2UZVA86mmBbMOU/tsBfSZEk+H7h59shWwi+ZmzZM", + "kP/ObBtgRZFSDJw/JBQtdSf4iRiq5sw41bPG9pFmxr2x2MJzS/DfYFhJc8EjvM2x3DzLB1UVR49svTh9", + "u1P6LI1F5TNWAaXegMtBwlMO7G0LfiDHO6+Ighsaf+ZibkePhe10iklieTfFIXlHk9ydU6R2WN+x8NI/", + "jnRgVG4Wszw5Ipe/OyGsvh/FThZS0e5+32maEtQwtUoy706IppfMEyTWgie1NnCMaHzJlOHaaWMvyJYj", + "p4GzcyHdaSIYi4mQY1FtCw4ykikMsSnyE6uI0aSxSGNRim0NT3J/tXmTYaGK8IJPd43N+hdNi43h418t", + "13ginxWP+sZFkvF2qyroIvGg0Khcpsr7lZFXGRNQKpZRnjJhCBcz6YvWvchGw702z7SW7s+q10rRvV5I", + "ZcicG6IYiHFSOG427g1irgwcf53PZvw9viJz1AJZmQZb0n3gUONeLi6EvBLjXn2YDw/p3iye7m/cDT/s", + "fm0Ng/uATM9ppVvj9dbb0J6uNZiBqEmMhLdz9eXvKPwTmMvKkCS49NQAzkrCWVyh7b711p0zwYBzxdYH", + "1BAvobWYfAZBUfbWrUrXssS07Kj1nG3d0A1zmCdyiheen0jpdDokx/W16xOKH1Bgrq9yES1RcQwlW2w4", + "H8LtkkV8AJ6hA7o3GI0Goya1JwcD6/KeUWOYggH+P7/Qwe9PBn8bDR7/Wv45GQ5+/ct/BB/kHb1VvV3b", + "zXPLn6U+8YOturA2B7rJvfXGlrXq8NsPL6oWWnc6UgyPAU02jgE8Dp5WirfT+tPjVa83u1KxBD30kMud", + "hE8VVcsdMefi/RGIc9rU12192XUKlkBwjBWMfAFCNXEall92LqniVJhf7RjJuKocAPmN4K0B9/4zHBEZ", + "DHw7jn15z/uaAWBIfpaGUGeRhifdERZa7XeuaJpS1HZe0oTHeDicmU3zmFnlrMBv5NJKSYqZXAlyMBr5", + "L5PKoEBukNpPZdxzilMVLYhTCCmCqhAbIROxDLo0MqaNMKXKUqyeNDhmf9kp/tryfw5//cv2//6PT0rq", + "SGhraNyqM67Jzxp+3lCIbCXyiqmIakYSBrPVfRLzOTfaCm8xukahQu1bEllnD+u6KBVhwquosVydnNPl", + "gGZ84DQv1tz9kom5WfSODveDi7vl/hj8+p/+p5aFVXkSeq+9ljlqXvFzNfDaj6GT8cWvbp6g0SXl4thW", + "210VJz9uh/1E1u20fdS1bjXcMUVU0IaBrEbgg8dRGhIiLplScBQtX3t68gyeMBfMMTR4NZJxPhrtR1gA", + "/2TuFxDi4CmDv23XJR7bXEjY+aXHooVEN8gkkddRUvabTL3NS3CtjSy02iX3X/UpRCO6NYujs7srSKZK", + "XjCFJhlYYs40uWBLqy6Y1xSaTFySS6q8afANhiVgEf8JrUEMn8fwiMXgfIhlZcqxRfdms2FQjCYlt0wp", + "mLvwXakjmTFgqK4YTo2cM3F5TlKa2We9YlZpnlLDFIfnYqlYTFnMgUePBcODQTIKZz6KpAIVK+wt6mDL", + "VfhGk3P7TDzH5s+5ALI+twezoSr9o/fq7ZvvX739+dnk1enzn58cT356/j/ws63UO/rlj54F0ygcr75n", + "VDFF/uMPnO8H6zyLPn0Q6r6Qyr12vYke/VVoxocyY4LyYSTTXr/3n9V//vrhV+/uaSP3LntHwYF9CIqZ", + "TvcZ8MH3AVCaSFV6nlLEBUEW5dUOFXVDQ5VvJb9rHQl8onA5mWY6GJvS1FuUcujuaHTy/Y7zHnng/7E9", + "JM50i8MHHiSVE4/1AsinwOVAe1qSyMhFaMzaQvB9VyEGj7bWTHJhNjKnsugqjxoMyq/XYEU7Uy52NGzD", + "ILreuiPd3NhR+bm45EoKfJ6juDRNmK6flZ9fPXs+ef7zu94RXARxjnGEvX7v9NXrN72j3v5oNOqFCJS9", + "z7hiOuj4+GSqZZIbRrBQ6WM6JCe5RudOxzxmOepBUT7TJM1BsZQsCXsfJbkGPoXSgDFJKP740WBv983u", + "o6PR6Gg0+ltnh1Ug/Q3M+8XpWxuhjued0cQsJmia21TxRyz7FIt+6HvdJD7tA6tU7KnXVsLpdXXI1qIu", + "XdnJESTIcW//xff2QO2+GPcqhOhCgYtWbMON1XvxfWhdKtqoVUmv+GYj3ETsqX4VjMbq63Pt1ERVS/hA", + "UITYgZIxVTGZSqrib+u+NFgTFPnfaILaCpo4O5EVvbGOlSUl0/hW9j4+pbJV9xtLgJqf1KoSqSAMLaWX", + "HFjOIE259PXqbMkpU2o61GHFwBTQ41XchXr9HszM/cdPsNfvXf5ej6nerA50rjGd3oobHoE0ybhga16B", + "n4mwDzrlRNJ4sPuJZf2Kv1sDhsZ+qG938Tj1BL6C7jSlIr7isVlMICIehhy4Id0XUhQursn3MBOa/Osf", + "/3x3UipEdl9MM3dn7u49+Mg7s3FLQtPBWJtiInkWnsbbLDyJdyf/+sc//UzudxJWpLyReO72/7ltoQlQ", + "43C7rDd+i921kMPK+FzZYkvfgGEQunLlJVMJXVauEzem3u4IeXpjVIpbc6OrV1Ehr79coDUvrlWvF2Dz", + "M84S66NlXW/xfaG9hcjaDvG0AE/6RvsIJEamUho7AlgbRlewUcK30seppIbu5oikMJQLptx4ixvDXin2", + "xZIeHgzJ8/cZUxwEJpqQmKorLqzTi6/p3tyECgAESrhgA546C65dEjtJ5MnQMBjavMORtg5Hq1p7VFbH", + "xN2PtrJUDYXYbWqWFEPGPMkKJL11x+e1LV0++gMkGaDI76kuLLNd6LAgw929E/fnniXIKvEVh8ueNa4K", + "4rNdfQtiKDydDXaqLazdlBHoiVFtyAF5wb8nXEAof8LICf+eUHc7cUHe/c3uA/I63UmW0hc8s34eEzov", + "wlLXIf+dXfDMveexhvcysHdQnEPLeIaGY2GRfYBtIO2AR7633RoIItDWcQFuW7zPVmU00A6UN5otbs20", + "Khd9Ms0NPPmlYcQpXxzEAIwFC08ZyUsn0sYD3E1w1TECl+WCKcGSiX1g644rYysRV6l1cXCqM6odEpky", + "eVZfr2c/nZyRrWdLQVMekZ9sqycyzsEb0PKE7frq9cciUwjvAp3AWeCuXzkjMjcDORsYxZgfYoqNFXYY", + "B3Jx+eL0rYNJ0eCk9JrBwjIRO4dXL+wUHq/iG2NN4bVmq/03Fr0NCseb9zue7DNXvDza3TWS/Z4xSQjc", + "LrFsDYKIbBwFGK4K6vOhfX04gS9kicVgucGQvNXADUbg9iIVei2QqTSL6usSryVdfQGVEJplMXh/hp+Y", + "5Wu2cb4PFkG7dZTlddLd67diGl5ahwkQZ3RXznUJhSuMCwGXApxrryOjCoLNuCCf1R2z8LJVvZIdYLln", + "1NQBObqqwm3LCAa70TvUabRxSbprtCs61hXdtlcA/tExaGJ9+8fCD2StRr/UAX5EX2e2keYSubb7fmY3", + "WKXjYk0adoBPszxPdEVn2gm31co8Vp7RZOsc1KyOjkGxet4n5/9Z+wH4qZen4LlwRexqII8W8FO1fb3i", + "2bxBj3tj72aqb74fT3Qr6ha53CVGUaGtn/2CZmxIfsSLkRiWgrQINxWvuhYKefUtkfaN4quOBQxN130t", + "vTZf87kA1zN0mgeGE8dW5e80dt7/Rofgjr0ebkVit6NzoEXIy5y3PTn3qvfzOv9dVcyv6q2cpn5FYYG/", + "W0UFqnLMTpob6N6+XpxNWubGgpi5qTcEvLr6f5NjkBtLoYq8wf6fFeyiiSF9GdBYwOTkrLTXVAxHbfYZ", + "J/yFbUcXbIlb7u1EdMVSVDURhQ05zqXdiTKVqqg7tFcJ1xX7krUUOeMQOh/Xj2jQbLJpK2C9Oi9//eUf", + "jMMZ+MmWFOMe8x6f1HMhmJztD5WOmuHioybhyIZ1nfet6oOhfpEIIJbE++Y3m0e3ZHRiPHe/DF1r50WQ", + "4KfAHC+jP6pbSyo760VpbMaLYobF/bpsAOjUevOkrHvqosXsqdiV4p6ReYDLjiKvi5pJ3cPr4/RAzyuN", + "rXEI69rEapSsXd/KmB09WWBnH0Ri9wPtXzXE4ybwf9x4hlvkmHqX5zRJzsmWK7RNFPs7osK6vQIPu2L7", + "AeXWkUDhbvbupD8WlgucL4zJJvB/egKn+LzZmKvrT3gJyv5ohKLrwcF+PWbDDrjRbF1dHkTDad8a8Lu6", + "Jf+tFqW8R7/Hz8UNBIg3ZMvQORwdB3sE98T2xyrsudAsyhXbfNafoHjkoHdwTAnlYvDjmzenRLE51xhq", + "HAxqsK7xLRhtTllQuqv56RpZ19ru7R88OHz46PEwvlBDFqlhrgfwkBnsDmlKf5eCXmkQu3bSJc2yo8vd", + "jXzebkExvnaW71+27aQgU2DJ3iG5yyv5aVmlNBhecBF3beAnKLvZ6dU/4m/b7JIpNsizuaIx+8RGlxv7", + "NeFqtu/qhkwo6x2ac21AEVFCAG+t92OuL9alTAYxNfQ6bsx2uKsey+nSNmWf5KH2Wt3Dz5o+4TXcnmA+", + "hI91MvNjCW1LG6awVSSweGJkIHzXyw7Hz2DNfdlO4KkIgWzk5HLG5aYgfxhaPXQXq3v1BjQxyCLuNHXe", + "qsELrb9V1r07qfrWDMdigFLYEXlWdFA0WzRJ8YlBY2vl35KqFj+MhoApQJ+QdydD8qYY7TfaGwXsmEgR", + "OZqjQRGFooEVyaoDyDHQhptmdaeWtHjM2+hCJN23YZG1BRWghQIN9WVT3piPjevGjXJ3SRV6pqNCeB06", + "2Wt7H61DKKt5hDwYjHYHuw/e7I6u6xFyC5DTobae1HmL8/2vcp+nb4+f7Tm15/aNQck+OSh1mJU9K4MW", + "yFauWSUgV6pgqEIlIqAlFOHGEQa3FDBQwm+uK2tX4g2UvA207RAeGBbp3wAPu8kwN4KuVia3qtBZZqh+", + "KE9JRZC3O5pFPBh5DS6D3ytGLzCLwOq9jTgUE3ufhf0NESxkukRwLFRyKSnNTJMCQ6ZUfu8ePDx4tH94", + "8Gg0CuA5rxK8jPgELbKdBgAuOAldFtCGW+jGEJNpIqd1Qn+wf/jo4ejx7l7XcVjDfbd1KN7fvhbZcivy", + "F5/pzX+pDWpv7+Hh/v7+6PBw76DTqGxj3QblytZFkof7Dw92H+0ddFqF0LvuucfXbgrwoWQPT2x+JPjX", + "QGcs4jMeEcSlI1ABkQMWYMb3Vuj6mZzS2KfECN8dhvJEr/Xus525klbfmuaJ4VnC7DfckE4mCZz5M2wp", + "mGNMiCIFyfVacqjkG/2+/FyKIijOxGyaz+cO8q9YuhMXzV8KT5wl8REpnmxr+RzuZjmwX9vowM2hIzW8", + "lFcF+F6FCOzVZZEqFCMFndhNq83KhwtxkeVBkmhdyh9yZUE4sFFCpzI31oLvcsaUnVg8G4EW7VzE3dQd", + "P0h1sTG+A27iIjVOR4WBD/SfIRBAI8S/KvQVlnbrj+C+a/La1rCKwvLnLK/nBexjT06hKDw6QVzojV0z", + "XaXLsNyCOnPv1Gf7K3nnHXk0DmbWCezTvrDVnGFKKLNRYgFKsf5TZ1i8c7gYVNyoSOmw7oJd3cWiY8Dj", + "AMh2ACqU21nxdY7Tha6hLESkCzcAfE6YeVyBl2qctDOPuOGWfDgWLt6q+Mk5BUBFuw7WkUAqPuf1jut6", + "1jt0Xb4OZXriujF1Viu2+QzqwgwY4gEWcHpmPYGt92IV79btSa/fOytyZznGVF8a/2NgRcoAg1XoodO3", + "13VAzpSc8RAeEHodua/uoeZdc18ejM4Gu/9tvTuB/FBi48J6KoEdYdjIf4Tlu11EL07fnraNqUg+Raqj", + "W5nTGpSj0rOmjIopFsfFmnPtW64k6qWCGHrBQMOEJ65WzMJ3sXhInlpfG5Rr+Aw12sYXs/AqwIBs35fs", + "qAaVAz5hghWNWlNqnANVVXHeoBiIAtpbPREj7sXp2zpew+OQ+I2AK9McBjZJA/rAH+A7sQUKwIiT7+si", + "+N5BqOkw4z6tERC+9Gc0Au+AzhQS0CE2prEJkfHF6dvXLuFSG1Y1kFORlMnCVA/Jz0VKMlheXeJRDQPK", + "xQYJWtdzKDDRiQw5sPygGCP4rUyFCnSDRm+Lblw4GtAY0a2k8AkZwfffYQ3+llNFheGCxeTdD/pbMnJW", + "9nc/EBt/5HgW11W/y/rj8mFoS1vjCE8XSw26PbssNikKF1XFJnKBzi+T07KiUwEH3ifhdHye45AtSMKI", + "C3j2enD86t1OGrPLfm1M8NF66b44fbtdYcuXHsyiKFvnxpdtGia/VUV2yEmZkbJ5EBBH26k+m9QBezMi", + "UxZR79Td3LyoSOA8RXcNVJSBmdswQRR1emoqao0g6wLH+zxJho5n+iET1EzPkHVXsF/t7eaYmmIZxcdX", + "IXm9N9gpIzqPIsbihsdLc9QVejsiuUip0guaNCd3VAAsRAuqaGSYIt+8/6YXzsCErLTr3VChzuJy6kyW", + "lasoQI+Vc9d2xN/9UD+eVjqwYdVOp84FNwgTh9jgW8UgLZr5kJzQpVO9ZHjNOnopd4zOZhYstbhHFEsY", + "1Zg5XZN3P2z25TTS0KRtDm/go+NTWzAhOEUwzD7JamwAfq+ilFW7PQxeGeV8ApYp7LBp5hiSYxFme9Zh", + "CHEGLWSY1nwu4Dmonc8Q3Kd+JVf2rn53BjVJTVzGXn3hatPpBy6AEMWEbisbB7r6MlrN5Cov6mxKXmzO", + "Hmobae/3qQ9VbZjC3PPTn1uMaPXOS8RWdRzDKlQ1y6iihiVL+x4pqLOeDYKt+qFAtMA1YmWfQ/EPNgNQ", + "rtjELBTT4DNWU1Ds91ezSGuMSrhkLnGenVPFXGckSam6wFPmn98kF3YF6nGx+5vAsRbGZNeYFDhVWJ2c", + "YeqSJs2ILb3il/OMQZTWlJkrxoSfCtWEVr3imxHvuiV5GcTyMMVlfQ17+4F+z2ykBJkrGjFia/kc9AVb", + "g6PXdSl7+y3DggtH65b93V23v67qLE+67XFoWLubdthE19ngN09PfRKuIsm5X+a91VWGRBL2yLkaG7Z2", + "T69PB+e7EhZFe0VmnzLMD+dizapR7N4rEfPfCYzYq0SOV5iDrsY6u37wFNil6ttz/munN2HzuIfcbyDS", + "KJQsHj+4KL05gg1ibILK8eXFYysp2Tu5cj+7SCqQtLgAXOa633eUq6TX7w1mblZHOzvA7xPEWz3Y3320", + "s975e63Xv3NynMR8nVbIlXLOch4CwIJV4qTrJLFDs6yD3hxrbrofkD2tbIJNEA93W+WR5UNcwF7TcF58", + "TyPjU1qiIr3mKEGrxxZR0euCDDSY2ixV8Np9XD2fo6DnCrih1sl/Z4X2YWbWExloxCopG+toifz3oHQq", + "VShvoVTGhStPmX+0F/ehdwT2+N81k/roUXWWhw8e7D/YxIeQ2TSOuTt3gana0nVABlZZbnd+XQPwH12X", + "OdyX9WSF67KBpt48DZDU+gWVGRPXWs8HB/t711vPrhM59s6cDb4UwrMCACuUicrI4pQZGlND60wGNdDA", + "ZTAxFmUpBg7Ovl3PWlq8nqoAVTfN6vepfGZaEny+9nj2KRV8ho8kW7Las17QvQeHRzYNccxmBw8Oh8Ph", + "dWF7nhffum3FjnWtrSD4DPXi4/bhFtB5uszlj97pkzc/AiPLtbKX1o6ecnFU+Xfxz/ID/mH/OeUijOrT", + "JXM1n61krK5tb5Y72HQWH8FMhAvG8pJSB6++FhMSxhQglFgQI7PmW317YJgFjfN6zpNrgGKuCeUHcQU0", + "8q1rc+Mk1KLQhZpK8umqXqFDImpw/Vzn+eE1zljG9Wk1HUWO7lV/mBtlWddrE9Gu5CnMmChSzyaJ/SuS", + "AlHJQ3loa1ek/9YhjSFeIy5fYdFl9cei98qPT6sDqfzu0yBWfnIJaX+9piPbWoH0r6ty6GYu5MXRDYc5", + "rP4vboWuyb8diGYwguOe78KbOHvWe381/6/f/o8+ffj33d9evnv3P5cv/uvZz/x/3iWnrz4KoGk9fuq9", + "gqB+MtxT9HCsgZ92JaUTsMys0hM8/1pW2H2xGgcTLTBdMZmyI3DBfskNUzb5ZSNkGfI1MHwpYS0iBWYp", + "dGgW21D51PohQOU/vBj8odlG7GArlNuQAihJ59NYppSL7bEYC9cW8RPR+C6Av2IS0cxmsOQCFLPg2K0w", + "3agzDJed98kfNMs+bENKFGoIe29TAWQUMzbOCtuE8G4dblTWed0VZwXeso0jHoviti4ARK13wLDMNsJZ", + "0oz0a1mU9e8393J6NApBrWIUGmwkZtTCJ0hB2UBGRXgceTTaXn3PbXhjFDS0hvyc+4wNU4Yw4wAptoQ2", + "P2Mxj5Cv+OheW7ISWG0pzSnxMiXfL3FvXtuQ05jQ3CyYMM4xsk8iKS846+OW9gkYZ9FfC2taL5yFzAbT", + "5WAhswKGpbCFWz+W+iP7/wzcRAfvmOIz11MQTQNIJCB04pFxM8MypXZhZWJvbDIiodEo74rbrFbaJjKy", + "ARgmV8Ln2mGYcRfBhgrqYyGZ/FsSJRy1TnqBNrwFooUaaCYE+NkbFYFudBrFbNb8d90Rae/BIb5g/b/3", + "9zrHmdulW0dleRJ406ae9XXg2JZN4gCseDDxivANroNwA1pOTFBPYST+94z4hsoTVzASa+mzka/apW5J", + "dCXmdTsYnOiOgQMVmtDcdJ3d6imsxTp2aMH6FGG1pAOc0XOcB3nz8owYplKPerEVwe7gKbHYNQOude6y", + "CD55evJ8e9gLosTVTFq4VWtjIeuDDiCk2HPQ6jpV6mhoyvrk+BnGuLtrpXyLYVDSD1KRxN6K5WV0hHA+", + "dXUPtflHj585ATRZlo5KVmwZ97Z9i1nzejsir323hBZDKaKVS9ryTZaXCTbr3FZtxNRK643gdrSPueef", + "u48xPgqTs1pejEi3rfdXd52jW3H42NSQXftCqlRs139V9v5TY8J/ehl9/3oyurNCT7IF1SHqXlStmlho", + "Zd+rhuw6u1ftnn61fieaB8/WX32Kqso1ZBjcc7XqIRDkB4Pd3Te7B9dX310XzruO1FfB+CwQvbtDcd8G", + "pHUAoJqbSWtICIHPLgDE60XenUB0JSCvwceGdmR3/2EXpQT22jWYovIRRGocUsGlPPJeEQRgARAveJJY", + "AUbzuaAJeUy2zo5f/HT88uU2GZBXr06aW7GuRnB/boDsjbcArqN1jgqAr9WAPEgR8fvmzUv4EiUMg6as", + "HH5xc7zvjarFDvjffnAvTt9CjQXVE+9u3R5hTMsoffaea6NXcRc7RS18DN64rVoqxrpM0rbhnCo3gpb/", + "WEPVDqJ4bt8C2rgPOVlZznsA4r7P0N7PDwR8LWz3x2JvOz3DLUFvt15pIdjqBo7Jg7bb7eYg2rcynBoM", + "VIhtVSUcj7twY9Tqfo8HYs6fOD8+cnxapqkrjRG++cacHu8Ndw8fYabl3VEXxp7SaE3fJ0+edu98tGdv", + "mSM6PYriI3yw39Rm5QjbPkFocgWRuWP/SBz37IVZed1Wjq0t0y3mYhUc/GZY4E0xrgWWG8VZ77ikJ+na", + "REkdgoqbUIdp7kBjU54kXLNIiljXZeQF1URnFivZJgwqJPixwAH2SZG3HaUUQqNI5aXq0UnXTt7PM0f3", + "mLQ4k0Izl7XkJ7bUJOVoBC26R9dHTYrYtXgstpSPcywCGjFfcQw/YNRQn7inUB+9hjExElQYC73IDTCx", + "bQhAEZDeWDmtLJlytBhtE53bJy2OF1djCQxT85ipsYBiATTmP4rnydEhICP0e8VLDpK7jEYharpV4+fQ", + "wa5jCxgCQ4VLp4fe7ioXJBcxU+S11MwYSpglh6aL3DUNpx+Jce6rdxOvXPVSrgofzE2Q6J2EqY/GeMah", + "trzP0Tv0Bo/zBzcX0TtFCnr51cUIulqT63gwuJAMePFN4bazCjkWOzWkZi7Wo4i9eWsTB9en7lyPjQSj", + "ulqSdycnNbcHF6HRbeLIJVr2QWbX2oa9DTqSjaO5jnm5goi9GQX745Gpm5JKRUL85DjUVQujhy6wFFpT", + "bNUezuv0a/CqDMZAcWH3CYhmzQQbuaMt0EvpGFltVS/1zKGyOlHax1Q4RPoCtKU06uul3pnmeieL+I79", + "rncQUecRIuocBCEPYnY5yfPQ0wg+uX7I27cAe4O/ICK0DZCpdU/p4e6j0aPHg0fT3cPBQTzaHdDd/cPB", + "3gM6mu1HD/d39/bXBBV1iIG9eVhr8MUc8GMuvNYn3ns+5NTcFrvQkE2cP/YVF7G8ql1/QQfZau/O+XZT", + "96uu9Z2HEAwJSqg2Vn3RwslO8JJnEbSN5XycdZEnLqzoPHwz2v1Y7Q8OruWOeKNyYc2qWKA0IaSVAVc3", + "qz7Om7F8HJAPfNm0WtXOuy/a6OjB46MHH7toPnhj0xib5HSHm9vmEeaR0xvRIT5IuKJH8grKnpOJrFbf", + "Fl/2+r1cVP9GYaDhS1187hTE1XZg+2E2su5aaYE8OK69V9BTxSJnxkcgqfj3COZ8KYA1QAR6CrowUtHF", + "WSBBtMMdV94u0AyaxZyKzgYu22AMjIzk2sXzcQGMGO2P0IjDRzgiL7AsfqKpfda5QdjE3VXTG42X1l8G", + "zpfv2j6y1g/5zL2vsA48tgj8C6cNy+BUtuubsNLZEflZYp3itSdkU/dri+Mza7V4oyzZciD7Hu8GO3Oi", + "5hH5oRAvCwHVCaRbmrk/J45hlTBT2zWwD7fj4EhY2bkKcEW/Z1e01+/5hYI/A1AXb0uqXzl/VVIMOZIx", + "muBZLsP0c8MTB66PM+Ha8Ei74BHY3Daxx8VmsnhiH09tPqk29NWWIUUlL1W9OyFbiJ/6F+IU2/Cv7cJ/", + "tXbX7T0+eHz4cO/xYSeUtHKAm0Xjp4hMsDq4jXJylOUTpxtpmzok0cM1j6xWofB9AWtXSeaZksB6YOa+", + "wWrnj4ePq+BwscyBDxdDckiSFosaNyyIf1jwohY/yN94cslnM/Hb79HF3t8VT3ffH+q96W4L6LXtKKx2", + "O646F6zoqNl0YFN7hvG7kKCUboW4e800zoCcMUOQfgaERvi8KeKpHcnlurbiQcI62N/ff/TwwV4nunKj", + "qxycCSrhApeyG0FZkmBJsvX67IzsVAjOtumRYLIAzEHjnBGXoH1UF0iHu6P9EJW0XNwl1bi2L9PWJX/n", + "no9uUm7RMXi7eFqunPLgau/vjx4ePHj0oNsxdurhiXq/nsO4crY/nz6juvNbKE2+eXJKoHU1oxHksn9i", + "9cGj1oyKuWbk5ydvLLSTkCRjalDeiL6lepir8zW71vzMteaHSWRs8ocNU6x5qjw8hOit3W6ojyH3FYdn", + "Wjv6dS4YOL4B8grsa2ApVpl4v+3eCQlmllRfsyihPH0S+WCNxj1mkzxMlC1WbkKXK8a95VeuwA51O6nJ", + "CuWXDfXBOkQqkositdBws3H3k9ho2/m/vWg23w+hAB8Bq+fgyWy6xhssZabYJZe5/gQNSWOjb2eJlOpa", + "dduePq+ZBgueB8V6d/KNJo7WiDYsq7/GHDWuAXG74eSudZ5rJBIm8rbF6rQbXbZ+3YT7Lae2vw6ao8YN", + "WqETY+BcudjsRvqUJlGOybRosZ8wK7woEFwgy5KljRJIEikFiRZUoLlFudR9Yk4oARSGYdCnFr5MZkH/", + "DHlFEmlB3yHZj8szZQcB1UAYAs3SVjWboyWlRvrnB6llMi6TUJ0aH6ThTLVhdKYiqB7WkxpZwSO3VWo6", + "1UTONT4vDcY/DJtpMDKqbFgDFTZv2mVqX6UB1+3AEBvMPHSj4kc4mPap7IQXDFm3K0kjJXUFWOzdiR52", + "jZ4r4qE3u4bWB9uBdK1pNHCV4RWnO6dXDN2Pgcigj7khkYYx+nCN06VXc6ZU5Jh5qkLITqU/7OxYCXrt", + "SQGYdc3BajPBdDKgCixsYEX8fqFZ8mWC96JnbTdZLufBfKPaK1QVbqptgO08Nbii4dXqFzQYIuNVhLq1", + "oHglVGATFus6YJVl/hGusVVewSAkWxg9U2FLFSzJ7S7uNuHHL/Sz8u51mYjB4tEVo3E9JOMpNYtjMZOr", + "nOA6Nliv03Z+rxlTHiYxZoKz2L9KC2OsU5px7fLQxTlzK4fd1oD00FmC4svJadsgWKjO65sddlE02zGs", + "zzaD/bqCHXaS63Bo7huV41ohAQHxlEG6ndxJuZ6EDWOrDSs2zxOqEDqw45D1Mk24uOjSul6mU5nwiECF", + "poV9JiGEbAKf9Hc4l+1Os4MKk7bUZGd2cC7C0G5Io99yCt/BLLcb8c2o09mx9Xegfic/tKDP+Q88YQ7l", + "8q3g7yuEXk/HcLA3aou7b2m0FnG/CvN6Xc7tSDZ44nMdiFJcK+X4pGosdjkysDQmh8Oj1uFW8iDM3ph4", + "M9tQPQTl40BOnlp+3YA4IVOGEUR+aivNdWGLXaYSTC0D2fb+Lqd11WpXB+JAwsItUYJtKDYLRgrgjq5V", + "bdsSK2tS2d3roFkgW4WJYqVrgkRsSu1Yemq18ZPXK1kOF8wtGfdztBkPO2Tw8Z4kBRCC67U7IkIzH2XA", + "8xnxbrRZYmZnTKe1rCRZhTRlSiECPUg4UvjZIICLVHatPYIWxHUviWIp5WIsuCjUrQjLxohgl0xV4m2l", + "gkfWHII9/lp54sHvLM3M0iWDQDX8N5qAMc6PcSyqg4TGQY0+JE+E1SyqPDO1dLEsdq8+IBSfJF8xozB/", + "KjcLMlNML6pzD+XMBRnvSqq4NRnZkvgiMA2K1lpi5AUTVVZWNBN8GtqGJrbWqj+gTXiNX937k9RyUJNm", + "jun1/eWaqbCQWEypKNLJCaZyVCqPEzyQMACXW9T9ZVl8gafSAT2lbP6/fZPlT6dF4/XfGsUqCCkeM/yJ", + "VdsGVbBRlq+uTsMmt9HpBQPq1uHirFolyJZ3xvYJmvR2S6KkTvdkt5i+ZtiBH82OZlG994NHDx4edsxU", + "9VFmP4sD9qmNfJfpGuNey06ddLEgPXrw6PHj/YMHj/euZWHxESot+9MWpVLdH4ARMXBYk3/945/vTurU", + "svcAvblH1xpUnrUP6W3WYUDvTv71j3/6Ud14QCFGswr33+IB0Orvk1R30rsc1I2B3Yxka973T2pKAlqw", + "GbLFENQb3Hfsug3KwTQARbpJwTSjETfLACOnVz7BhSvSQO3u0HpjsCGR17bt8FWBc+l8WpQAHxZX4D+t", + "lblBC486J7zT+bTNov2q2SuWI6XVouot0cFZwVJEWMFdzOeK6ppreMys3aFfxKqtxu3YEuvxe5tBFTAU", + "hOSttRWFshY05ElXqbr9je2s2C1rap3miv+65hy2H8FraX0DN3JA6RtledeGSrkdLsCb1ZpMq6ko1+b6", + "rOWtLG/d6/fbIQ55NelJcYNdv79K6OV1KjZozdKjG4Nb8rLtfo0kWqipElUTUKDJhA0qaRCwNNG5tQjC", + "mXeA9YFY0OhCzmZ11NwH7SjrCCCEYWO+F2oMvEz6oPV3OosmRLdFCxr3Huhxj0gFIZnpuNcwWwUDMVP6", + "fuI6qKPEjNbBnpeB9I1Baj+DaSKjC5tlEZP3D8mIpIwKTXKBh79hVdsdrbcO9XtZZW/8OuLjbmUp3WaS", + "KVvQS46ZWZxNZV5z6WTvudHoeortHJFYWtyoWoppN0MoZsMkj8pJ46VDxdI1DA1COSm8b2xZFhV8M0xs", + "LX5nSvYd7AFw7FevTvrWgQGdGO3Aap6SfqJ2BL1+r+yikaih/D3siQzGChx3E/g/XV3HanQ7WlYV08xo", + "hwRekkODCCB8SZhmRoC020OuHqC2eiXlAt0GnfsHLKjv3RIIiVmEJ1KvnsU6od+AuBsRCG6lQyEI+yES", + "xkOxJpPPa2cQbg7AKhsq2eFtO1UPcWslnGgjs179VE/Y+4ixuAkdGi7S1eve1Qx63b+kDm2oSNzuSqPn", + "9OrshrcXKoZjbVvtanSAkGIAp67YUns4LXKdR72pE1p19avgGJMQUGuoQIfxC/Z+/Vr/zN4bRFqP84TF", + "q2tbHEnLqtxltGnFbxwj2XagpWIb03PeQtpK67l+o8SVtiq7j9yVrtSt5Ktc2Z0zZnzZM0dGrTtUTxlT", + "M2m5JogvUvexsaTUJ+6CJ7vpdoMEDxZhrYiD9+0Y7Qn6FjC+z/j7NcRjC9iHcR0gpTxInhqIRSrdSul7", + "cvAQk4jpxtgFny9Msqw74BwEUJk+KqmrYoYJryjssvPlbvqKq95ubjurrYeE47MKyNBKchQnkk7WIXA/", + "LYp57XxGl6jFaTUSPtw/GI3290Y3guD2w7rGcj0tq7gUqPV22oLzKvUIq1rMLAeqtlCEa6/m1baJ78rV", + "JNooRtMjDOHJaMRIwmYIt1eExm62LDa7Xj94J1C5eNyC/v1GuX3zNvh68p2iKzGv/MuqPnCudTSj6vcN", + "BtEWNhOtgPMFovf2B6PDN7v7Rw8Oj3Z3bwM2u1iktriRh7/vXj1M9ujsIHm0fPjb7uLhfC/dD77DLriI", + "u9LqT1C21cumvCTrqEg1lka23BwyppoJ0+taI0BzTrhgA13EWm0OeFzDC6z9feP5v56e385grexwVp9k", + "VYSghuggZd0PkpebzFrbRXM2x8/Wz+JGsUzNgYTprTkUKNVxMJjrYrf3kRgPueh4Db2tFOx8Ea2Nr9t0", + "FYUs7HjSg7vcsuIh8m5APFRmve4CX73kArrTuVTcLNL1t0VRrAAkR7/p37WJ68hRQ3I8F1Kx+s+Fm1z1", + "EQWVe/1e8vtB/cy437tjiDks44IA3VZXpYIObmQJu2TJ+lXAIuXDQ1lPdqoYLsR3u4Pdx+iHkPx+8N1o", + "8LjucdC3q1Vdvl1fuvbrqMsaVpMJ+iRUu4+v5XHt13MdBf3EQ6nwynvZoRw7Gi9z0/urw4fu1ja4/Lyy", + "xw1MoFYB9GMlPXe5TapCUwwa2RDKfUVRqxuvxyqRkSmbc6G76G33R4Xi9gEobskTBzVuE8LLop9q80Ar", + "VTrhacpiTg2OSpk1EQx7HXVxzbfE9bKc+FoBaW0YFtcebwZb2BRwtemaHH5EZO9HvX67vXjX4YCgXs0/", + "URGNDAv2CZ+BDrrevk97bR8dGBiJ/mpHHvKtJFnHA3QpB3o9SZ/MpSFlMH5HfVsu2vWCxfghR2e+Fn3D", + "EsTeJ4FWKaDI+Dr2dfyMZErGeVTGjyY46BI7ROVi2FXI3zi0W9VvYIg3KDk26zfaFBrd9JNt+93QTQLB", + "tm/17mjzVt+KUqTfy7N4Mw+zhbpxsGthwG8IQQyoaOrL3pAEK5P5tQNHf11dwdU3r9UlRyAS5Zk3sABN", + "rVJSwNyCJoaQX+8zljDDQo0QmcRllATXJRfdzFJ3Dx8t2kycaJFaHchPjGWEGoukhP2lYM0LDcwnMC3u", + "kq2Rxw3X1uA1sImP3GrVB/dwoyTWulVVFW5bMgbL5Rs6b4u8clFRf/vZ1ESzVSwVz/BrQtrr9mQCrqZ3", + "e2vVH9+GWu4+hbRXzvTQAID18MQFzrrvv/QFBrGuTrwHIfN8iCze0Dlm3dtgR2kllVCGSjRyUGKoS5OF", + "qTrhnwTzkWoP/eIFBKzuI3pd2kLCDaFz2sxB0Zbg0lqiN/sQtaeMeUPna9F9m+GwVfX7k8HfrLqdTIZH", + "O9/95f8e/PqfQbV7Q4GgmRrEbIYeVxdsObD5nEBZMaxj22IyiZ42dO7ODKMpKtMQN95xpep4H4wK7rn8", + "maYrU0BXtUoypt2NE/rLf7Q7elWW8S1eGBvP7kfnWrmNnLRG+nt5K2Vq7p3qfUTd9nAsMEjvgi01qaR4", + "my5rJ/YbXVSpuOKTc0vuQyYuz8mUY85MPRbwvKdRxMCePXRZgziORUlkw4rRpNqOSzXnz4uzzFrHCgif", + "XgFGfvX2zfev3v78bPLq9PnPT44nPz3/n95RT7Crge0hHgDtHTw4dOneqyu5G0w5cv3MGUNy4uIVnM/D", + "LMeXPUKfgY0UQtCSJWHvoyTX4DaBS29McvMcGatRyzfPOfGRoMrGJCH3DAf+DYDeMGZ7rzrvIq49MXKN", + "efKdlocLsiq6WMIZ9/BKMdJLEsEMJbAV4dUuN7a+6IeLbneJhZsNHHYM/YXDFtCivuQagTu8E0ylMNnC", + "EBqfzNdHEG9fD/72SdFg0AXzE+dsGj3+FHlV365NpHopkwG881qSTwTV6nYtgiEE2JQNzei1WV/m08Bj", + "xum453xOAwaXkGHlk+Q/9QPaGDq2sv+tieDCAR3Pmpk57LG0S9XIJNHQlmgzaI/3SEG8b8EyRrBmG8TL", + "K06GteZ3UmF2XJ7iEHBILBEffl3YdnnKPODkACttjkZe+8CszKwykva9OZF5aFvWLRDE4wOPVKyyEViB", + "xTdcMheg1AGOBumtjk7mKqN4jnZ3TbYKFZhfgiLsetUusD6jxQl9X/QAJeA10TTE4jzKjFq7L77H7AOv", + "3S4B+3RN4DAar9ww1n6ditatiaeq1c2oUtXqvG354MFzvGoN92s7Ww3iLPuokWaIHv9KuflBKnwXt4O/", + "3DpkP17+MVMIq9cE5O+EZs9TFk9kbtaffygG0qO98otMs2WmYq8DoEjEUS2uuY0XeHiScgyrKw3LwaJc", + "cbMEpZF7mUwxHtSnB8aFxI7w57JjTMn64QNqz2eByJkXTDDFI0x4a6EGBb6YALavzHtoU2CuQOCiCPTq", + "6bHTu3gUZXxHc4Ok5x1Qn5wegzGeKat76I2G+8MRHuaMCZrx3lFvf7g7HPXwVbXAKe5MIWG/CyR3L+ri", + "BX8cO0noe18IaiqaMoM1fglAIqADpiuOTxA6rzwiM8qVe0VmiYwLguFQGzM4+Av1yN7KfbvsnZXHGG+N", + "YUAse+U291cUlPHs4DT3RiOHFW/c9YtBTDZyYufvzo227LeTVOeWKJDQYOWZ52XLYuk/9HsHo91rjWnd", + "UPDshjp+K6iLZmb4On8wGt1+p8fCxie6aHPnF1Y9cUhI1bP2y6+wZzpPU6qWfsGqq5VJ3SYYM9BrubL2", + "HWc0iYBVYNolALRh9juoRKl14Va5wGzVviJQaP0U2Lb9JhdoTd/LePnJlrDWh9dRfKizMzguH1bo+dPR", + "TkHGqxvpPnnQcku1d0BA39Mi1fq9nZSD0ePb7xR0vwmPIFrVE7BzzOYafZ8ShGD3IExSkd9yaSgp4hq+", + "oCPtZNZpQW798ira+YPHH+zxTljIHnDKVEqFjRKxZTYc+pXjbG0z5XFee6t5wj9+1nM3lUcjshcVCnL1", + "I1q9tprC4Op1dNA7auvTTi++R8I/uIMT7iZbZNu9zyOHOU7BrvwlHSdna5yWQkhQlnvBzOdC86O7vLJc", + "XoY/4Sn6Ugj4BSskvHK3Vi6FnUzlwj6AgxLg6zJy09X7pi78vSm/VNyF0K4BTWOGEGMNzDReDolfU/vo", + "B6WvthD96tK6m9XP1ykM73M5YXt3ccJwxoWl6Os19fWaWnfKLbX4KeDBrJzyDjqIa2kg/nz6h2trH77q", + "HrrrHjppHgS7sqUBD3JInGtuJGNG9ALynYKx2QI/eSccQ9Vw/juhKlrwS4bofpj3Lk8Mz6hCF5uUgJXA", + "2tBbFRNr1RJFczvQ3MA7ZJYL3AT00GyCLjCTNiDO0hWTCwHKWqq910yJq7iSod2e/aCCvWiwvBrJ1UJq", + "VgAbClO5zTHOW9vXMTY7HIs3DvEWFhC9yj2v0SxB3N41+h8pCB0LV+Fbz0K8R5ymacm5qELwRG4hOu22", + "rEwVRzrRkQyBDr1hggoz0BmL+IxHbloXbOkcW4MNdkplBQP244S8MN5Ra287DFyHrkthlOJnxTfiKKlu", + "vxHSuCTfpZHLYylRBbkPghlK5omc0mRi1+eCBWyCL7CEW5TS4FJak4SMmc3Kny3NQgr7dz7Nhcnt31Ml", + "rzRTYHMaC4xIcWsN3gqFgEiuMDdemkllrJ8D9rljh7jzxwVbfhiOxZM45cLW0LYKTbQEMwXUM9KDh1ju", + "1UIP9jSF7eBPc21kWoWA9XRnhylzk+XGhdZoZvoh+NOxMJL84UEuP+z8Ufb4AY3FjMZAJ5UidkooW7eN", + "Wk8ozH6CRQPmdoYLMO7BRWrdPOaKCoNjKlE6yby6pVtFmghMQttc4YgKksnMpthAolpQILlaGwhaQZOE", + "GDxKvi4I7riTLfNJabTggk1KBhcwfr3PmOIpExDEiSbVBNxkY6quuNixydKR23nOt4QR7j04dIxvOBav", + "i3ADe/OkNJJ6giwJ9FP2n3bI9lXizg3CMsBhQ6BXy9ZgHQ2qtJSUxienQp5F7W81C/OQnNg5ujbHAmjT", + "ohHBThwekBP+fSUO+1vLKv3KeHjCVMasPxbOJueTPPVdoNkknfYJcCScD8hKpOozRihA6Wges7FwU8Ny", + "cE5KXvz09O2ObY4AWhDhYsEUN5bwqYZs+i8xzbw9lZpFihm9gwRJZGbfbtCtYn9H1Lf6C89pAsfCHh8C", + "BkSHcnEl/Hyr/AUqnDx52ifauoK6ehEVY6FycKOFGVWHt47EqgsZ4NvWy8qPwpEv+szWsrRwLeEcxEPy", + "/D0ISNwQhmk8XYicHotcKAbdwwqQd3/DnGXAP2xGYQB5ZQo5H6HK8BmNbP5NtCN+Szj8i2ng5mPBxEzC", + "ZsUypdxmoLtKuDZ6SHAn/DgdiFSuWWNPa1ghfuzwE445mDWjIKhWTFCk3ubNxgU5+b5yv40OHoWvOEc2", + "q63/19mrnwkKysAWbbHSg9KuvgDCAsc2TOfjxIzngFxmbceYMnXc4/G4V3hYxNs4VlgcVOAPBmjm/w6G", + "9p3tps/j74ZDaMp6EByRX/6wrRzB9ZalFqN43PvQJ5UPc24W+bT49muYANugFc9qdzPZspLntuc3MMOK", + "EG6lVjgS0glmwLxIKRRUfcmmXFC1XBfkHFh6t4JyZp2LK4vxxxidice9o7F3Jx73+uMeE5f4m/M5Hvc+", + "hFegwbRa9OuFv0FBRIej0fZmlH63vgE3gg62uk+slmlVVBTJhWEHHUT0n8tk92+tEvq+UKII6LyETbM3", + "sLeH3fdTG3q/g83/WRqQ4hOWMnTyGlTTJMJNgek9B1bOgev4JsbKyuu+qrUKmCsbT3QqIpb4J/pmpfDd", + "GxqBiiIGQu7dnpz7OjeFKb1Ib/JF2c5xs8rzvdbUd88UN7qr265m4rsf+v3ibG0BS5uzs7FLHxYRTu6E", + "wF1O7UZsYXiLnuGYBmdMGPIcfx26/3o9EQLRnidyfn5k1XyQp4QkXLjYoUpQA8gtbi2xksXuKurZfxaZ", + "NbesiPOvf/wTB8XF/F//+KdLiPGvf/wTj/uOxaTEFP/nC0aVmTJqzo8IRMgOaALvMTcZTMLNLplakv2R", + "sw/iJxiSYFERgCFzeGDCg9/kSlRirGySS+0adGZFmA8XOdMO+wwK8pnLwGX9oAM6Xn+W7VLe6YnuBzBk", + "cQaVCcCt6GkAATi54BgTZp/PvbB5xc65ZmBpunSvOPZu5i+GvTeWegd2gNdkMLjEoXOHH9ykydbZ2fPt", + "IcFnoKUKzLKGj5qyGfe+GX7lSZt5kuUodYaCq2x5U0QzOuUJ9+aJlhRR9gg6lUoZi1AkaPBNHPmRAo8B", + "t2znNN3HomPx6mwHzTGGRSZXrO84gXKwzGUOSeli4rAH5F/coCfpwJUdixmj0AY5fmaZQCVzQRFEXTQs", + "EP0I/eG5qaWr7I+Fhd92WiU4eCDJJlgJ+59Tw67osk8qkq9NKZVQAy913YfCY2HDkt0aDBDfiVSGOUR+", + "Zoc08F7/Lr5TsRkoenAiNkOf7XsLFB4uGqJfxqD77mxkuh0WkU4ch/nN8YkqrfoSW3p15ndjG/VwUcKR", + "GlAJN0enQY94LkVtV4vg0wVV8SCScAlUEfAgWiFh8byNxz6ttNW7RUmm1k/gOP3YJNcvTbhYrE4ADrFF", + "9Vxv5X/mynQz87sW/0x2frtA1zH0W2sPs/zGru5Xo38Ho3943bwDQMgK/8zD1t5edIDt4p6CAzztra65", + "/VJZsvtQNZItjweGFlSpyOnTY0LjWDGtt/+9FZEwU0ulpfwH9yOw4vtwU3Njkcruhde31AnkS2EHr92o", + "CfXzaiYlr95vO7WMZa03XZG8rLzybv/2aHR6nWukFHpLWvt6k2x0zOc6gjDnKrUMUDRKWCG+FOe0SkWb", + "tMrW5b+4ctaKS7YU+FO5A3l3+mXXdS6ad8MdMMVnDYZ4j4ywDstAqCjQG74kan5b7KKb1zr18+dFmqO7", + "k4LuWhUdIvMv6bkYN5YNuKAFRWq9QF8wY6GQbvOd7noITBz03O5U24Eu7ayLadmq1gHHTgg1Mevfvse2", + "SLenr23vz/TyxeW5jsTilvyriNLhsVuu1boH7rHLm39779sa6uMd+9M4AgssMnxwummEUINUtFQvRbT9", + "1aXmk1M0LnblEav8vFlcaLLxe/nOuiu57onwGvUc3MZ18a7lgswSQJS2RoCYz9CJ0FQQEi0j2ruDUb52", + "7t1EUcOc7+SXiBFwmifWuhFJcclUBdW1eqXu/IEO7pufSp55rb1d375+OWAiknFhPGmXSd2XT/xgsvRf", + "i/u/+1P3Bca+cy8etAmMH7H/NvCEFDC//2vvBwf0+7/2fqBJxgX7X/tPEmqYNtu3Riyju7rp7voB8wUT", + "H7xfeH3RVljTjqHzasx6nTI91PV1KLMGUe2V3UXQ3Sejvk8v5DVxvTtJeHdG9/CSumt0o2OfwsRuqVQe", + "6a4BX/7vLfSd1Qj6zvV1te45hsJYBIgvEFJJKo/QTB3EPffE7XiTwFCcTcqIolRHfYQv/6dSSdhJX0sp", + "UazrV71EF71EdbnWqiZswdtVTtg+7sn6XhBbaLXx01dovjsw6DiKrEDz1SzcJTjfQmqDnz4PnJZ/p0gc", + "XhyF6n3W0WRacoq195o/U8fP+rjDfdjT42clPsMdGVD9OO5cI+D6vXtp7Ek65fNc5roamp1SEy2YdrAo", + "CavfDF+arqKUG1q1FZ8xlY7u8k67c2XEV7q/JTVJc0Mt87ZeEJseI75U18eIK4/QHAgZ76BjXIK3vk/+", + "ud0SneJB47uScQ1b/0O/y7hCjyRImKNJ+Y4h+LQ5Gov/DQ+jXwyj6a/f+YD4fDTaO8Tfmbj89TsfEy9O", + "PKkwYRR3uZqf/PwMXU/mGF2A6ZxLRJzmOCwMCZKez9v0b/dyK71vuj/dPBV+fbp1erpVlmv9080WvOW3", + "Wz33250/3jy9hRbcfvuTPt/+5Dbl2tNS57MZjzgTmEEPtZJ6xYnaPjG/mpNvph+lwjlxVDwwa5JI52dk", + "wbU2SOhFWvZP6uLYb02kIwk1hqWZASS2iEFCbxs6qBe5AeAnn1cHJ+hTNPJyPqHr3Tc18Y2EI3dr+X/W", + "Be4GX7pFbtm7fuq6jr/Q0Flpg+Vj/7gsRZv21+X9Eu/tvik7XLV3/6r8kknMPt9Wly6DF0IgT6S1P6W5", + "jTMuapYQs0Py5s1LH1NsAfxc1lEjfapRn45+LKqpRhEGz+dwtQXqEIAOgwDq++SdMaNxwgXDIAymQ+G/", + "9QTB93osPr0EHM5+fNe29y7H0m7oPUrA98YK7kTWLJwNjLMUF0ezkhi5OC1e3sRT80XxK8eAAownJOvt", + "0NzIgUMp2FlIC3MbRvo+TWiEQN9QzGLQOmAYCzpdbQrRXpRMEqYstnCWGy9uAcy3GxwXRhbZ851kdg7N", + "T3JheHLetz6QUDbRhIqlQ/Mbi1pnTuZD8AYEJsERKpbZETdSgcOgOagaoRTiLFS7JDS5oks9Fg7uwVZP", + "JUZFRxaGO0mG5EeJSDs2JX6F8dp81N/osTjnccImDijnnHBN9EIqwwSLSSovma73y6hKOFM4iacUVk6T", + "lC5LnFG3PjJjFq6yBscj4d9UxBwzG0PPxZSPxoKSvdGIpIwK7cA1NJ3hhePaIDiI2oC+JZQcjB67Wo19", + "wwwMfvm34DQpxS5lRKeQxBqoGIbIzTZuYOoyjSumjVSwfTOutN2vQr/pUsjWNpZrnzg77pNclPAhqOuv", + "WNdgu0yuBM7TmScZV8U16BCTpiyisJ5C1vtBXGsZRbkKXZCw1ZWU9/+OgmNleme4VGFwjgRVBhHEAEhE", + "flzgmZZ4lLa/baGqkqj+HBdN8JBIRSip0HWp0WBRjqxxC0Fnz8v8zWLiyp9vf+vPDhxfxwj88RfzL+l+", + "QiKSs1ntAG6+muwBXhcUt0rCf9Zz+tQxvRqLizmdC6kNjzwz9DD71Ufz1wdhpwfh+pUNUvNMqot2T+gf", + "pLro+gLzHptf1kOsOsPP0BABw0Mnnfu3R6A23D5WgGju/JHWpK/ilKLQxY0ufJUJJDpgqqqVv3OzQfVV", + "t2WRNuEyVdbYXeCiwSNk4n60uf8r/uZoYohcq39aZzUkNtjs0mdturRPjxJJ7nq8Ek5VFfDBvgW19T/o", + "e3EI6cpv2BZK76vbFWSqADS6Eam16NzDkgagWsfirbbJLc6t6emcFDwYBFqb48GlHwHYVvgN27eorjTL", + "zgso/e0j8gIPcmXbbOdbNnMS0JqWCbNorJdpen60mv3+3ckJVsIyC5vn/vyI+Iz3xf2hoVQVhhVmkVBt", + "yM8OXHareIzjjp6DDrMyv20H0FqmOhiLEFgrYJ3aBvmMnFdwW89bQAU9v30p5/qzMRWVOaPsXIwk7umI", + "tMlE3Gtz8uBJ2PCzOxqF8iZ0hI+1w7hl9NiVwbyU8yJfVY2UaZZ1JV83TKTiyzRdQ8Nkq+QgRJtY5uYv", + "2sRMKazsqLuNuMkWjew/DL0AQnXIo/5gb49Fy1LZGYaXCphqJV+L/ddlmvb6PTceqHdlsjSQueXj4Xib", + "DX7oh3aogrn7JxbP7+1KdLnNiqPikwVZice9oADKFc8LbulNoH/rN1QF+7dx3TkdQPsz4rUt8Kc3cXpF", + "452J7+5F0xCfgvRxb6cYxayGLvxzkuQr463rzYQsUJVxZI/vYmQIvUyiRGpWs5N9OcCNTl/YEL3b9W1+", + "jQcwvDj3WTG7OAKduapnvuZnoMLY5HLjx0z8dO/c92Z1BF8yCIVemQ2wvwba3yannM+ekD7dlqxMtQuF", + "fKXN6ytrOxEmPLdWWYSrELuMo7mRKTU8wnSA0UJKXSH7Aprf5tJ1OviCMlFDZZUFLhDjHEj13Gnzz52I", + "eeQ0j4RWP7k+hljdhW+Ea/hPZY0fKsqVguP3/QsKM9NoSJmmOJuRjOaagZyZg0ZsGSXM539kYFePaGZy", + "xTDbNCMpFzzN02rOBdixS4r4UOe76XmfTHNDEtCyqeKj91mKZJoyEVs5aSwWjF5ypmBREmqYiJYDzYTm", + "6BoFaR8wZydoarKYosEMAykVAwrEBBYpMzSmhqKgcw4nfmJjwc5JphgSkdWOCPa+pIYY85t+a7PpQLPn", + "fqDnhGG6CK4XRYLziMZMRME0CmefNxv79Cr9M2aaE70nB6sb8dL79Liqqq79cD4PZ6wvFvKjA5tfI/Tq", + "9kd1PYjGk9G/55G2c/VzvCc7XbHE607x52GgK4juszHS3b8VTioS57a7yqlEMv+z6hELhlJDhIAAXbuN", + "N7WvFdlZi2W+Fs/b+cP/eXwD7eJnwgn7rQ/7tjyA5aQ/B5brVvVGPPee1KpOl1TVyd0fC3aDuj/xSaoK", + "l/tSlK015L2Cb1e5k1EUX19SfGXbTbbt/EZuyra9bnbFM6LCyLkYoKttmIM7NW4rq3aqg3/ToJ7G7Cos", + "895ZZGm5uHMYTc8aM7oEVcmfwdd6jf0okkpZFBFq2JcF/V3RGlajLFA3VyYY7fug13cnJ9ttXEKZtTxC", + "mS+YQ1Qim6BaGq8qcF9dMqV47OFNn548K22yKhdD8irlhhhJLhjLysAgDM4E1M4CT6Ux7CZwSr/HhFHL", + "THJhNo6iLHo7gyl/kA5W5jMUJV0uia/m8M7mcNTsf3nsDLkMocUE1r9MDTUbc1xzMZMqtXIZncocWgce", + "BMsE+2mdLWY8YXqpDUutcydAQcBxw7xELve8q2d3uY+uzXBybNRhxlTKteZS6LFwITcZU9A3VIf2K35q", + "QYOAoQV/PbVM8vPwgYTBWLc/atpWDZGvMKd176i3Q7NsJ6aGtvjZueF9xJB+QKdGopfpFAIowSvyQpOt", + "hF/Y5wm51CSBP7bXekVOsN6nzqx/85MFK30sZjKYt9TSbEHMf6rgOMfWvGHys2Br9/TO/GgvwhesenA9", + "L5zJVharN/JYxWgywLz8HouJ5IYn/HfLdqERrg2PbBQZLfbx3UnB4IdjccKM4pG190bgf44gFfjS3cmU", + "jHYAx28/yjgC+u0zHBwy3/bPKfb49PQtlktZKtWyPxZUuIbfPDm1luYZjVhzoIIZsCST451XG3zWz3CZ", + "/o39J+0E1wJSBDf8q3ny+rAxrWdItxxRma17tsnsT+/g66TJrzqOL1PHATRczmarwGrzAGthfcalTPIU", + "/mH/ON4EVWdotHiHRT8bydsOZ2M3foJfxKF0c4qZzfF8LwYYu2Bfqv8sLJyfAgoxNc/E4C3wxPwZqfvT", + "mxKq6/gZml7divr86Z/N2brrm8+NwYOmVNfjSznmltL8TIxcrwm7orxdE/Z9IqML7dB1qipMohhFyHz4", + "sYQ4d+ZKFBMw2Jc4VCqLfcZ0fywaylAL4qQJJYaplAua7OCcbSMI1u41avRScoy5jzBmZqBB1Z3JJEFE", + "dkQ0hNlAiZlvoGJd1i7DZLVM1TBqJMA2yZR5APvt0NPtr5SbH6Sqo9F/LnzxTWX9EeWRctT9bwDgb+/x", + "owD5T+h7dNuOc2fc9iPaeiHLH61aqk9wb8a9/ZEe9/pk3NtLxz3YgacU1bnUkAck5SI3gGz5zOraMKr6", + "cEQ0i6SItcfR99rE/ZFumbkjy5aA3UOsd5dij6MqXMrXrpMQe4ByBOpjABHZqh44dybjPh66mMjcWNOD", + "O1euVMwMqke279xuXDkjX9/2XTj5X93xrfEo3GVgl5Wtt5w9y/WCtavcXtqkWbmZwjoUSbb1gvxdTnUf", + "og6sZh4Q81b4HtQ+tR3cRe4I6Oo6eSPc3L8mjeiQNKJcqzD+pnX2hCvZU4cF4WTvM6kM/O5wCxwN4UsC", + "wUAwFearp8djEQErsmiRiqUSuZODuLe38JO/npHnT1/3yTNM+Ex+zKfbQ/IKIm2wVe3sRWNhJTHLvCIK", + "wOp2BnHoerZjR+q5Tcd16OBaUvzep7P04MkIWHn8XnmH9X5vwWiMEskfvZfSdhYAkn79Eg4Q7J+rWWx7", + "b63wATp8tRw8mRmmVps9cTFbooBBcZe0RxV0ght2jB1qD6ZX9mllA4t2sr/XC4CffPiax+P2E4XfjcUO", + "+3WpHSGa70vNEYwHsWCOIRZYva6LTBhtEcuOl619YGCXx88+yaPiVq3U63hXLV3Av+vpwpl+sYamrLZP", + "QMRFBp2Nll4fqLywCNe2GoloRiNuln1Ck8TdUe4mKLxjBoX4O1WMXoDefggoZK5nH1wMttq+N9SSmOsL", + "24KzxQ4JuMfpfFoMjuBBs1ZjXHMI0TWSRDSJ8oQaRthsxiKMC8aUPLrFllsMpXeLZ6fsJAgLU/Gwz7+4", + "tIVhmsDdK8miSXE7dqt3FIsSytN2PHknqKHzI7oaTKFRKcB3IbEURyMltSauqQFL+JxDWg30lIBMGqDm", + "oikbiyyhQjBFcm099GHog0wxrXPFfANwdzmK6pMSKzJT0jjXhERKpcfCU/i7E6INy9aQ2Wvb8gnO+ZZk", + "W9u46+melNSNMbSrQlwRAhtiKcUuONARqIbuwS3eDui+pcQv5eC/UXwOvq+ZktQyWeuXZI+1X0576GvR", + "060pTM+KUt1SmBatViIkK9GDa7H+JiV8ety7ngdioPML3goH6T5dL6L5J6jUse965Gx4EO7TR84ylI31", + "3zHx6VklYLGrAquk8C9NnVQZee2o1oJ+N0N8dY7yvc2o285YXvcG4fUlI3fRWihv24P38yOE0d0iTtx1", + "3rwvm7ZqyFu1t2kL/MDmBAmfBQXeTmaEe0ZcuUFmhM8KAwBo5x6xWIIH9b5i+mu2Z6m+Jje41VB+m+EA", + "oeHaQvkt13POq2sfSu9cmW7PJNfin0mCtwt0HfndL/vXV3+HJ0NlsTaZoIHgWZqZpavkbZWl05nmv7Nh", + "iyG48Fu9PVPwDVw6Px15eDptdei8R/SyezTk3IvPqMsGyTW4+ZVZ93xg9xeGd1g9c7WLZQdunQFV0YJf", + "snale/0EuyXKFBtkMkPjSmwXzK2Hv8sMVcP578Q17/Bf3b8wnSgmMmAxiblikQFvEWEkcgTbxzeaKCmN", + "+y7Vst1LxB6RH5RMn7jZbLgP3ZlyyrDSzzBdDmJq6ODSc5s1KrSP8O70/pTA8AgX5MX3ZIu9N8omLSEz", + "ePlAtLlfUvY+YizWSJPb1QHvjsJDhIYn82mXUa5JP/MK/6AJiXJtZOr3/vgZ2cJ0dnMmYC9A1J+hJJsp", + "ecljFtfGCM77dlV3Wxb0unpXECqKXIT+cWEHdy8yTJcLaf47z+psoXCJmXJBcXAbE7zUz5QFFID+KBfe", + "AcftkR/F1yvMvfy2/GMHKBFTq7pFNFJauOntr9fcl3zNVYOh/J1Wu+28e8565XW3+KiOYUu3kYSiiJ27", + "W7X1u88npIfrLzKax6nOL4sHaZva/PMiwdHd3Q93rS5/9wWHgIKy/LKxbLYBdRkmmJfo0x2zS5bILMUU", + "91i21+/lKukd9RbGZEc7O+j7vZDaHB08frjf+/Drh/9/ABXpBjflAQIA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 8de7c33d3..ff2334317 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -6,6 +6,7 @@ import ( "log/slog" "net/netip" "os" + "path/filepath" "strconv" "strings" "time" @@ -425,15 +426,20 @@ func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager ins log.Info("registry CA certificate loaded", "file", cfg.Registry.CACertFile) } - registryURL, err := builderRegistryURL(cfg.Registry.URL, networkManager) - if err != nil { - return nil, err + registryURL := cfg.Registry.URL + if !cfg.MacOSBuilds { + var err error + registryURL, err = builderRegistryURL(registryURL, networkManager) + if err != nil { + return nil, err + } } if registryURL != cfg.Registry.URL { log.Info("resolved registry URL for builder VMs", "original", cfg.Registry.URL, "resolved", registryURL) } buildConfig := builds.Config{ + MachineBuildOnly: cfg.MacOSOnly, MaxConcurrentBuilds: cfg.Build.MaxConcurrentSourceBuilds, BuilderImage: cfg.Build.BuilderImage, RegistryURL: registryURL, @@ -444,6 +450,17 @@ func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager ins DockerSocket: cfg.Build.DockerSocket, } + if cfg.MacOSBuilds { + workDir := filepath.Join(p.BuildsDir(), "machine-work") + if err := os.MkdirAll(workDir, 0700); err != nil { + return nil, err + } + buildConfig.MachineBuild = &builds.MachineBuildBackend{ + Driver: &builds.GuestMachineBuildDriver{Instances: instanceManager, WorkDir: workDir}, + Publisher: &builds.StreamingMachinePublisher{RegistryURL: registryURL, Tokens: builds.NewRegistryTokenGenerator(cfg.JwtSecret)}, + } + } + // Configure secret provider (use NoOpSecretProvider as fallback to avoid nil panics) var secretProvider builds.SecretProvider if cfg.Build.SecretsDir != "" { diff --git a/lib/registrypush/macos_test.go b/lib/registrypush/macos_test.go new file mode 100644 index 000000000..a9acd4275 --- /dev/null +++ b/lib/registrypush/macos_test.go @@ -0,0 +1,48 @@ +package registrypush + +import ( + "context" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/ocicache/testutil" + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +func TestPushMacOSImagePreservesManifestAndPlatform(t *testing.T) { + host, _ := newTestRegistry(t) + img, err := random.Image(256, 1) + require.NoError(t, err) + config, err := img.ConfigFile() + require.NoError(t, err) + config.OS, config.Architecture = "darwin", "arm64" + config.Config.Labels = map[string]string{ + images.MacOSMachineVersionLabel: "1", images.MacOSMachineKindLabel: "macos-image", + images.MacOSMachineFormatLabel: "raw", images.MacOSMachineDiskLabel: "disk.img", + images.MacOSMachineAuxLabel: "aux.img", images.MacOSMachinePlatformLabel: "config.json", + } + img, err = mutate.ConfigFile(img, config) + require.NoError(t, err) + img = mutate.MediaType(img, types.OCIManifestSchema1) + img = mutate.ConfigMediaType(img, types.OCIConfigJSON) + p := paths.New(t.TempDir()) + digest, err := testutil.WriteImage(p, img) + require.NoError(t, err) + target := host + "/releases/macos:canary" + result, err := PushFromCache(context.Background(), p, digest, target, nil, Options{Insecure: true}) + require.NoError(t, err) + require.Equal(t, digest, result.Digest) + ref, err := name.ParseReference(host+"/releases/macos@"+digest, name.Insecure) + require.NoError(t, err) + published, err := remote.Image(ref) + require.NoError(t, err) + actual, err := published.ConfigFile() + require.NoError(t, err) + require.Equal(t, config, actual) +} diff --git a/openapi.yaml b/openapi.yaml index 73aa90cde..a51553ab5 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -4683,6 +4683,25 @@ paths: dockerfile: type: string description: Dockerfile content. Required if not included in the source tarball. + machine_base_image: + type: string + description: | + Experimental installed darwin/arm64 base pinned by sha256 digest. + Requires server macos_only and macos_builds. The source must contain + build.sh at its root, which runs as root in the guest. Machine source + is limited to 64 MiB compressed; only machine_network_mode, + timeout_seconds, memory_mb, cpus and tags are accepted alongside + source and this selector. CPU/memory zero inherit the base. + Linux/cache/secrets/admin options are rejected. The builder and its + output get their own machine identifier and MAC, so the output can + run beside the base. + machine_network_mode: + type: string + enum: [isolated, egress] + description: | + Native machine builds only. Defaults to isolated. Explicit egress enables + unrestricted VZ NAT for checksum-verified artifact downloads; it does not + enforce domain allowlists. Linux builds do not use this selector. base_image_digest: type: string description: Optional pinned base image digest