fix(statewrite): preserve unmodeled manifest config across state writes

internal/config/statemerge.go

@@ -0,0 +1,135 @@
+package config
+
+import (
+	"fmt"
+
+	"gopkg.in/yaml.v3"
+)
+
+// WriteManifestState rewrites manifest bytes in place, replacing only the
+// mutable state subtree (the `state` and `latest_release` keys under
+// manifestKey) and leaving every other key untouched.
+//
+// State writers (reset, promote/finalize, hotfix/finalize, rollback) only ever
+// change `state` and `latest_release`; the rest of the manifest is read-only at
+// write time. Earlier writers re-marshaled the typed CICDFile, which silently
+// dropped any key the running binary does not model (for example a config field
+// added in a newer cascade release). Operating on the parsed YAML node and
+// touching only the two mutable keys preserves all other content verbatim,
+// including configuration this binary does not model and any comments.
+//
+// state is written when non-empty and the `state` key is removed when empty,
+// matching the previous `omitempty` behavior; the same rule applies to
+// latest_release against nil. manifestKey defaults to DefaultManifestKey when
+// empty.
+func WriteManifestState(current []byte, manifestKey string, state map[string]*EnvState, latest *LatestReleaseState) ([]byte, error) {
+	if manifestKey == "" {
+		manifestKey = DefaultManifestKey
+	}
+
+	var doc yaml.Node
+	if err := yaml.Unmarshal(current, &doc); err != nil {
+		return nil, fmt.Errorf("parsing manifest for state write: %w", err)
+	}
+
+	root := documentMapping(&doc)
+	if root == nil {
+		// Empty or non-mapping document: start a fresh document root so the write
+		// still produces a well-formed manifest.
+		root = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+		doc = yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{root}}
+	}
+
+	section := mappingValue(root, manifestKey)
+	if section == nil || section.Kind != yaml.MappingNode {
+		section = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+		setMappingValue(root, manifestKey, section)
+	}
+
+	if len(state) == 0 {
+		deleteMappingKey(section, "state")
+	} else {
+		node, err := valueNode(state)
+		if err != nil {
+			return nil, fmt.Errorf("encoding state for state write: %w", err)
+		}
+		setMappingValue(section, "state", node)
+	}
+
+	if latest == nil {
+		deleteMappingKey(section, "latest_release")
+	} else {
+		node, err := valueNode(latest)
+		if err != nil {
+			return nil, fmt.Errorf("encoding latest_release for state write: %w", err)
+		}
+		setMappingValue(section, "latest_release", node)
+	}
+
+	data, err := yaml.Marshal(&doc)
+	if err != nil {
+		return nil, fmt.Errorf("encoding manifest after state write: %w", err)
+	}
+	return data, nil
+}
+
+// documentMapping returns the top-level mapping node of a parsed YAML document,
+// or nil when the document is empty or its root is not a mapping.
+func documentMapping(doc *yaml.Node) *yaml.Node {
+	if doc.Kind == yaml.DocumentNode && len(doc.Content) == 1 && doc.Content[0].Kind == yaml.MappingNode {
+		return doc.Content[0]
+	}
+	return nil
+}
+
+// mappingValue returns the value node for key in a mapping node, or nil when the
+// key is absent. A mapping node stores alternating key/value children.
+func mappingValue(m *yaml.Node, key string) *yaml.Node {
+	for i := 0; i+1 < len(m.Content); i += 2 {
+		if m.Content[i].Value == key {
+			return m.Content[i+1]
+		}
+	}
+	return nil
+}
+
+// setMappingValue replaces the value for key in place when present, preserving
+// its position, or appends a new key/value pair when absent.
+func setMappingValue(m *yaml.Node, key string, val *yaml.Node) {
+	for i := 0; i+1 < len(m.Content); i += 2 {
+		if m.Content[i].Value == key {
+			m.Content[i+1] = val
+			return
+		}
+	}
+	keyNode := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}
+	m.Content = append(m.Content, keyNode, val)
+}
+
+// deleteMappingKey removes the key/value pair for key from a mapping node when
+// present.
+func deleteMappingKey(m *yaml.Node, key string) {
+	for i := 0; i+1 < len(m.Content); i += 2 {
+		if m.Content[i].Value == key {
+			m.Content = append(m.Content[:i], m.Content[i+2:]...)
+			return
+		}
+	}
+}
+
+// valueNode marshals v through YAML and returns the resulting value node, so a
+// typed value can be spliced into the document tree.
+func valueNode(v any) (*yaml.Node, error) {
+	data, err := yaml.Marshal(v)
+	if err != nil {
+		return nil, err
+	}
+	var n yaml.Node
+	if err := yaml.Unmarshal(data, &n); err != nil {
+		return nil, err
+	}
+	if n.Kind == yaml.DocumentNode && len(n.Content) == 1 {
+		return n.Content[0], nil
+	}
+	return &n, nil
+}

internal/config/statemerge_test.go

@@ -0,0 +1,176 @@
+package config
+
+import (
+	"strings"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+// manifestWithUnmodeledKeys is a manifest that carries keys the CICDFile struct
+// does not model: a sibling under config (config.future_knob), a top-level key
+// outside the manifest key (custom_top_level), and a nested unmodeled block. A
+// state write must preserve all of them.
+const manifestWithUnmodeledKeys = `ci:
+  config:
+    trunk_branch: main
+    environments:
+      - dev
+      - prod
+    future_knob:
+      enabled: true
+      nested:
+        - a
+        - b
+  state:
+    dev:
+      sha: oldsha
+      version: v1.0.0
+custom_top_level:
+  retained: true
+`
+
+func parseTop(t *testing.T, data []byte) map[string]any {
+	t.Helper()
+	var m map[string]any
+	if err := yaml.Unmarshal(data, &m); err != nil {
+		t.Fatalf("unmarshal output: %v", err)
+	}
+	return m
+}
+
+func TestWriteManifestState_PreservesUnmodeledKeys(t *testing.T) {
+	state := map[string]*EnvState{
+		"prod": {SHA: "newsha", Version: "v1.1.0"},
+	}
+
+	out, err := WriteManifestState([]byte(manifestWithUnmodeledKeys), "ci", state, nil)
+	if err != nil {
+		t.Fatalf("WriteManifestState: %v", err)
+	}
+
+	top := parseTop(t, out)
+
+	// The unmodeled top-level key must survive.
+	custom, ok := top["custom_top_level"].(map[string]any)
+	if !ok {
+		t.Fatalf("custom_top_level dropped or wrong type: %#v", top["custom_top_level"])
+	}
+	if custom["retained"] != true {
+		t.Errorf("custom_top_level.retained = %v, want true", custom["retained"])
+	}
+
+	ci, ok := top["ci"].(map[string]any)
+	if !ok {
+		t.Fatalf("ci key missing: %#v", top)
+	}
+	cfg, ok := ci["config"].(map[string]any)
+	if !ok {
+		t.Fatalf("ci.config missing: %#v", ci)
+	}
+
+	// The unmodeled config-level key (mirrors how drift_check would look to an
+	// older binary) must survive.
+	knob, ok := cfg["future_knob"].(map[string]any)
+	if !ok {
+		t.Fatalf("ci.config.future_knob dropped: %#v", cfg)
+	}
+	if knob["enabled"] != true {
+		t.Errorf("future_knob.enabled = %v, want true", knob["enabled"])
+	}
+	if nested, ok := knob["nested"].([]any); !ok || len(nested) != 2 {
+		t.Errorf("future_knob.nested not preserved: %#v", knob["nested"])
+	}
+
+	// Modeled config still present.
+	if cfg["trunk_branch"] != "main" {
+		t.Errorf("config.trunk_branch = %v, want main", cfg["trunk_branch"])
+	}
+
+	// The new state must be written and replace the old.
+	st, ok := ci["state"].(map[string]any)
+	if !ok {
+		t.Fatalf("ci.state missing: %#v", ci)
+	}
+	prod, ok := st["prod"].(map[string]any)
+	if !ok {
+		t.Fatalf("state.prod missing: %#v", st)
+	}
+	if prod["sha"] != "newsha" {
+		t.Errorf("state.prod.sha = %v, want newsha", prod["sha"])
+	}
+	if _, stale := st["dev"]; stale {
+		t.Errorf("stale state.dev should have been replaced: %#v", st)
+	}
+}
+
+func TestWriteManifestState_EmptyStateRemovesKeyButKeepsConfig(t *testing.T) {
+	// reset clears state by passing a nil/empty map and nil latest release. The
+	// state and latest_release keys must disappear while config and unmodeled
+	// keys survive.
+	out, err := WriteManifestState([]byte(manifestWithUnmodeledKeys), "ci", nil, nil)
+	if err != nil {
+		t.Fatalf("WriteManifestState: %v", err)
+	}
+
+	top := parseTop(t, out)
+	ci := top["ci"].(map[string]any)
+	if _, ok := ci["state"]; ok {
+		t.Errorf("state key should be removed on empty state, got: %#v", ci["state"])
+	}
+	if _, ok := ci["custom_top_level"]; ok {
+		t.Error("unexpected key promotion")
+	}
+	if _, ok := top["custom_top_level"]; !ok {
+		t.Error("custom_top_level dropped on reset write")
+	}
+	cfg := ci["config"].(map[string]any)
+	if _, ok := cfg["future_knob"]; !ok {
+		t.Error("config.future_knob dropped on reset write")
+	}
+}
+
+func TestWriteManifestState_WritesLatestRelease(t *testing.T) {
+	out, err := WriteManifestState([]byte(manifestWithUnmodeledKeys), "ci",
+		map[string]*EnvState{"prod": {SHA: "s"}},
+		&LatestReleaseState{Version: "v2.0.0", SHA: "rsha"})
+	if err != nil {
+		t.Fatalf("WriteManifestState: %v", err)
+	}
+	if !strings.Contains(string(out), "latest_release") {
+		t.Fatalf("latest_release not written: %s", out)
+	}
+	ci := parseTop(t, out)["ci"].(map[string]any)
+	lr, ok := ci["latest_release"].(map[string]any)
+	if !ok {
+		t.Fatalf("latest_release missing: %#v", ci)
+	}
+	if lr["version"] != "v2.0.0" {
+		t.Errorf("latest_release.version = %v, want v2.0.0", lr["version"])
+	}
+}
+
+func TestWriteManifestState_CustomManifestKey(t *testing.T) {
+	src := `pipeline:
+  config:
+    trunk_branch: main
+    unknown_field: keep-me
+  state:
+    dev:
+      sha: old
+`
+	out, err := WriteManifestState([]byte(src), "pipeline",
+		map[string]*EnvState{"dev": {SHA: "new"}}, nil)
+	if err != nil {
+		t.Fatalf("WriteManifestState: %v", err)
+	}
+	pipeline := parseTop(t, out)["pipeline"].(map[string]any)
+	cfg := pipeline["config"].(map[string]any)
+	if cfg["unknown_field"] != "keep-me" {
+		t.Errorf("unknown_field dropped under custom key: %#v", cfg)
+	}
+	st := pipeline["state"].(map[string]any)
+	if st["dev"].(map[string]any)["sha"] != "new" {
+		t.Errorf("state.dev.sha not updated: %#v", st)
+	}
+}

internal/hotfix/finalize.go

@@ -7,8 +7,6 @@ import (
 	"strings"
 	"time"
 
-	"gopkg.in/yaml.v3"
-
 	"github.com/stablekernel/cascade/internal/config"
 	"github.com/stablekernel/cascade/internal/git"
 	"github.com/stablekernel/cascade/internal/release"
@@ -483,7 +481,7 @@ func (f *Finalizer) Finalize(targetEnv, mergeSHA string, fixSHAs []string, baseS
 			if err := f.applyHotfixState(fresh, capturedTarget, capturedMerge, capturedVersion, capturedBaseSHA, capturedTimestamp, capturedFixSHAs); err != nil {
 				return nil, err
 			}
-			data, err := yaml.Marshal(map[string]any{key: fresh})
+			data, err := config.WriteManifestState(current, key, fresh.State, fresh.LatestRelease)
 			if err != nil {
 				return nil, fmt.Errorf("marshaling merged manifest: %w", err)
 			}
@@ -716,13 +714,16 @@ func (f *Finalizer) isPrereleaseEnv(cfg *config.TrunkConfig, env string) bool {
 	return env == envs[len(envs)-2]
 }
 
-// writeConfig writes the updated manifest back to disk, wrapped in the manifest
-// key, matching the layout promote's finalize produces.
+// writeConfig writes the updated manifest back to disk, rewriting only the
+// mutable state subtree so any configuration this binary does not model is
+// preserved rather than dropped. It matches the layout promote's finalize
+// produces.
 func (f *Finalizer) writeConfig() error {
-	wrapper := map[string]any{
-		f.manifestKey: f.cicd,
+	current, err := os.ReadFile(f.configPath)
+	if err != nil {
+		return fmt.Errorf("reading manifest: %w", err)
 	}
-	data, err := yaml.Marshal(wrapper)
+	data, err := config.WriteManifestState(current, f.manifestKey, f.cicd.State, f.cicd.LatestRelease)
 	if err != nil {
 		return fmt.Errorf("marshaling manifest: %w", err)
 	}

internal/promote/finalize.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/stablekernel/cascade/internal/config"
 	"github.com/stablekernel/cascade/internal/statewrite"
-	"gopkg.in/yaml.v3"
 )
 
 // getEnv returns the value of an environment variable or a default value.
@@ -310,21 +309,21 @@ func (f *Finalizer) updateState() {
 	}
 }
 
-// WriteConfig writes the updated manifest back to disk.
-// The output is wrapped in the manifest key (default: "ci") to match the expected format.
+// WriteConfig writes the updated manifest back to disk. It rewrites only the
+// mutable state subtree of the on-disk manifest, so any configuration this binary
+// does not model is preserved rather than dropped on the round-trip.
 func (f *Finalizer) WriteConfig() error {
 	// Get the manifest key from config (defaults to "ci")
 	key := config.DefaultManifestKey
 	if f.cicdFile.Config != nil && f.cicdFile.Config.ManifestKey != "" {
 		key = f.cicdFile.Config.ManifestKey
 	}
 
-	// Wrap the CICDFile in the manifest key
-	wrapper := map[string]any{
-		key: f.cicdFile,
+	current, err := os.ReadFile(f.configPath)
+	if err != nil {
+		return fmt.Errorf("failed to read config: %w", err)
 	}
-
-	data, err := yaml.Marshal(wrapper)
+	data, err := config.WriteManifestState(current, key, f.cicdFile.State, f.cicdFile.LatestRelease)
 	if err != nil {
 		return fmt.Errorf("failed to marshal config: %w", err)
 	}
@@ -415,7 +414,7 @@ func (f *Finalizer) writeStateViaAPI(message string) error {
 				return nil, fmt.Errorf("parsing current manifest: %w", err)
 			}
 			f.overlayOwnedState(into)
-			data, err := yaml.Marshal(map[string]any{key: into})
+			data, err := config.WriteManifestState(current, key, into.State, into.LatestRelease)
 			if err != nil {
 				return nil, fmt.Errorf("marshaling merged manifest: %w", err)
 			}