|
| 1 | +package engine |
| 2 | + |
| 3 | +import ( |
| 4 | + "encoding/json" |
| 5 | + "strings" |
| 6 | + |
| 7 | + "github.com/GrayCodeAI/hawk/internal/types" |
| 8 | +) |
| 9 | + |
| 10 | +// Prompt-cache segment planning, extending the caveman-style break-even gate |
| 11 | +// into a full client-side planner: the stable prefix (system prompt + tool |
| 12 | +// catalog) is split into cacheable segments, breakpoints are computed at the |
| 13 | +// last tool and last system boundaries, and the whole thing is enabled only |
| 14 | +// when the provider economics pay off for the expected reuse count. Full |
| 15 | +// wire-format lowering and key sharding across a fleet remain eyrie-side; |
| 16 | +// this is the deterministic client planner. |
| 17 | + |
| 18 | +// Anthropic 5m cache economics: uncached = 1.0x, cache write = 1.25x, |
| 19 | +// cache read = 0.1x per reuse. The write premium (0.25x) is paid once. |
| 20 | +const ( |
| 21 | + cacheWritePremium = 0.25 |
| 22 | + cacheReadCost = 0.1 // per-reuse cost of reading a cached segment |
| 23 | + // cacheMinSegmentBytes: segments below this are not worth a cache write. |
| 24 | + cacheMinSegmentBytes = 8 * 1024 |
| 25 | + // cacheDefaultReuse is the expected number of turns reusing the prefix. |
| 26 | + cacheDefaultReuse = 2 |
| 27 | + // cacheMinPrefixBytes is the historical name for the minimum stable-prefix |
| 28 | + // size worth a cache write (alias for cacheMinSegmentBytes). |
| 29 | + cacheMinPrefixBytes = cacheMinSegmentBytes |
| 30 | +) |
| 31 | + |
| 32 | +// CacheSegment is one cacheable unit of the stable prefix. |
| 33 | +type CacheSegment struct { |
| 34 | + Index int `json:"index"` |
| 35 | + Label string `json:"label"` // "system" | "tools" |
| 36 | + Bytes int `json:"bytes"` |
| 37 | +} |
| 38 | + |
| 39 | +// CachePlan is the outcome of cache planning for one call. |
| 40 | +type CachePlan struct { |
| 41 | + Enabled bool `json:"enabled"` |
| 42 | + Provider string `json:"provider"` |
| 43 | + Segments []CacheSegment `json:"segments"` |
| 44 | + Breakpoints int `json:"breakpoints"` // number of cache breakpoints to emit |
| 45 | + ReuseCount int `json:"reuse_count"` |
| 46 | + WriteBytes int `json:"write_bytes"` |
| 47 | + ReadSaving int `json:"read_saving_bytes"` |
| 48 | + UncachedCost int `json:"uncached_cost_bytes"` |
| 49 | + CachedCost int `json:"cached_cost_bytes"` |
| 50 | + Reason string `json:"reason,omitempty"` |
| 51 | +} |
| 52 | + |
| 53 | +// planCache computes whether and how to cache the stable prefix for a call. |
| 54 | +// Deterministic and pure so it can be tested without a provider connection. |
| 55 | +func planCache(provider, systemPrompt string, tools []types.EyrieTool, expectedReuse int) CachePlan { |
| 56 | + if expectedReuse <= 0 { |
| 57 | + expectedReuse = cacheDefaultReuse |
| 58 | + } |
| 59 | + if !strings.EqualFold(provider, "anthropic") { |
| 60 | + return CachePlan{Provider: provider, ReuseCount: expectedReuse, Reason: "no explicit cache controls for provider"} |
| 61 | + } |
| 62 | + |
| 63 | + plan := CachePlan{Provider: "anthropic", ReuseCount: expectedReuse} |
| 64 | + total := 0 |
| 65 | + if systemPrompt != "" { |
| 66 | + plan.Segments = append(plan.Segments, CacheSegment{Index: 0, Label: "system", Bytes: len(systemPrompt)}) |
| 67 | + total += len(systemPrompt) |
| 68 | + } |
| 69 | + if len(tools) > 0 { |
| 70 | + if raw, err := json.Marshal(tools); err == nil && len(raw) > 0 { |
| 71 | + plan.Segments = append(plan.Segments, CacheSegment{Index: len(plan.Segments), Label: "tools", Bytes: len(raw)}) |
| 72 | + total += len(raw) |
| 73 | + } |
| 74 | + } |
| 75 | + plan.WriteBytes = total |
| 76 | + |
| 77 | + if len(plan.Segments) == 0 { |
| 78 | + plan.Reason = "no stable prefix to cache" |
| 79 | + return plan |
| 80 | + } |
| 81 | + |
| 82 | + // A tiny prefix is not worth the write premium (matches the old gate). |
| 83 | + if total < cacheMinSegmentBytes { |
| 84 | + plan.Reason = "prefix below break-even size" |
| 85 | + return plan |
| 86 | + } |
| 87 | + |
| 88 | + // Breakpoints: one at the last system boundary and one at the last tool |
| 89 | + // boundary. We emit breakpoints on every segment so a later prefix reuse |
| 90 | + // hits an early breakpoint (Anthropic caches at the nearest breakpoint |
| 91 | + // before reused content). 2 segments -> 2 breakpoints. |
| 92 | + plan.Breakpoints = len(plan.Segments) |
| 93 | + |
| 94 | + // Economics: break-even reuse R satisfies |
| 95 | + // R*S > 1.25*S + (R-1)*0.1*S => R > 1.25 + 0.1*(R-1) |
| 96 | + // => 0.9*R > 1.15 => R > 1.278 |
| 97 | + // So caching pays off for reuse >= 2. Compute costs for the given reuse. |
| 98 | + plan.UncachedCost = total * expectedReuse |
| 99 | + write := int(float64(total) * (1 + cacheWritePremium)) |
| 100 | + reads := int(float64(total) * cacheReadCost * float64(expectedReuse-1)) |
| 101 | + plan.CachedCost = write + reads |
| 102 | + plan.ReadSaving = plan.UncachedCost - plan.CachedCost |
| 103 | + |
| 104 | + if plan.ReadSaving <= 0 { |
| 105 | + plan.Reason = "caching does not beat uncached at expected reuse" |
| 106 | + return plan |
| 107 | + } |
| 108 | + plan.Enabled = true |
| 109 | + plan.Reason = "break-even satisfied" |
| 110 | + return plan |
| 111 | +} |
| 112 | + |
| 113 | +// cacheDecision reports whether to request provider-native caching, delegating |
| 114 | +// to the planner at the default reuse. Kept for backward compatibility. |
| 115 | +func cacheDecision(provider, systemPrompt string, tools []types.EyrieTool) bool { |
| 116 | + return planCache(provider, systemPrompt, tools, cacheDefaultReuse).Enabled |
| 117 | +} |
0 commit comments