Skip to content

perf(ladder): read the whole timeline scope from one snapshot #1283

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
IAmJSD merged 2 commits into from
Oct 25, 2026
Merged

perf(ladder): read the whole timeline scope from one snapshot #1283

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fb26345
perf(ladder): read the whole timeline scope from one snapshot
IAmJSD Oct 23, 2026
940813e
Merge remote-tracking branch 'origin/main' into fix/issue-668
IAmJSD Oct 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions internal/config/ladder_timeline.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ import (
"github.com/LeanerCloud/cloud-commitments-go/pkg/ladder"
)

const LadderTimelinePageSize = 100

// LadderTimelineCursor is an optional (created_at, id) lower bound on the rows
// a timeline read returns. Reads never issue one: each returns the whole scope
// from one snapshot (#668), and the bound is only honored for a client that
// still follows a next_cursor from the earlier paginated API.
type LadderTimelineCursor struct {
CreatedAt time.Time `json:"created_at"`
ID string `json:"id"`
Expand All @@ -35,7 +37,6 @@ type LadderTimelineEvent struct {

type LadderTimelinePage struct {
Events []LadderTimelineEvent `json:"events"`
NextCursor *LadderTimelineCursor `json:"next_cursor,omitempty"`
TotalUSDHr string `json:"total_usd_hr"`
TotalODUSDHr string `json:"total_od_usd_hr"`
TotalCount int64 `json:"total_count"`
Expand Down Expand Up @@ -63,9 +64,8 @@ type LadderTimelineAction struct {
}

type LadderTimelineRunPage struct {
Runs []LadderTimelineRun `json:"runs"`
NextCursor *LadderTimelineCursor `json:"next_cursor,omitempty"`
TotalCount int64 `json:"total_count"`
Runs []LadderTimelineRun `json:"runs"`
TotalCount int64 `json:"total_count"`
}

type LadderTimelineStore interface {
Expand Down
46 changes: 20 additions & 26 deletions internal/config/store_postgres_ladder_timeline.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,18 @@ import (
"fmt"
)

// The timeline reads return the whole account scope from ONE statement, so
// the counts, totals and rows all come from a single snapshot: a concurrent
// amendment, run or tranche insert lands either wholly before or wholly after
// the read, never between two pages of it (#668). Every page of the earlier
// paginated read re-aggregated the full scope, so loading n rows cost about
// n/100 statements each scanning n rows; this is one statement scanning n.
//
// $3/$4 is an optional (created_at, id) lower bound kept for a client that
// still follows a next_cursor from before this change: it narrows the rows
// returned, never the totals, and no next cursor is ever issued.
const ladderTimelineQuery = `
WITH scoped AS MATERIALIZED (
WITH scoped AS (
SELECT t.id, t.run_id, t.config_id, t.layer_type, t.term, t.payment_option,
t.status, r.status AS run_status, t.execution_id, t.revision,
t.amount_usd_hr::text AS amount_usd_hr, t.amount_basis, t.commitment_rate,
Expand All @@ -17,18 +27,15 @@ WITH scoped AS MATERIALIZED (
JOIN ladder_runs r ON r.id = t.run_id AND r.config_id = c.id
JOIN cloud_accounts a ON a.id = c.cloud_account_id AND a.provider = c.provider
WHERE c.cloud_account_id = $1 AND c.provider = $2
), page AS (
SELECT * FROM scoped
WHERE $3::timestamptz IS NULL OR (created_at, id) > ($3, $4::uuid)
ORDER BY created_at, id LIMIT $5
)
SELECT COUNT(*), COALESCE(SUM(amount_usd_hr::numeric), 0::numeric(20,6))::text,
COALESCE((SELECT jsonb_agg(to_jsonb(page) ORDER BY created_at, id) FROM page), '[]'::jsonb),
COALESCE(jsonb_agg(to_jsonb(s) ORDER BY created_at, id)
FILTER (WHERE $3::timestamptz IS NULL OR (created_at, id) > ($3, $4::uuid)), '[]'::jsonb),
COALESCE(jsonb_agg(jsonb_build_array(amount_usd_hr, amount_basis, commitment_rate)), '[]'::jsonb)
FROM scoped`
FROM scoped s`

const ladderTimelineRunsQuery = `
WITH scoped AS MATERIALIZED (
WITH scoped AS (
SELECT r.id, r.config_id, r.started_at, r.created_at, r.status,
r.baseline_usd_hr::text AS baseline_usd_hr, r.target_usd_hr::text AS target_usd_hr,
r.existing_usd_hr::text AS existing_usd_hr, r.gap_usd_hr::text AS gap_usd_hr,
Expand All @@ -42,20 +49,17 @@ WITH scoped AS MATERIALIZED (
JOIN ladder_configs c ON c.id = r.config_id
JOIN cloud_accounts a ON a.id = c.cloud_account_id AND a.provider = c.provider
WHERE c.cloud_account_id = $1 AND c.provider = $2
), page AS (
SELECT * FROM scoped
WHERE $3::timestamptz IS NULL OR (created_at, id) > ($3, $4::uuid)
ORDER BY created_at, id LIMIT $5
)
SELECT COUNT(*),
COALESCE((SELECT jsonb_agg(to_jsonb(page) ORDER BY created_at, id) FROM page), '[]'::jsonb)
FROM scoped`
COALESCE(jsonb_agg(to_jsonb(s) ORDER BY created_at, id)
FILTER (WHERE $3::timestamptz IS NULL OR (created_at, id) > ($3, $4::uuid)), '[]'::jsonb)
FROM scoped s`

func ladderTimelineArgs(accountID, provider string, cursor *LadderTimelineCursor) []any {
if cursor == nil {
return []any{accountID, provider, nil, nil, LadderTimelinePageSize + 1}
return []any{accountID, provider, nil, nil}
}
return []any{accountID, provider, cursor.CreatedAt, cursor.ID, LadderTimelinePageSize + 1}
return []any{accountID, provider, cursor.CreatedAt, cursor.ID}
}

func (s *PostgresStore) ListLadderTimeline(ctx context.Context, accountID, provider string, cursor *LadderTimelineCursor) (*LadderTimelinePage, error) {
Expand All @@ -71,11 +75,6 @@ func (s *PostgresStore) ListLadderTimeline(ctx context.Context, accountID, provi
if err := fillLadderTimelineOnDemand(page, scopeRaw); err != nil {
return nil, err
}
if len(page.Events) > LadderTimelinePageSize {
page.Events = page.Events[:LadderTimelinePageSize]
last := page.Events[len(page.Events)-1]
page.NextCursor = &LadderTimelineCursor{CreatedAt: last.CreatedAt, ID: last.ID}
}
return page, nil
}

Expand All @@ -89,10 +88,5 @@ func (s *PostgresStore) ListLadderTimelineRuns(ctx context.Context, accountID, p
if err := json.Unmarshal(raw, &page.Runs); err != nil {
return nil, fmt.Errorf("decode ladder runs: %w", err)
}
if len(page.Runs) > LadderTimelinePageSize {
page.Runs = page.Runs[:LadderTimelinePageSize]
last := page.Runs[len(page.Runs)-1]
page.NextCursor = &LadderTimelineCursor{CreatedAt: last.CreatedAt, ID: last.ID}
}
return page, nil
}
25 changes: 13 additions & 12 deletions internal/config/store_postgres_ladder_timeline_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ func TestLadderTimelineScopedPagesAndSnapshots(t *testing.T) {
Plan: []byte(`{"secret":"never-serialize-this","actions":[{"action":"Hold","layer":"compute-sp","rationale":"Baseline already covered","data_sources":["aws-ce"],"secret":"never-serialize-this"}]}`),
})
require.NoError(t, err)
tranches := make([]LadderTrancheDB, LadderTimelinePageSize+1)
tranches := make([]LadderTrancheDB, 101)
for i := range tranches {
tranches[i] = LadderTrancheDB{
ID: uuid.NewString(), ConfigID: &cfgID, RunID: &run.ID,
Expand All @@ -37,16 +37,18 @@ func TestLadderTimelineScopedPagesAndSnapshots(t *testing.T) {
require.NoError(t, store.SaveLadderTranches(ctx, tranches))
first, err := store.ListLadderTimeline(ctx, accountID, "aws", nil)
require.NoError(t, err)
require.Len(t, first.Events, 100)
require.Len(t, first.Events, 101, "one read returns the whole scope")
require.Equal(t, int64(101), first.TotalCount)
require.Equal(t, "12.469056", first.TotalUSDHr)
require.NotNil(t, first.NextCursor)
second, err := store.ListLadderTimeline(ctx, accountID, "aws", first.NextCursor)
// A cursor from the earlier paginated API narrows the rows, never the totals.
legacy, err := store.ListLadderTimeline(ctx, accountID, "aws", &LadderTimelineCursor{CreatedAt: first.Events[99].CreatedAt, ID: first.Events[99].ID})
require.NoError(t, err)
require.Len(t, second.Events, 1)
require.Nil(t, second.NextCursor)
require.Len(t, legacy.Events, 1)
require.Equal(t, first.Events[100].ID, legacy.Events[0].ID)
require.Equal(t, int64(101), legacy.TotalCount)
require.Equal(t, "12.469056", legacy.TotalUSDHr)
ids := make(map[string]bool)
for _, event := range append(first.Events, second.Events...) {
for _, event := range first.Events {
require.False(t, ids[event.ID], "cursor must never duplicate an event")
ids[event.ID] = true
require.Zero(t, event.Revision, "untouched tranches report revision 0")
Expand Down Expand Up @@ -82,13 +84,12 @@ func TestLadderTimelineScopedPagesAndSnapshots(t *testing.T) {
runs, err = store.ListLadderTimelineRuns(ctx, accountID, "aws", nil)
require.NoError(t, err)
require.Equal(t, int64(101), runs.TotalCount)
require.Len(t, runs.Runs, 100)
lastRunPage, err := store.ListLadderTimelineRuns(ctx, accountID, "aws", runs.NextCursor)
require.Len(t, runs.Runs, 101)
lastRunPage, err := store.ListLadderTimelineRuns(ctx, accountID, "aws", &LadderTimelineCursor{CreatedAt: runs.Runs[99].CreatedAt, ID: runs.Runs[99].ID})
require.NoError(t, err)
require.Len(t, lastRunPage.Runs, 1)
for _, item := range runs.Runs {
require.NotEqual(t, lastRunPage.Runs[0].ID, item.ID)
}
require.Equal(t, runs.Runs[100].ID, lastRunPage.Runs[0].ID)
require.Equal(t, int64(101), lastRunPage.TotalCount)
otherConfig := seedLadderConfigWithExtID(ctx, t, store, "222222222222")
otherRun, err := store.SaveLadderRun(ctx, &LadderRunDB{ConfigID: &otherConfig, Status: ladder.RunStatusPlanned, StartedAt: time.Now().UTC(), Plan: []byte(`{}`)})
require.NoError(t, err)
Expand Down
Loading
Loading