From 2e6c8a947c5329dba220694a3bf997359da6bbeb Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:47:02 +0300 Subject: [PATCH 1/2] Builds: a plain NOR package a newer split one replaced leaves the index The index takes, for each file name, the newest retained build that published it, so a package the firmware stops publishing stays on offer for the whole retention period. hi3516ev300 and hi3518ev300 lite and ultimate ship only the split package since OpenIPC/firmware#2656. Their last plain packages (nightlies of 2026-10-19 to 21) are still in the index, so the wizard gives them fixed slots, and SplitOnlyEdition, which looks for "split published, plain not", never sees them split-only. The stale assets are gone from the latest release; the index did not notice, because it does not read the release. LoadIndex now leaves out a plain NOR package whose split package came from a newer build than the plain one's newest. A board that publishes both does so in one build and keeps both (the gk7205v200 and gk7205v500 families). An edition with no split package (hi3516ev300 neo) and a board whose build failed, publishing neither, keep what they had. Test: TestLoadIndexDropsPlainSupersededBySplit stores two firmware builds, the 21st with the plain ev300 lite/ultimate, neo, gk7205v200 lite and cv300 lite, and the 22nd with the ev300 split pair, neo and gk7205v200 plain plus split. The ev300 plain pair is gone; split, neo, gk7205v200's both, and cv300's from the 21st stay. It fails on the old LoadIndex. service/run.sh test (all packages, Postgres) passes. --- service/internal/builds/builds_test.go | 51 ++++++++++++++++++++++++++ service/internal/builds/source.go | 29 ++++++++++++++- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/service/internal/builds/builds_test.go b/service/internal/builds/builds_test.go index 95a2bcc1..a906d6a4 100644 --- a/service/internal/builds/builds_test.go +++ b/service/internal/builds/builds_test.go @@ -798,3 +798,54 @@ func TestLoadIndexBuilderEditions(t *testing.T) { } } } + +// A board that goes split-only stops publishing its plain NOR package; the +// index must stop offering the last one it kept. A board publishing both, an +// edition with no split package and a board whose build failed keep theirs. +func TestLoadIndexDropsPlainSupersededBySplit(t *testing.T) { + pool := dbtest.New(t) + ctx := context.Background() + d1 := time.Date(2026, 10, 21, 18, 0, 0, 0, time.UTC) + build := func(id string, at time.Time, names ...string) *Payload { + p := &Payload{Schema: 1, Source: "firmware", + Build: Build{ID: id, Release: id, SHA: strings.Repeat("a", 40), BuiltAt: at, PublishedAt: at.Add(time.Hour)}} + for _, n := range names { + p.Assets = append(p.Assets, Asset{Name: n, Size: 7_000_000, SHA256: sum}) + } + return p + } + old := build("nightly-20261021-57ae948", d1, + "openipc.hi3516ev300-nor-lite.tgz", "openipc.hi3516ev300-nor-ultimate.tgz", + "openipc.hi3516ev300-nor-neo.tgz", "openipc.gk7205v200-nor-lite.tgz", + "openipc.hi3516cv300-nor-lite.tgz") + cur := build("nightly-20261022-020f895", d1.Add(24*time.Hour), + "openipc.hi3516ev300-nor-lite-split.tgz", "openipc.hi3516ev300-nor-ultimate-split.tgz", + "openipc.hi3516ev300-nor-neo.tgz", + "openipc.gk7205v200-nor-lite.tgz", "openipc.gk7205v200-nor-lite-split.tgz") + for _, p := range []*Payload{old, cur} { + if _, err := Save(ctx, pool, p, "test"); err != nil { + t.Fatal(err) + } + } + idx, err := LoadIndex(ctx, pool) + if err != nil { + t.Fatal(err) + } + for _, name := range []string{"openipc.hi3516ev300-nor-lite.tgz", "openipc.hi3516ev300-nor-ultimate.tgz"} { + if a, ok := idx.Asset(name); ok { + t.Errorf("%s is still offered, from %s", name, a.Release) + } + } + for name, release := range map[string]string{ + "openipc.hi3516ev300-nor-lite-split.tgz": "nightly-20261022-020f895", + "openipc.hi3516ev300-nor-neo.tgz": "nightly-20261022-020f895", + "openipc.gk7205v200-nor-lite.tgz": "nightly-20261022-020f895", + "openipc.gk7205v200-nor-lite-split.tgz": "nightly-20261022-020f895", + // its build failed on the 22nd: the last good package stays + "openipc.hi3516cv300-nor-lite.tgz": "nightly-20261021-57ae948", + } { + if a, ok := idx.Asset(name); !ok || a.Release != release { + t.Errorf("%s: %+v %v, want it from %s", name, a, ok, release) + } + } +} diff --git a/service/internal/builds/source.go b/service/internal/builds/source.go index a69944d5..f3b60ae6 100644 --- a/service/internal/builds/source.go +++ b/service/internal/builds/source.go @@ -26,7 +26,7 @@ func LoadIndex(ctx context.Context, pool *pgxpool.Pool) (*firmware.Index, error) } rows, err := pool.Query(ctx, ` - SELECT DISTINCT ON (a.name) a.name, a.size, a.sha256, b.release + SELECT DISTINCT ON (a.name) a.name, a.size, a.sha256, b.release, b.built_at FROM build_assets a JOIN builds b ON b.id = a.build_id WHERE b.source IN ('firmware', 'uboot') ORDER BY a.name, b.built_at DESC, b.id DESC`) @@ -34,20 +34,24 @@ func LoadIndex(ctx context.Context, pool *pgxpool.Pool) (*firmware.Index, error) return nil, err } var assets []firmware.Asset + built := map[string]time.Time{} for rows.Next() { var a firmware.Asset var sha string - if err := rows.Scan(&a.Name, &a.Size, &sha, &a.Release); err != nil { + var at time.Time + if err := rows.Scan(&a.Name, &a.Size, &sha, &a.Release, &at); err != nil { rows.Close() return nil, err } a.Digest = "sha256:" + sha assets = append(assets, a) + built[a.Name] = at } rows.Close() if err := rows.Err(); err != nil { return nil, err } + assets = dropSupersededPlain(assets, built) builder, err := loadBuilderEditions(ctx, pool) if err != nil { return nil, err @@ -272,3 +276,24 @@ func (s *Source) listen(ctx context.Context) error { s.reload(ctx, "build "+n.Payload) } } + +// dropSupersededPlain leaves out a plain NOR package whose split package was +// published by a newer build than the plain one's newest. A board that goes +// split-only (hi3516ev300 and hi3518ev300 lite and ultimate) stops +// publishing the plain package, but the index keeps each name's newest +// retained build, so the last plain package would otherwise stay on offer as +// fixed slots for the whole retention period. A board that publishes both +// does so in the same build and keeps both; a board whose build failed +// publishes neither and keeps both as they were. +func dropSupersededPlain(assets []firmware.Asset, built map[string]time.Time) []firmware.Asset { + out := assets[:0] + for _, a := range assets { + if board, storage, edition, ok := ParseAssetName(a.Name); ok && storage == "nor" { + if at, split := built[firmware.SplitName(board, edition)]; split && at.After(built[a.Name]) { + continue + } + } + out = append(out, a) + } + return out +} From 5ac7ccbb77d445b683812ec643bcfe21de95233a Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin <6576495+widgetii@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:59:26 +0300 Subject: [PATCH 2/2] Builds: a split package replaces a plain one by the query's build order From review: two builds with one built_at rank by id in LoadIndex's query; the split check now does the same, comparing (built_at, id). Test: TestLoadIndexSplitSupersedesOnTiedTimestamp, two builds at one time, the later id carrying the split package. service/run.sh test passes. --- service/internal/builds/builds_test.go | 30 ++++++++++++++++++++++++++ service/internal/builds/source.go | 26 ++++++++++++++++------ 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/service/internal/builds/builds_test.go b/service/internal/builds/builds_test.go index a906d6a4..c8797f3f 100644 --- a/service/internal/builds/builds_test.go +++ b/service/internal/builds/builds_test.go @@ -849,3 +849,33 @@ func TestLoadIndexDropsPlainSupersededBySplit(t *testing.T) { } } } + +// Two builds with one timestamp rank by id, in LoadIndex's query and in the +// split check alike: the later id's split package replaces the earlier's +// plain one. +func TestLoadIndexSplitSupersedesOnTiedTimestamp(t *testing.T) { + pool := dbtest.New(t) + ctx := context.Background() + at := time.Date(2026, 10, 22, 18, 0, 0, 0, time.UTC) + for id, name := range map[string]string{ + "nightly-20261022-aaaaaaa": "openipc.hi3518ev300-nor-lite.tgz", + "nightly-20261022-bbbbbbb": "openipc.hi3518ev300-nor-lite-split.tgz", + } { + p := &Payload{Schema: 1, Source: "firmware", + Build: Build{ID: id, Release: id, SHA: strings.Repeat("a", 40), BuiltAt: at, PublishedAt: at.Add(time.Hour)}, + Assets: []Asset{{Name: name, Size: 7_000_000, SHA256: sum}}} + if _, err := Save(ctx, pool, p, "test"); err != nil { + t.Fatal(err) + } + } + idx, err := LoadIndex(ctx, pool) + if err != nil { + t.Fatal(err) + } + if a, ok := idx.Asset("openipc.hi3518ev300-nor-lite.tgz"); ok { + t.Errorf("plain lite still offered, from %s", a.Release) + } + if _, ok := idx.Asset("openipc.hi3518ev300-nor-lite-split.tgz"); !ok { + t.Errorf("split lite missing") + } +} diff --git a/service/internal/builds/source.go b/service/internal/builds/source.go index f3b60ae6..ab3ae3ea 100644 --- a/service/internal/builds/source.go +++ b/service/internal/builds/source.go @@ -26,7 +26,7 @@ func LoadIndex(ctx context.Context, pool *pgxpool.Pool) (*firmware.Index, error) } rows, err := pool.Query(ctx, ` - SELECT DISTINCT ON (a.name) a.name, a.size, a.sha256, b.release, b.built_at + SELECT DISTINCT ON (a.name) a.name, a.size, a.sha256, b.release, b.built_at, b.id FROM build_assets a JOIN builds b ON b.id = a.build_id WHERE b.source IN ('firmware', 'uboot') ORDER BY a.name, b.built_at DESC, b.id DESC`) @@ -34,12 +34,12 @@ func LoadIndex(ctx context.Context, pool *pgxpool.Pool) (*firmware.Index, error) return nil, err } var assets []firmware.Asset - built := map[string]time.Time{} + built := map[string]buildOrder{} for rows.Next() { var a firmware.Asset var sha string - var at time.Time - if err := rows.Scan(&a.Name, &a.Size, &sha, &a.Release, &at); err != nil { + var at buildOrder + if err := rows.Scan(&a.Name, &a.Size, &sha, &a.Release, &at.at, &at.id); err != nil { rows.Close() return nil, err } @@ -285,11 +285,11 @@ func (s *Source) listen(ctx context.Context) error { // fixed slots for the whole retention period. A board that publishes both // does so in the same build and keeps both; a board whose build failed // publishes neither and keeps both as they were. -func dropSupersededPlain(assets []firmware.Asset, built map[string]time.Time) []firmware.Asset { +func dropSupersededPlain(assets []firmware.Asset, built map[string]buildOrder) []firmware.Asset { out := assets[:0] for _, a := range assets { if board, storage, edition, ok := ParseAssetName(a.Name); ok && storage == "nor" { - if at, split := built[firmware.SplitName(board, edition)]; split && at.After(built[a.Name]) { + if at, split := built[firmware.SplitName(board, edition)]; split && at.newer(built[a.Name]) { continue } } @@ -297,3 +297,17 @@ func dropSupersededPlain(assets []firmware.Asset, built map[string]time.Time) [] } return out } + +// buildOrder is where a build stands in the order the index query ranks +// them: built_at, then id when two builds share a timestamp. +type buildOrder struct { + at time.Time + id string +} + +func (b buildOrder) newer(o buildOrder) bool { + if !b.at.Equal(o.at) { + return b.at.After(o.at) + } + return b.id > o.id +}