diff --git a/cmd/bucket-lifecycle-audit.go b/cmd/bucket-lifecycle-audit.go index 1fa76e090aeb2..5f8c6fafec0e9 100644 --- a/cmd/bucket-lifecycle-audit.go +++ b/cmd/bucket-lifecycle-audit.go @@ -39,6 +39,7 @@ const ( lcEventSrc_s3PutObject lcEventSrc_s3CopyObject lcEventSrc_s3CompleteMultipartUpload + lcEventSrc_AccessTier ) //revive:enable:var-naming diff --git a/cmd/config-current.go b/cmd/config-current.go index 9e17c0d78c9df..80abfb84be6db 100644 --- a/cmd/config-current.go +++ b/cmd/config-current.go @@ -227,7 +227,7 @@ func initHelp() { }, config.HelpKV{ Key: config.ILMSubSys, - Description: "manage ILM settings for expiration and transition workers", + Description: "manage ILM settings for expiration, transition, and access-tier workers", Optional: true, }, } @@ -704,6 +704,9 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf if globalExpiryState != nil { globalExpiryState.ResizeWorkers(ilmCfg.ExpirationWorkers) } + if globalAccessTierState != nil { + globalAccessTierState.UpdateWorkers(ilmCfg.AccessWorkers) + } globalILMConfig.update(ilmCfg) } } diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go index 288f4d7162c4a..e2182e6c46023 100644 --- a/cmd/data-scanner.go +++ b/cmd/data-scanner.go @@ -899,6 +899,7 @@ type scannerItem struct { objectName string // Only the object name without prefixes. replication replicationConfig lifeCycle *lifecycle.Lifecycle + poolIdx int Typ fs.FileMode heal struct { enabled bool @@ -909,6 +910,7 @@ type scannerItem struct { type sizeSummary struct { totalSize int64 + hotTierSize int64 versions uint64 deleteMarkers uint64 replicatedSize int64 @@ -1159,6 +1161,14 @@ eventLoop: globalExpiryState.enqueueNoncurrentVersions(i.bucket, toDel, noncurrentEvents) } i.alertExcessiveVersions(remainingVersions, cumulativeSize) + if globalILMConfig.accessTieringEnabled() { + for idx, oi := range objInfos { + if oi.IsLatest && events[idx].Action == lifecycle.NoneAction { + applyAccessTransition(ctx, i, oi) + break + } + } + } } func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, obj ObjectInfo) lifecycle.Event { @@ -1474,6 +1484,8 @@ const ( ILMFreeVersionDelete = "ilm:free-version-delete" // ILMTransition - audit trail for ILM transitioning. ILMTransition = " ilm:transition" + // ILMAccessTier - audit trail for moving objects between server pools. + ILMAccessTier = "ilm:access-tier" ) func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]string, traceFn func(event string, metadata map[string]string, err error)) { @@ -1485,6 +1497,8 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags ma apiName = "ILMFreeVersionDelete" case ILMTransition: apiName = "ILMTransition" + case ILMAccessTier: + apiName = "ILMAccessTier" } auditLogInternal(ctx, AuditLogOptions{ Event: event, diff --git a/cmd/data-usage-cache.go b/cmd/data-usage-cache.go index 5752daae057f7..8d1e78aba91ad 100644 --- a/cmd/data-usage-cache.go +++ b/cmd/data-usage-cache.go @@ -61,6 +61,7 @@ type dataUsageEntry struct { Children dataUsageHashMap `msg:"ch"` // These fields do no include any children. Size int64 `msg:"sz"` + HotTierSize int64 `msg:"hts"` Objects uint64 `msg:"os"` Versions uint64 `msg:"vs"` // Versions that are not delete markers. DeleteMarkers uint64 `msg:"dms"` @@ -134,8 +135,8 @@ func (ts tierStats) add(u tierStats) tierStats { } } -//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7 -//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7 +//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7 dataUsageEntryV8 +//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7 dataUsageEntryV8 //msgp:tuple dataUsageEntryV2 type dataUsageEntryV2 struct { @@ -199,14 +200,30 @@ type dataUsageEntryV7 struct { Compacted bool `msg:"c"` } +// dataUsageEntryV8 is the on-disk shape before access-tier accounting was +// introduced. Keep it so caches written by the previous release decode +// without being discarded. +type dataUsageEntryV8 struct { + Children dataUsageHashMap `msg:"ch"` + // These fields do no include any children. + Size int64 `msg:"sz"` + Objects uint64 `msg:"os"` + Versions uint64 `msg:"vs"` + DeleteMarkers uint64 `msg:"dms"` + ObjSizes sizeHistogram `msg:"szs"` + ObjVersions versionsHistogram `msg:"vh"` + AllTierStats *allTierStats `msg:"ats,omitempty"` + Compacted bool `msg:"c"` +} + // dataUsageCache contains a cache of data usage entries latest version. type dataUsageCache struct { Info dataUsageCacheInfo Cache map[string]dataUsageEntry } -//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7 -//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7 +//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7 dataUsageCacheV8 +//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7 dataUsageCacheV8 // dataUsageCacheV2 contains a cache of data usage entries version 2. type dataUsageCacheV2 struct { @@ -244,6 +261,12 @@ type dataUsageCacheV7 struct { Cache map[string]dataUsageEntryV7 } +// dataUsageCacheV8 contains a cache of data usage entries version 8. +type dataUsageCacheV8 struct { + Info dataUsageCacheInfo + Cache map[string]dataUsageEntryV8 +} + //msgp:ignore dataUsageEntryInfo type dataUsageEntryInfo struct { Name string @@ -272,6 +295,7 @@ type dataUsageCacheInfo struct { func (e *dataUsageEntry) addSizes(summary sizeSummary) { e.Size += summary.totalSize + e.HotTierSize += summary.hotTierSize e.Versions += summary.versions e.DeleteMarkers += summary.deleteMarkers e.ObjSizes.add(summary.totalSize) @@ -291,6 +315,7 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) { e.Versions += other.Versions e.DeleteMarkers += other.DeleteMarkers e.Size += other.Size + e.HotTierSize += other.HotTierSize for i, v := range other.ObjSizes[:] { e.ObjSizes[i] += v @@ -431,6 +456,7 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo { flat := d.flatten(*e) dui := DataUsageInfo{ LastUpdate: d.Info.LastUpdate, + ScannerCycle: d.Info.NextCycle, ObjectsTotalCount: flat.Objects, VersionsTotalCount: flat.Versions, DeleteMarkersTotalCount: flat.DeleteMarkers, @@ -781,6 +807,7 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke flat := d.flatten(*e) bui := BucketUsageInfo{ Size: uint64(flat.Size), + HotTierSize: uint64(max(flat.HotTierSize, 0)), VersionsCount: flat.Versions, ObjectsCount: flat.Objects, DeleteMarkersCount: flat.DeleteMarkers, @@ -980,7 +1007,8 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) // Bumping the cache version will drop data from previous versions // and write new data with the new version. const ( - dataUsageCacheVerCurrent = 8 + dataUsageCacheVerCurrent = 9 + dataUsageCacheVerV8 = 8 dataUsageCacheVerV7 = 7 dataUsageCacheVerV6 = 6 dataUsageCacheVerV5 = 5 @@ -1181,6 +1209,33 @@ func (d *dataUsageCache) deserialize(r io.Reader) error { } } + return nil + case dataUsageCacheVerV8: + // Zstd compressed. + dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2)) + if err != nil { + return err + } + defer dec.Close() + dold := &dataUsageCacheV8{} + if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil { + return err + } + d.Info = dold.Info + d.Cache = make(map[string]dataUsageEntry, len(dold.Cache)) + for k, v := range dold.Cache { + d.Cache[k] = dataUsageEntry{ + Children: v.Children, + Size: v.Size, + Objects: v.Objects, + Versions: v.Versions, + DeleteMarkers: v.DeleteMarkers, + ObjSizes: v.ObjSizes, + ObjVersions: v.ObjVersions, + AllTierStats: v.AllTierStats, + Compacted: v.Compacted, + } + } return nil case dataUsageCacheVerCurrent: // Zstd compressed. diff --git a/cmd/data-usage-cache_gen.go b/cmd/data-usage-cache_gen.go index df9d34a164a7e..369e073dd6811 100644 --- a/cmd/data-usage-cache_gen.go +++ b/cmd/data-usage-cache_gen.go @@ -1591,6 +1591,145 @@ func (z *dataUsageCacheV7) Msgsize() (s int) { return } +// DecodeMsg implements msgp.Decodable +func (z *dataUsageCacheV8) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + err = z.Info.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV8, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + var za0002 dataUsageEntryV8 + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageCacheV8) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + bts, err = z.Info.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV8, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + var za0002 dataUsageEntryV8 + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageCacheV8) Msgsize() (s int) { + s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize + if z.Cache != nil { + for za0001, za0002 := range z.Cache { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + return +} + // DecodeMsg implements msgp.Decodable func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte @@ -1623,6 +1762,12 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) { err = msgp.WrapError(err, "Size") return } + case "hts": + z.HotTierSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "HotTierSize") + return + } case "os": z.Objects, err = dc.ReadUint64() if err != nil { @@ -1801,12 +1946,12 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { // check for omitted fields - zb0001Len := uint32(9) - var zb0001Mask uint16 /* 9 bits */ + zb0001Len := uint32(10) + var zb0001Mask uint16 /* 10 bits */ _ = zb0001Mask if z.AllTierStats == nil { zb0001Len-- - zb0001Mask |= 0x80 + zb0001Mask |= 0x100 } // variable map header, size zb0001Len err = en.Append(0x80 | uint8(zb0001Len)) @@ -1836,6 +1981,16 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { err = msgp.WrapError(err, "Size") return } + // write "hts" + err = en.Append(0xa3, 0x68, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteInt64(z.HotTierSize) + if err != nil { + err = msgp.WrapError(err, "HotTierSize") + return + } // write "os" err = en.Append(0xa2, 0x6f, 0x73) if err != nil { @@ -1900,7 +2055,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { return } } - if (zb0001Mask & 0x80) == 0 { // if not omitted + if (zb0001Mask & 0x100) == 0 { // if not omitted // write "ats" err = en.Append(0xa3, 0x61, 0x74, 0x73) if err != nil { @@ -1981,12 +2136,12 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) { o = msgp.Require(b, z.Msgsize()) // check for omitted fields - zb0001Len := uint32(9) - var zb0001Mask uint16 /* 9 bits */ + zb0001Len := uint32(10) + var zb0001Mask uint16 /* 10 bits */ _ = zb0001Mask if z.AllTierStats == nil { zb0001Len-- - zb0001Mask |= 0x80 + zb0001Mask |= 0x100 } // variable map header, size zb0001Len o = append(o, 0x80|uint8(zb0001Len)) @@ -2003,6 +2158,9 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) { // string "sz" o = append(o, 0xa2, 0x73, 0x7a) o = msgp.AppendInt64(o, z.Size) + // string "hts" + o = append(o, 0xa3, 0x68, 0x74, 0x73) + o = msgp.AppendInt64(o, z.HotTierSize) // string "os" o = append(o, 0xa2, 0x6f, 0x73) o = msgp.AppendUint64(o, z.Objects) @@ -2024,7 +2182,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) { for za0002 := range z.ObjVersions { o = msgp.AppendUint64(o, z.ObjVersions[za0002]) } - if (zb0001Mask & 0x80) == 0 { // if not omitted + if (zb0001Mask & 0x100) == 0 { // if not omitted // string "ats" o = append(o, 0xa3, 0x61, 0x74, 0x73) if z.AllTierStats == nil { @@ -2088,6 +2246,12 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) { err = msgp.WrapError(err, "Size") return } + case "hts": + z.HotTierSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "HotTierSize") + return + } case "os": z.Objects, bts, err = msgp.ReadUint64Bytes(bts) if err != nil { @@ -2265,7 +2429,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) { // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message func (z *dataUsageEntry) Msgsize() (s int) { - s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4 + s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4 if z.AllTierStats == nil { s += msgp.NilSize } else { @@ -3258,6 +3422,438 @@ func (z *dataUsageEntryV7) Msgsize() (s int) { return } +// DecodeMsg implements msgp.Decodable +func (z *dataUsageEntryV8) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ch": + err = z.Children.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + case "sz": + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "os": + z.Objects, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + case "vs": + z.Versions, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + case "dms": + z.DeleteMarkers, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "DeleteMarkers") + return + } + case "szs": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + case "vh": + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ObjVersions") + return + } + if zb0003 != uint32(dataUsageVersionLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003} + return + } + for za0002 := range z.ObjVersions { + z.ObjVersions[za0002], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ObjVersions", za0002) + return + } + } + case "ats": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + z.AllTierStats = nil + } else { + if z.AllTierStats == nil { + z.AllTierStats = new(allTierStats) + } + var zb0004 uint32 + zb0004, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + for zb0004 > 0 { + zb0004-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + switch msgp.UnsafeString(field) { + case "ts": + var zb0005 uint32 + zb0005, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers") + return + } + if z.AllTierStats.Tiers == nil { + z.AllTierStats.Tiers = make(map[string]tierStats, zb0005) + } else if len(z.AllTierStats.Tiers) > 0 { + clear(z.AllTierStats.Tiers) + } + for zb0005 > 0 { + zb0005-- + var za0003 string + za0003, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers") + return + } + var za0004 tierStats + var zb0006 uint32 + zb0006, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + for zb0006 > 0 { + zb0006-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + switch msgp.UnsafeString(field) { + case "ts": + za0004.TotalSize, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize") + return + } + case "nv": + za0004.NumVersions, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions") + return + } + case "no": + za0004.NumObjects, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + } + } + z.AllTierStats.Tiers[za0003] = za0004 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + } + } + } + zb0001Mask |= 0x1 + case "c": + z.Compacted, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Compacted") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.AllTierStats = nil + } + + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageEntryV8) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ch": + bts, err = z.Children.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + case "sz": + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "os": + z.Objects, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + case "vs": + z.Versions, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + case "dms": + z.DeleteMarkers, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "DeleteMarkers") + return + } + case "szs": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + case "vh": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjVersions") + return + } + if zb0003 != uint32(dataUsageVersionLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003} + return + } + for za0002 := range z.ObjVersions { + z.ObjVersions[za0002], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjVersions", za0002) + return + } + } + case "ats": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.AllTierStats = nil + } else { + if z.AllTierStats == nil { + z.AllTierStats = new(allTierStats) + } + var zb0004 uint32 + zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + for zb0004 > 0 { + zb0004-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + switch msgp.UnsafeString(field) { + case "ts": + var zb0005 uint32 + zb0005, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers") + return + } + if z.AllTierStats.Tiers == nil { + z.AllTierStats.Tiers = make(map[string]tierStats, zb0005) + } else if len(z.AllTierStats.Tiers) > 0 { + clear(z.AllTierStats.Tiers) + } + for zb0005 > 0 { + var za0004 tierStats + zb0005-- + var za0003 string + za0003, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers") + return + } + var zb0006 uint32 + zb0006, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + for zb0006 > 0 { + zb0006-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + switch msgp.UnsafeString(field) { + case "ts": + za0004.TotalSize, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize") + return + } + case "nv": + za0004.NumVersions, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions") + return + } + case "no": + za0004.NumObjects, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + } + } + z.AllTierStats.Tiers[za0003] = za0004 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + } + } + } + zb0001Mask |= 0x1 + case "c": + z.Compacted, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Compacted") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.AllTierStats = nil + } + + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageEntryV8) Msgsize() (s int) { + s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4 + if z.AllTierStats == nil { + s += msgp.NilSize + } else { + s += 1 + 3 + msgp.MapHeaderSize + if z.AllTierStats.Tiers != nil { + for za0003, za0004 := range z.AllTierStats.Tiers { + _ = za0004 + s += msgp.StringPrefixSize + len(za0003) + 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize + } + } + } + s += 2 + msgp.BoolSize + return +} + // DecodeMsg implements msgp.Decodable func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) { { diff --git a/cmd/data-usage-utils.go b/cmd/data-usage-utils.go index 1f6b3f11e2487..4bdb4869c1b69 100644 --- a/cmd/data-usage-utils.go +++ b/cmd/data-usage-utils.go @@ -46,7 +46,8 @@ type BucketTargetUsageInfo struct { // - total objects in a bucket // - object size histogram per bucket type BucketUsageInfo struct { - Size uint64 `json:"size"` + Size uint64 `json:"size"` + HotTierSize uint64 `json:"hotTierSize,omitempty"` // Following five fields suffixed with V1 are here for backward compatibility // Total Size for objects that have not yet been replicated ReplicationPendingSizeV1 uint64 `json:"objectsPendingReplicationTotalSize"` @@ -78,6 +79,10 @@ type DataUsageInfo struct { // LastUpdate is the timestamp of when the data usage info was last updated. // This does not indicate a full scan. LastUpdate time.Time `json:"lastUpdate"` + // ScannerCycle changes only after a complete scanner pass. Background + // consumers use it to distinguish a partial cache update from a baseline + // that has visited every bucket and server pool. + ScannerCycle uint32 `json:"scannerCycle,omitempty"` // Objects total count across all buckets ObjectsTotalCount uint64 `json:"objectsCount"` diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index dc65d06eed038..28cc5d2780373 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -615,7 +615,17 @@ func (z *erasureServerPools) getPoolIdxExistingNoLock(ctx context.Context, bucke }) } -func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, object string, size int64) (idx int, err error) { +func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, object string, size int64, dstPoolIdx *int) (idx int, err error) { + if dstPoolIdx != nil { + if *dstPoolIdx < 0 || *dstPoolIdx >= len(z.serverPools) { + return -1, errInvalidArgument + } + if z.IsSuspended(*dstPoolIdx) || z.IsPoolRebalancing(*dstPoolIdx) { + return -1, toObjectErr(errDiskFull) + } + return *dstPoolIdx, nil + } + idx, err = z.getPoolIdxExistingNoLock(ctx, bucket, object) if err != nil && !isErrObjectNotFound(err) { return idx, err @@ -634,7 +644,17 @@ func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, objec // getPoolIdx returns the found previous object and its corresponding pool idx, // if none are found falls back to most available space pool, this function is // designed to be only used by PutObject, CopyObject (newObject creation) and NewMultipartUpload. -func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) { +func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64, dstPoolIdx *int) (idx int, err error) { + if dstPoolIdx != nil { + if *dstPoolIdx < 0 || *dstPoolIdx >= len(z.serverPools) { + return -1, errInvalidArgument + } + if z.IsSuspended(*dstPoolIdx) || z.IsPoolRebalancing(*dstPoolIdx) { + return -1, toObjectErr(errDiskFull) + } + return *dstPoolIdx, nil + } + pinfo, _, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, ObjectOptions{ SkipDecommissioned: true, SkipRebalancing: true, @@ -656,6 +676,13 @@ func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object stri return idx, nil } +func dataMovementDstPool(opts ObjectOptions) *int { + if !opts.DataMovement { + return nil + } + return opts.DstPoolIdx +} + func (z *erasureServerPools) Shutdown(ctx context.Context) error { g := errgroup.WithNErrs(len(z.serverPools)) @@ -1090,14 +1117,20 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec object = encodeDirObject(object) if z.SinglePool() { - _, err := z.getPoolIdx(ctx, bucket, object, data.Size()) + idx, err := z.getPoolIdx(ctx, bucket, object, data.Size(), dataMovementDstPool(opts)) if err != nil { return ObjectInfo{}, err } + if dataMovementDstPool(opts) != nil && idx == opts.SrcPoolIdx { + return ObjectInfo{}, DataMovementOverwriteErr{ + Bucket: bucket, Object: object, VersionID: opts.VersionID, + Err: errDataMovementSrcDstPoolSame, + } + } return z.serverPools[0].PutObject(ctx, bucket, object, data, opts) } - idx, err := z.getPoolIdx(ctx, bucket, object, data.Size()) + idx, err := z.getPoolIdx(ctx, bucket, object, data.Size(), dataMovementDstPool(opts)) if err != nil { return ObjectInfo{}, err } @@ -1145,6 +1178,28 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob return ObjectInfo{}, z.deletePrefix(ctx, bucket, object) } + // Access-tier moves must recreate delete markers on the explicitly + // selected destination. The regular data-movement path discovers a pool + // from existing object state, which is ambiguous while both source and + // destination temporarily contain the version stack. + if dstPoolIdx := dataMovementDstPool(opts); dstPoolIdx != nil { + if *dstPoolIdx < 0 || *dstPoolIdx >= len(z.serverPools) { + return ObjectInfo{}, errInvalidArgument + } + if *dstPoolIdx == opts.SrcPoolIdx { + return ObjectInfo{}, DataMovementOverwriteErr{ + Bucket: bucket, Object: decodeDirObject(object), VersionID: opts.VersionID, + Err: errDataMovementSrcDstPoolSame, + } + } + if z.IsSuspended(*dstPoolIdx) || z.IsPoolRebalancing(*dstPoolIdx) { + return ObjectInfo{}, toObjectErr(errDiskFull) + } + objInfo, err = z.serverPools[*dstPoolIdx].DeleteObject(ctx, bucket, object, opts) + objInfo.Name = decodeDirObject(object) + return objInfo, err + } + gopts := opts gopts.NoLock = true @@ -1323,10 +1378,16 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec dstOpts.NoLock = true } - poolIdx, err := z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size) + poolIdx, err := z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size, dataMovementDstPool(dstOpts)) if err != nil { return objInfo, err } + if dataMovementDstPool(dstOpts) != nil && poolIdx == dstOpts.SrcPoolIdx { + return ObjectInfo{}, DataMovementOverwriteErr{ + Bucket: dstBucket, Object: dstObject, VersionID: dstOpts.VersionID, + Err: errDataMovementSrcDstPoolSame, + } + } if cpSrcDstSame && srcInfo.metadataOnly { // Version ID is set for the destination and source == destination version ID. @@ -1771,29 +1832,41 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj }() if z.SinglePool() { + idx, err := z.getPoolIdx(ctx, bucket, object, -1, dataMovementDstPool(opts)) + if err != nil { + return nil, err + } + if dataMovementDstPool(opts) != nil && idx == opts.SrcPoolIdx { + return nil, DataMovementOverwriteErr{ + Bucket: bucket, Object: object, VersionID: opts.VersionID, + Err: errDataMovementSrcDstPoolSame, + } + } return z.serverPools[0].NewMultipartUpload(ctx, bucket, object, opts) } - for idx, pool := range z.serverPools { - if z.IsSuspended(idx) || z.IsPoolRebalancing(idx) { - continue - } + if dataMovementDstPool(opts) == nil { + for idx, pool := range z.serverPools { + if z.IsSuspended(idx) || z.IsPoolRebalancing(idx) { + continue + } - result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList) - if err != nil { - return nil, err - } - // If there is a multipart upload with the same bucket/object name, - // create the new multipart in the same pool, this will avoid - // creating two multiparts uploads in two different pools - if len(result.Uploads) != 0 { - return z.serverPools[idx].NewMultipartUpload(ctx, bucket, object, opts) + result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList) + if err != nil { + return nil, err + } + // If there is a multipart upload with the same bucket/object name, + // create the new multipart in the same pool, this will avoid + // creating two multiparts uploads in two different pools. + if len(result.Uploads) != 0 { + return z.serverPools[idx].NewMultipartUpload(ctx, bucket, object, opts) + } } } // any parallel writes on the object will block for this poolIdx // to return since this holds a read lock on the namespace. - idx, err := z.getPoolIdx(ctx, bucket, object, -1) + idx, err := z.getPoolIdx(ctx, bucket, object, -1, dataMovementDstPool(opts)) if err != nil { return nil, err } @@ -1827,7 +1900,7 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object, } if z.SinglePool() { - _, err := z.getPoolIdx(ctx, bucket, object, data.Size()) + _, err := z.getPoolIdx(ctx, bucket, object, data.Size(), dataMovementDstPool(opts)) if err != nil { return PartInfo{}, err } @@ -2987,7 +3060,7 @@ func (z *erasureServerPools) DecomTieredObject(ctx context.Context, bucket, obje defer ns.Unlock(lkctx) opts.NoLock = true } - idx, err := z.getPoolIdxNoLock(ctx, bucket, object, fi.Size) + idx, err := z.getPoolIdxNoLock(ctx, bucket, object, fi.Size, dataMovementDstPool(opts)) if err != nil { return err } diff --git a/cmd/ilm-access-tier.go b/cmd/ilm-access-tier.go new file mode 100644 index 0000000000000..28007b1411f48 --- /dev/null +++ b/cmd/ilm-access-tier.go @@ -0,0 +1,1123 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "maps" + "net/http" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/config/ilm" + "github.com/minio/minio/internal/hash" +) + +const ( + accessTierMetadataKey = ReservedMetadataPrefixLower + "ilm-atier" + accessTierQueueSize = 100000 +) + +var ( + errAccessTierNotEligible = errors.New("object is no longer eligible for access tiering") + errAccessTierTooManyVersion = errors.New("object has too many versions for access tiering") + errAccessTierRemoteVersion = errors.New("access tiering does not move remotely transitioned versions") +) + +type accessTierDirection uint8 + +const ( + accessTierPromote accessTierDirection = iota + 1 + accessTierDemote +) + +type accessTierTask struct { + ctx context.Context + bucket string + object string + src int + dst int + direction accessTierDirection + bytes uint64 // promotion reservation; zero for demotion +} + +// accessTierState owns the leader-only mover queues and the usage deltas made +// since the data scanner's last published snapshot. +type accessTierState struct { + ctx context.Context + objAPI ObjectLayer + z *erasureServerPools + + promoteCh chan accessTierTask + demoteCh chan accessTierTask + + mu sync.Mutex + numWorkers int + workerStops []chan struct{} + pending map[string]struct{} + baseUsage map[string]uint64 + baseTotal uint64 + delta map[string]uint64 + deltaCycle uint32 + deltaCycleValid bool + reserved map[string]uint64 + usageAt time.Time + usageCycle uint32 + usageCycleSeen bool + usageReady bool + + active atomic.Int64 + promotions atomic.Uint64 + demotions atomic.Uint64 + bytesMoved atomic.Uint64 + failures atomic.Uint64 + skippedWatermark atomic.Uint64 + skippedMaxSize atomic.Uint64 + skippedQuota atomic.Uint64 +} + +var globalAccessTierState *accessTierState + +func newAccessTierState(ctx context.Context) *accessTierState { + return &accessTierState{ + ctx: ctx, + promoteCh: make(chan accessTierTask, accessTierQueueSize), + demoteCh: make(chan accessTierTask, accessTierQueueSize), + pending: make(map[string]struct{}), + baseUsage: make(map[string]uint64), + delta: make(map[string]uint64), + reserved: make(map[string]uint64), + } +} + +func (s *accessTierState) Init(objAPI ObjectLayer) { + if s == nil { + return + } + z, ok := objAPI.(*erasureServerPools) + if !ok { + return + } + + s.mu.Lock() + s.objAPI = objAPI + s.z = z + s.updateWorkersLocked(globalILMConfig.accessCfg().AccessWorkers) + s.mu.Unlock() + + go s.runLeader() +} + +func (s *accessTierState) UpdateWorkers(n int) { + if s == nil || n < 1 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.objAPI == nil { + return + } + s.updateWorkersLocked(n) +} + +func (s *accessTierState) updateWorkersLocked(n int) { + for s.numWorkers < n { + stop := make(chan struct{}) + s.workerStops = append(s.workerStops, stop) + go s.worker(stop) + s.numWorkers++ + } + for s.numWorkers > n { + last := len(s.workerStops) - 1 + close(s.workerStops[last]) + s.workerStops = s.workerStops[:last] + s.numWorkers-- + } +} + +func (s *accessTierState) PendingTasks() int { + if s == nil { + return 0 + } + return len(s.promoteCh) + len(s.demoteCh) +} + +func (s *accessTierState) ActiveTasks() int64 { + if s == nil { + return 0 + } + return s.active.Load() +} + +func accessTierPendingKey(bucket, object string) string { + return accessKey(bucket, object) +} + +func (s *accessTierState) enqueueDemote(task accessTierTask) bool { + key := accessTierPendingKey(task.bucket, task.object) + s.mu.Lock() + if _, exists := s.pending[key]; exists { + s.mu.Unlock() + return false + } + s.pending[key] = struct{}{} + s.mu.Unlock() + + select { + case <-task.ctx.Done(): + s.releasePending(task, false) + return false + case s.demoteCh <- task: + return true + default: + s.releasePending(task, false) + return false + } +} + +func addUsage(a, b uint64) uint64 { + if ^uint64(0)-a < b { + return ^uint64(0) + } + return a + b +} + +func (s *accessTierState) bucketUsageLocked(bucket string) uint64 { + return addUsage(addUsage(s.baseUsage[bucket], s.delta[bucket]), s.reserved[bucket]) +} + +func (s *accessTierState) totalUsageLocked() uint64 { + var delta uint64 + var reserved uint64 + for _, v := range s.delta { + delta = addUsage(delta, v) + } + for _, v := range s.reserved { + reserved = addUsage(reserved, v) + } + return addUsage(addUsage(s.baseTotal, delta), reserved) +} + +// reservePromotion makes all queued promotions participate in quota checks, +// so one sweep cannot admit many objects against the same stale scanner value. +func (s *accessTierState) reservePromotion(task accessTierTask, cfg ilm.Config, quota uint64) (string, bool) { + key := accessTierPendingKey(task.bucket, task.object) + s.mu.Lock() + defer s.mu.Unlock() + if _, exists := s.pending[key]; exists { + return "", false + } + if !s.usageReady { + switch { + case cfg.AccessMaxSize > 0: + return "max-size", false + case quota > 0: + return "quota", false + } + } + if cfg.AccessMaxSize > 0 && addUsage(s.totalUsageLocked(), task.bytes) > cfg.AccessMaxSize { + return "max-size", false + } + if quota > 0 && addUsage(s.bucketUsageLocked(task.bucket), task.bytes) > quota { + return "quota", false + } + s.pending[key] = struct{}{} + s.reserved[task.bucket] = addUsage(s.reserved[task.bucket], task.bytes) + return "", true +} + +func (s *accessTierState) replaceReservation(bucket string, old, current uint64, cfg ilm.Config, quota uint64) (string, bool) { + s.mu.Lock() + defer s.mu.Unlock() + reserved := s.reserved[bucket] + if old >= reserved { + reserved = 0 + } else { + reserved -= old + } + s.reserved[bucket] = reserved + if cfg.AccessMaxSize > 0 && addUsage(s.totalUsageLocked(), current) > cfg.AccessMaxSize { + s.reserved[bucket] = addUsage(s.reserved[bucket], old) + return "max-size", false + } + if quota > 0 && addUsage(s.bucketUsageLocked(bucket), current) > quota { + s.reserved[bucket] = addUsage(s.reserved[bucket], old) + return "quota", false + } + s.reserved[bucket] = addUsage(s.reserved[bucket], current) + return "", true +} + +func (s *accessTierState) enqueuePromote(task accessTierTask, cfg ilm.Config, quota uint64) bool { + reason, ok := s.reservePromotion(task, cfg, quota) + if !ok { + s.recordSkip(reason) + return false + } + select { + case <-task.ctx.Done(): + s.releasePending(task, true) + return false + case s.promoteCh <- task: + return true + default: + s.releasePending(task, true) + return false + } +} + +func (s *accessTierState) releasePending(task accessTierTask, releaseReservation bool) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.pending, accessTierPendingKey(task.bucket, task.object)) + if releaseReservation { + if task.bytes >= s.reserved[task.bucket] { + delete(s.reserved, task.bucket) + } else { + s.reserved[task.bucket] -= task.bytes + } + } +} + +func (s *accessTierState) completeTask(task accessTierTask, moved uint64) { + s.mu.Lock() + delete(s.pending, accessTierPendingKey(task.bucket, task.object)) + if task.direction == accessTierPromote { + if task.bytes >= s.reserved[task.bucket] { + delete(s.reserved, task.bucket) + } else { + s.reserved[task.bucket] -= task.bytes + } + s.delta[task.bucket] = addUsage(s.delta[task.bucket], moved) + if s.usageCycleSeen { + s.deltaCycle = s.usageCycle + s.deltaCycleValid = true + } + } + s.mu.Unlock() + + if task.direction == accessTierPromote { + s.promotions.Add(1) + } else { + s.demotions.Add(1) + } + s.bytesMoved.Add(moved) +} + +func (s *accessTierState) recordSkip(reason string) { + switch reason { + case "watermark": + s.skippedWatermark.Add(1) + case "max-size": + s.skippedMaxSize.Add(1) + case "quota": + s.skippedQuota.Add(1) + } +} + +func (s *accessTierState) nextTask(stop <-chan struct{}) (accessTierTask, bool) { + // Prefer demotion whenever one is already waiting, without starving + // promotions when the demotion queue is empty. + select { + case <-stop: + return accessTierTask{}, false + default: + } + select { + case task := <-s.demoteCh: + return task, true + default: + } + select { + case <-stop: + return accessTierTask{}, false + case <-s.ctx.Done(): + return accessTierTask{}, false + case task := <-s.demoteCh: + return task, true + case task := <-s.promoteCh: + return task, true + } +} + +func (s *accessTierState) worker(stop <-chan struct{}) { + for { + task, ok := s.nextTask(stop) + if !ok { + return + } + s.active.Add(1) + err := s.processTask(task) + s.active.Add(-1) + if err != nil { + if !errors.Is(err, errAccessTierNotEligible) && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) && !errors.Is(err, context.Canceled) { + s.failures.Add(1) + ilmLogIf(task.ctx, fmt.Errorf("access tier move failed for %s/%s: %w", task.bucket, task.object, err)) + } + } + } +} + +func (s *accessTierState) runLeader() { + for s.ctx.Err() == nil { + leaderCtx, cancel := globalLeaderLock.GetLock(s.ctx) + s.runSweeps(leaderCtx) + cancel() + } +} + +func (s *accessTierState) runSweeps(ctx context.Context) { + cfg := globalILMConfig.accessCfg() + interval := cfg.AccessFlush + if interval <= 0 { + interval = time.Minute + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + newCfg := globalILMConfig.accessCfg() + if newCfg.AccessFlush > 0 && newCfg.AccessFlush != interval { + interval = newCfg.AccessFlush + ticker.Reset(interval) + } + cfg = newCfg + if !cfg.AccessTiering { + continue + } + if !validAccessTopology(s.z, cfg) { + ilmLogOnceIf(ctx, fmt.Errorf("access tiering needs at least two existing server pools; configured pools are %v", cfg.AccessPools), "access-tier-topology") + continue + } + if s.z.IsRebalanceStarted(ctx) || s.z.IsDecommissionRunning() { + continue + } + s.refreshUsage(ctx) + s.sweepDemotions(ctx, cfg) + s.sweepPromotions(ctx, cfg) + } + } +} + +func validAccessTopology(z *erasureServerPools, cfg ilm.Config) bool { + if z == nil || len(cfg.AccessPools) < 2 || len(z.serverPools) < 2 { + return false + } + for _, idx := range cfg.AccessPools { + if idx < 0 || idx >= len(z.serverPools) { + return false + } + } + return true +} + +func (s *accessTierState) refreshUsage(ctx context.Context) { + dui, err := loadDataUsageFromBackend(ctx, s.objAPI) + if err != nil || dui.LastUpdate.IsZero() { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if !dui.LastUpdate.After(s.usageAt) { + return + } + cycleAdvanced := s.usageCycleSeen && dui.ScannerCycle != s.usageCycle + s.baseUsage = make(map[string]uint64, len(dui.BucketsUsage)) + s.baseTotal = 0 + for bucket, usage := range dui.BucketsUsage { + // Only retain buckets that actually hold hot-tier bytes. Keeping a + // zero entry per bucket would emit one Prometheus series per bucket + // in the cluster from hotUsageSnapshot. + if usage.HotTierSize == 0 { + continue + } + s.baseUsage[bucket] = usage.HotTierSize + s.baseTotal = addUsage(s.baseTotal, usage.HotTierSize) + } + if !s.usageCycleSeen { + s.usageCycleSeen = true + s.usageCycle = dui.ScannerCycle + if len(s.delta) > 0 && !s.deltaCycleValid { + s.deltaCycle = dui.ScannerCycle + s.deltaCycleValid = true + } + } else if cycleAdvanced { + s.usageCycle = dui.ScannerCycle + // A move can finish while the next scanner pass is already in + // progress. Retain promotion deltas until one additional complete + // pass has elapsed, so quotas can over-count temporarily but never + // under-count. Demotions rely only on the scanner to release space. + if s.deltaCycleValid && scannerCyclesApart(dui.ScannerCycle, s.deltaCycle) >= 2 { + clear(s.delta) + s.deltaCycleValid = false + } + s.usageReady = true + } + s.usageAt = dui.LastUpdate +} + +func scannerCyclesApart(current, previous uint32) uint32 { + distance := current - previous + if distance >= 1<<31 { + return 0 + } + return distance +} + +func (s *accessTierState) hotUsageSnapshot() map[string]uint64 { + if s == nil { + return nil + } + s.mu.Lock() + defer s.mu.Unlock() + out := make(map[string]uint64, len(s.baseUsage)+len(s.delta)+len(s.reserved)) + for _, m := range []map[string]uint64{s.baseUsage, s.delta, s.reserved} { + for bucket := range m { + if used := s.bucketUsageLocked(bucket); used > 0 { + out[bucket] = used + } + } + } + return out +} + +func (s *accessTierState) sweepDemotions(ctx context.Context, cfg ilm.Config) { + for _, candidate := range globalAccessTracker.collectDemoteCandidates(ctx, s.objAPI, cfg) { + if candidate.Bucket == minioMetaBucket { + continue + } + pinfo, _, err := s.z.getPoolInfoExistingWithOpts(ctx, candidate.Bucket, candidate.Object, ObjectOptions{ + SkipDecommissioned: true, + SkipRebalancing: true, + }) + if err != nil || pinfo.Index != candidate.Pool { + continue + } + cold, _ := cfg.ColdPool() + if pinfo.Index == cold || !accessPoolIsHotter(cfg, pinfo.Index, cold) { + continue + } + lc, err := globalLifecycleSys.Get(candidate.Bucket) + if err != nil || !accessDemoteEligible(lc, pinfo.ObjInfo, pinfo.Index, cfg, time.Now()) { + continue + } + s.enqueueDemote(accessTierTask{ + ctx: ctx, bucket: candidate.Bucket, object: candidate.Object, + src: pinfo.Index, dst: cold, direction: accessTierDemote, + }) + } +} + +func (s *accessTierState) sweepPromotions(ctx context.Context, cfg ilm.Config) { + snapshot := globalAccessTracker.snapshot() + if snapshot == nil { + return + } + hot, _ := cfg.HotPool() + lifecycles := make(map[string]*lifecycle.Lifecycle) + missing := make(map[string]struct{}) + for key, entry := range snapshot.entries { + bucket, object, ok := splitAccessKey(key) + if !ok || bucket == minioMetaBucket { + continue + } + lc := lifecycles[bucket] + if lc == nil { + if _, skip := missing[bucket]; skip { + continue + } + var err error + lc, err = globalLifecycleSys.Get(bucket) + if err != nil || !lc.HasAccessTransition() { + missing[bucket] = struct{}{} + continue + } + lifecycles[bucket] = lc + } + + if !accessHitsMightPromote(lc, object, entry, snapshot.binWidth) { + continue + } + + pinfo, _, err := s.z.getPoolInfoExistingWithOpts(ctx, bucket, object, ObjectOptions{ + SkipDecommissioned: true, + SkipRebalancing: true, + }) + if err != nil || pinfo.Index == hot || pinfo.ObjInfo.DeleteMarker || pinfo.ObjInfo.IsDir || pinfo.ObjInfo.IsRemote() { + continue + } + rule, _, ok := lc.AccessRule(pinfo.ObjInfo.ToLifecycleOpts()) + if !ok || entry.hits(rule.Window.D(), snapshot.binWidth) < uint64(rule.PromoteAfterAccesses) { + continue + } + if rule.Window.D() > cfg.HistoryWindow() { + ilmLogOnceIf(ctx, fmt.Errorf("access tier rule window %s for %s/%s exceeds retained history %s and is clamped", rule.Window.D(), bucket, object, cfg.HistoryWindow()), "access-window-"+bucket) + } + bytes, err := accessObjectBytes(ctx, s.z, pinfo.Index, bucket, object) + if err != nil { + continue + } + if !s.promotionWatermarkOK(ctx, cfg, hot, bucket, object, bytes) { + s.recordSkip("watermark") + continue + } + s.enqueuePromote(accessTierTask{ + ctx: ctx, bucket: bucket, object: object, src: pinfo.Index, dst: hot, + direction: accessTierPromote, bytes: bytes, + }, cfg, lc.AccessQuotaBytes()) + } +} + +// accessHitsMightPromote is the cheap prefilter for sweepPromotions: prefix +// plus hit count, without a quorum metadata read. Size and tag filters are +// applied later via AccessRule once object info is in hand. +func accessHitsMightPromote(lc *lifecycle.Lifecycle, object string, entry accessEntry, binWidth int64) bool { + if lc == nil { + return false + } + for _, rule := range lc.Rules { + if rule.Status == lifecycle.Disabled || rule.AccessTransition.IsNull() { + continue + } + prefix := rule.GetPrefix() + if prefix != "" && !strings.HasPrefix(object, prefix) { + continue + } + at := rule.AccessTransition + if entry.hits(at.Window.D(), binWidth) >= uint64(at.PromoteAfterAccesses) { + return true + } + } + return false +} + +func (s *accessTierState) promotionWatermarkOK(ctx context.Context, cfg ilm.Config, hot int, bucket, object string, bytes uint64) bool { + if bytes > ^uint64(0)>>1 { + return false + } + spaces := s.z.getServerPoolsAvailableSpace(ctx, bucket, object, int64(bytes)) + if hot < 0 || hot >= len(spaces) { + return false + } + space := spaces[hot] + return space.Available > 0 && space.MaxUsedPct < cfg.AccessPromoteWatermark +} + +func accessPoolIsHotter(cfg ilm.Config, src, dst int) bool { + srcPos, dstPos := -1, -1 + for pos, pool := range cfg.AccessPools { + if pool == src { + srcPos = pos + } + if pool == dst { + dstPos = pos + } + } + return srcPos >= 0 && dstPos >= 0 && srcPos < dstPos +} + +// accessTierStamp renders the marker written to every version a move touches. +// parseAccessTierStamp is its inverse; keep the two together. +func accessTierStamp(pool int, movedAt int64) string { + return strconv.Itoa(pool) + ":" + strconv.FormatInt(movedAt, 10) +} + +func parseAccessTierStamp(meta map[string]string) (pool int, movedAt time.Time, ok bool) { + value, ok := meta[accessTierMetadataKey] + if !ok { + return 0, time.Time{}, false + } + poolText, stampText, ok := strings.Cut(value, ":") + if !ok { + return 0, time.Time{}, false + } + pool, err := strconv.Atoi(poolText) + if err != nil { + return 0, time.Time{}, false + } + stamp, err := strconv.ParseInt(stampText, 10, 64) + if err != nil || stamp <= 0 { + return 0, time.Time{}, false + } + return pool, time.Unix(0, stamp), true +} + +func accessDemoteEligible(lc *lifecycle.Lifecycle, oi ObjectInfo, pool int, cfg ilm.Config, now time.Time) bool { + if lc == nil || oi.DeleteMarker || oi.IsDir || oi.IsRemote() { + return false + } + markedPool, movedAt, ok := parseAccessTierStamp(oi.UserDefined) + if !ok || markedPool != pool || now.Sub(movedAt) < cfg.AccessMinResidency { + return false + } + cold, ok := cfg.ColdPool() + if !ok || !accessPoolIsHotter(cfg, pool, cold) { + return false + } + rule, _, ok := lc.AccessRule(oi.ToLifecycleOpts()) + if !ok { + return false + } + last := globalAccessTracker.lastAccess(oi.Bucket, oi.Name) + if !last.IsZero() && now.Sub(last) < rule.DemoteAfterIdle.D() { + return false + } + hits := globalAccessTracker.hits(oi.Bucket, oi.Name, rule.Window.D()) + return hits <= uint64(rule.DemoteAfterAccesses) && hits < uint64(rule.PromoteAfterAccesses) +} + +// applyAccessTransition runs on scanner nodes and publishes eligible demotion +// candidates to the node's next access-counter shard. +func applyAccessTransition(_ context.Context, item *scannerItem, oi ObjectInfo) { + if item == nil || item.lifeCycle == nil || globalAccessTierState == nil || !item.lifeCycle.HasAccessTransition() { + return + } + cfg := globalILMConfig.accessCfg() + if !cfg.AccessTiering || !accessDemoteEligible(item.lifeCycle, oi, item.poolIdx, cfg, time.Now()) { + return + } + globalAccessTracker.noteDemoteCandidate(item.bucket, item.objectPath(), item.poolIdx) +} + +func (s *accessTierState) processTask(task accessTierTask) error { + completed := false + defer func() { + if !completed { + s.releasePending(task, task.direction == accessTierPromote) + } + }() + if err := task.ctx.Err(); err != nil { + return err + } + cfg := globalILMConfig.accessCfg() + if !cfg.AccessTiering || !validAccessTopology(s.z, cfg) || s.z.IsRebalanceStarted(task.ctx) || s.z.IsDecommissionRunning() { + return errAccessTierNotEligible + } + if s.z.IsSuspended(task.dst) || s.z.IsPoolRebalancing(task.dst) { + return errAccessTierNotEligible + } + hot, _ := cfg.HotPool() + cold, _ := cfg.ColdPool() + if task.direction == accessTierPromote && task.dst != hot || task.direction == accessTierDemote && task.dst != cold { + return errAccessTierNotEligible + } + + reserved := task.bytes + var auditOI ObjectInfo + moved, err := moveObjectPool(task.ctx, s.z, task.bucket, task.object, task.src, task.dst, + func(latest ObjectInfo, exactBytes uint64) error { + auditOI = latest + lc, err := globalLifecycleSys.Get(task.bucket) + if err != nil { + return errAccessTierNotEligible + } + rule, _, ok := lc.AccessRule(latest.ToLifecycleOpts()) + if !ok { + return errAccessTierNotEligible + } + hits := globalAccessTracker.hits(task.bucket, task.object, rule.Window.D()) + if task.direction == accessTierDemote { + if !accessDemoteEligible(lc, latest, task.src, cfg, time.Now()) { + return errAccessTierNotEligible + } + return nil + } + if hits < uint64(rule.PromoteAfterAccesses) { + return errAccessTierNotEligible + } + if !s.promotionWatermarkOK(task.ctx, cfg, task.dst, task.bucket, task.object, exactBytes) { + s.recordSkip("watermark") + return errAccessTierNotEligible + } + reason, ok := s.replaceReservation(task.bucket, reserved, exactBytes, cfg, lc.AccessQuotaBytes()) + if !ok { + s.recordSkip(reason) + return errAccessTierNotEligible + } + reserved = exactBytes + return nil + }) + task.bytes = reserved + if err != nil { + return err + } + s.completeTask(task, moved) + completed = true + + if auditOI.Bucket == "" { + auditOI = ObjectInfo{Bucket: task.bucket, Name: task.object} + } + direction := "demote" + if task.direction == accessTierPromote { + direction = "promote" + } + tags := newLifecycleAuditEvent(lcEventSrc_AccessTier, lifecycle.Event{}).Tags() + tags["ilm-access-direction"] = direction + tags["ilm-access-src-pool"] = strconv.Itoa(task.src) + tags["ilm-access-dst-pool"] = strconv.Itoa(task.dst) + auditLogLifecycle(task.ctx, auditOI, ILMAccessTier, tags, globalLifecycleSys.trace(auditOI)) + return nil +} + +// accessObjectVersions resolves a single object's complete version stack from +// the specified source pool using the same quorum listing primitive as +// rebalance/decommission. +func accessObjectVersions(ctx context.Context, z *erasureServerPools, src int, bucket, object string) ([]FileInfo, error) { + if src < 0 || src >= len(z.serverPools) { + return nil, errInvalidArgument + } + set := z.serverPools[src].getHashedSet(encodeDirObject(object)) + disks, _ := set.getOnlineDisksWithHealing(false) + if len(disks) == 0 { + return nil, errErasureReadQuorum + } + listingQuorum := (set.setDriveCount + 1) / 2 + resolver := metadataResolutionParams{dirQuorum: listingQuorum, objQuorum: listingQuorum, bucket: bucket} + + var ( + mu sync.Mutex + versions []FileInfo + found bool + foundErr error + ) + consume := func(entry metaCacheEntry) { + if entry.name != object { + return + } + mu.Lock() + defer mu.Unlock() + if found { + return + } + fivs, err := entry.fileInfoVersions(bucket) + if err != nil { + foundErr = err + found = true + return + } + versions = append(versions, fivs.Versions...) + found = true + } + err := listPathRaw(ctx, listPathRawOptions{ + disks: disks, bucket: bucket, path: object, recursive: true, + minDisks: listingQuorum, reportNotFound: false, + agreed: consume, + partial: func(entries metaCacheEntries, _ []error) { + if entry, ok := entries.resolve(&resolver); ok { + consume(*entry) + } + }, + }) + if err != nil { + return nil, err + } + if foundErr != nil { + return nil, foundErr + } + if !found || len(versions) == 0 { + return nil, ObjectNotFound{Bucket: bucket, Object: object} + } + if int64(len(versions)) > scannerExcessObjectVersions.Load() { + return nil, errAccessTierTooManyVersion + } + for _, version := range versions { + if version.IsRemote() { + return nil, errAccessTierRemoteVersion + } + } + if len(versions) == 1 && versions[0].Deleted { + return nil, errAccessTierNotEligible + } + versionsSorter(versions).reverse() + return versions, nil +} + +func accessObjectBytes(ctx context.Context, z *erasureServerPools, src int, bucket, object string) (uint64, error) { + versions, err := accessObjectVersions(ctx, z, src, bucket, object) + if err != nil { + return 0, err + } + var total uint64 + for _, version := range versions { + if !version.Deleted && version.Size > 0 { + total = addUsage(total, uint64(version.Size)) + } + } + return total, nil +} + +func deleteAccessTierPoolObject(ctx context.Context, z *erasureServerPools, pool int, bucket, object string) error { + _, err := z.serverPools[pool].DeleteObject(ctx, bucket, encodeDirObject(object), ObjectOptions{ + DeletePrefix: true, DeletePrefixObject: true, NoLock: true, NoAuditLog: true, + }) + return err +} + +// rollbackAccessTierDestination removes only what this move wrote. +// +// A blanket prefix delete would be wrong: the top-level namespace lock is +// taken on pool 0 / set 0 (see erasureServerPools.NewNSLock), while a client +// PutObject locks the hashed set of whichever pool it lands in, and lockers +// are per set. A concurrent client write can therefore land on the +// destination while a move is in flight, and it must survive our rollback. +// +// Versioned writes are removed by exact version ID, which can never touch a +// client's version. An unversioned write has no ID to target, so it is only +// removed while it still carries this move's own marker stamp. +func rollbackAccessTierDestination(ctx context.Context, z *erasureServerPools, dst int, bucket, object string, written []string, movedAt int64) { + stamp := accessTierStamp(dst, movedAt) + for _, versionID := range written { + if versionID == "" { + // Unversioned bucket: a single version with no ID. Only drop it + // if it is still the copy this move made. + oi, err := z.serverPools[dst].GetObjectInfo(ctx, bucket, encodeDirObject(object), ObjectOptions{NoLock: true}) + if err != nil { + if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { + ilmLogIf(ctx, fmt.Errorf("access tier rollback failed for %s/%s in pool %d: %w", bucket, object, dst, err)) + } + continue + } + if oi.UserDefined[accessTierMetadataKey] != stamp { + ilmLogIf(ctx, fmt.Errorf("access tier rollback skipped for %s/%s in pool %d: destination was overwritten concurrently", bucket, object, dst)) + continue + } + if err := deleteAccessTierPoolObject(ctx, z, dst, bucket, object); err != nil { + ilmLogIf(ctx, fmt.Errorf("access tier rollback failed for %s/%s in pool %d: %w", bucket, object, dst, err)) + } + continue + } + _, err := z.serverPools[dst].DeleteObject(ctx, bucket, encodeDirObject(object), ObjectOptions{ + Versioned: true, VersionID: versionID, NoLock: true, NoAuditLog: true, + }) + if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { + ilmLogIf(ctx, fmt.Errorf("access tier rollback failed for %s/%s version %s in pool %d: %w", bucket, object, versionID, dst, err)) + } + } +} + +// moveObjectPool copies the complete version stack oldest-first and removes +// the source only after every destination write succeeded. +// +// The namespace lock excludes concurrent deletes, which take the same +// top-level lock. It does not exclude a concurrent PutObject, which locks the +// hashed set of its target pool; see rollbackAccessTierDestination for how +// that is contained. +func moveObjectPool(ctx context.Context, z *erasureServerPools, bucket, object string, src, dst int, beforeCopy func(ObjectInfo, uint64) error) (uint64, error) { + if bucket == minioMetaBucket || src == dst || src < 0 || dst < 0 || src >= len(z.serverPools) || dst >= len(z.serverPools) { + return 0, errAccessTierNotEligible + } + if z.IsRebalanceStarted(ctx) || z.IsDecommissionRunning() || z.IsSuspended(dst) || z.IsPoolRebalancing(dst) { + return 0, errAccessTierNotEligible + } + + lk := z.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout) + if err != nil { + return 0, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + + versions, err := accessObjectVersions(ctx, z, src, bucket, object) + if err != nil { + return 0, err + } + versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object) + latest := versions[len(versions)-1].ToObjectInfo(bucket, object, versioned) + var total uint64 + for _, version := range versions { + if !version.Deleted && version.Size > 0 { + total = addUsage(total, uint64(version.Size)) + } + } + if beforeCopy != nil { + if err := beforeCopy(latest, total); err != nil { + return 0, err + } + } + + // A failed earlier attempt may have copied the full stack but failed to + // remove the source. It is safe to discard only a destination carrying + // our internal marker; an unrelated split-brain copy is left untouched. + dstOI, dstErr := z.serverPools[dst].GetObjectInfo(ctx, bucket, encodeDirObject(object), ObjectOptions{NoLock: true}) + if dstErr == nil { + markedPool, _, marked := parseAccessTierStamp(dstOI.UserDefined) + if !marked || markedPool != dst { + return 0, errAccessTierNotEligible + } + if err := deleteAccessTierPoolObject(ctx, z, dst, bucket, object); err != nil { + return 0, err + } + } else if !isErrObjectNotFound(dstErr) && !isErrVersionNotFound(dstErr) { + return 0, dstErr + } + + movedAt := time.Now().UnixNano() + var written []string + rollbackDestination := true + defer func() { + if len(written) == 0 || !rollbackDestination { + return + } + // Roll back before releasing the namespace lock. Use a fresh timeout + // when leadership cancellation caused the original failure. + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), globalDeleteOperationTimeout.Timeout()) + defer cancel() + rollbackAccessTierDestination(cleanupCtx, z, dst, bucket, object, written, movedAt) + }() + + set := z.serverPools[src].getHashedSet(encodeDirObject(object)) + for _, version := range versions { + if err := ctx.Err(); err != nil { + return 0, err + } + versionID := version.VersionID + if versionID == "" { + versionID = nullVersionID + } + if version.Deleted { + written = append(written, version.VersionID) + _, err = z.serverPools[dst].DeleteObject(ctx, bucket, encodeDirObject(object), ObjectOptions{ + Versioned: true, VersionID: versionID, MTime: version.ModTime, + DeleteReplication: version.ReplicationState, + SrcPoolIdx: src, DstPoolIdx: &dst, DataMovement: true, + DeleteMarker: true, NoLock: true, NoAuditLog: true, + }) + if err != nil { + return 0, err + } + continue + } + + gr, err := set.GetObjectNInfo(ctx, bucket, encodeDirObject(object), nil, http.Header{}, ObjectOptions{ + VersionID: versionID, NoDecryption: true, NoLock: true, NoAuditLog: true, + }) + if err != nil { + return 0, err + } + written = append(written, version.VersionID) + if err = moveAccessTierVersion(ctx, z, src, dst, bucket, gr, movedAt); err != nil { + return 0, err + } + } + + // Re-read while still holding the namespace lock and refuse the source + // delete if the latest version changed despite the lock contract. + current, err := z.serverPools[src].GetObjectInfo(ctx, bucket, encodeDirObject(object), ObjectOptions{NoLock: true}) + if err != nil { + return 0, err + } + if !current.ModTime.Equal(latest.ModTime) || current.VersionID != latest.VersionID || current.ETag != latest.ETag || current.DeleteMarker != latest.DeleteMarker { + return 0, errAccessTierNotEligible + } + // Once source deletion starts, an error can be an uncertain commit. Keep + // the complete destination in that case: preserving two copies is safer + // than risking zero. + rollbackDestination = false + err = deleteAccessTierPoolObject(ctx, z, src, bucket, object) + if err != nil { + return 0, err + } + return total, nil +} + +func moveAccessTierVersion(ctx context.Context, z *erasureServerPools, src, dst int, bucket string, gr *GetObjectReader, movedAt int64) (err error) { + oi := gr.ObjInfo + defer gr.Close() + + actualSize, err := oi.GetActualSize() + if err != nil { + return err + } + metadata := maps.Clone(oi.UserDefined) + if metadata == nil { + metadata = make(map[string]string, 1) + } + metadata[accessTierMetadataKey] = accessTierStamp(dst, movedAt) + + if oi.isMultipart() { + res, err := z.NewMultipartUpload(ctx, bucket, oi.Name, ObjectOptions{ + VersionID: oi.VersionID, UserDefined: metadata, NoAuditLog: true, + NoLock: true, DataMovement: true, SrcPoolIdx: src, DstPoolIdx: &dst, + }) + if err != nil { + return err + } + abort := true + defer func() { + if abort { + z.AbortMultipartUpload(ctx, bucket, oi.Name, res.UploadID, ObjectOptions{NoAuditLog: true}) + } + }() + parts := make([]CompletePart, len(oi.Parts)) + for i, part := range oi.Parts { + hr, err := hash.NewReader(ctx, io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize) + if err != nil { + return err + } + pi, err := z.PutObjectPart(ctx, bucket, oi.Name, res.UploadID, part.Number, NewPutObjReader(hr), ObjectOptions{ + PreserveETag: part.ETag, + IndexCB: func() []byte { return part.Index }, + NoAuditLog: true, NoLock: true, + }) + if err != nil { + return err + } + parts[i] = CompletePart{ + ETag: pi.ETag, PartNumber: pi.PartNumber, + ChecksumCRC32: pi.ChecksumCRC32, ChecksumCRC32C: pi.ChecksumCRC32C, + ChecksumSHA1: pi.ChecksumSHA1, ChecksumSHA256: pi.ChecksumSHA256, + } + } + _, err = z.CompleteMultipartUpload(ctx, bucket, oi.Name, res.UploadID, parts, ObjectOptions{ + SrcPoolIdx: src, DstPoolIdx: &dst, DataMovement: true, + VersionID: oi.VersionID, MTime: oi.ModTime, NoLock: true, NoAuditLog: true, + }) + if err != nil { + return err + } + abort = false + return nil + } + + hr, err := hash.NewReader(ctx, io.LimitReader(gr, oi.Size), oi.Size, "", "", actualSize) + if err != nil { + return err + } + opts := ObjectOptions{ + SrcPoolIdx: src, DstPoolIdx: &dst, DataMovement: true, + VersionID: oi.VersionID, MTime: oi.ModTime, UserDefined: metadata, + PreserveETag: oi.ETag, NoLock: true, NoAuditLog: true, + } + if len(oi.Parts) > 0 { + opts.IndexCB = func() []byte { return oi.Parts[0].Index } + } + _, err = z.PutObject(ctx, bucket, oi.Name, NewPutObjReader(hr), opts) + return err +} diff --git a/cmd/ilm-access-tier_test.go b/cmd/ilm-access-tier_test.go new file mode 100644 index 0000000000000..981c47149777b --- /dev/null +++ b/cmd/ilm-access-tier_test.go @@ -0,0 +1,263 @@ +// Copyright (c) 2015-2026 MinIO, Inc. + +package cmd + +import ( + "bytes" + "context" + "errors" + "strconv" + "strings" + "testing" + "time" + + "github.com/klauspost/compress/zstd" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/config/ilm" + "github.com/tinylib/msgp/msgp" +) + +func accessLifecycleForTest(t *testing.T) *lifecycle.Lifecycle { + t.Helper() + xml := "" + + "accessEnabled" + + "10m100" + + "51h" + + "" + lc, err := lifecycle.ParseLifecycleConfig(strings.NewReader(xml)) + if err != nil { + t.Fatal(err) + } + return lc +} + +func TestAccessTierStampAndDemoteEligibility(t *testing.T) { + oldTracker := globalAccessTracker + globalAccessTracker = newAccessTracker() + t.Cleanup(func() { globalAccessTracker = oldTracker }) + + now := time.Now() + cfg := ilm.Config{ + AccessTiering: true, AccessPools: []int{0, 1}, + AccessBinWidth: time.Minute, AccessBins: 12, + AccessMinResidency: 30 * time.Minute, + } + oi := ObjectInfo{ + Bucket: "bucket", Name: "object", Size: 10, IsLatest: true, + UserDefined: map[string]string{ + accessTierMetadataKey: "0:" + strconv.FormatInt(now.Add(-2*time.Hour).UnixNano(), 10), + }, + } + if !accessDemoteEligible(accessLifecycleForTest(t), oi, 0, cfg, now) { + t.Fatal("cold, resident object should be demotion eligible") + } + oi.UserDefined[accessTierMetadataKey] = "0:" + strconv.FormatInt(now.Add(-time.Minute).UnixNano(), 10) + if accessDemoteEligible(accessLifecycleForTest(t), oi, 0, cfg, now) { + t.Fatal("object inside minimum residency was eligible") + } + oi.UserDefined[accessTierMetadataKey] = "broken" + if accessDemoteEligible(accessLifecycleForTest(t), oi, 0, cfg, now) { + t.Fatal("object with malformed marker was eligible") + } +} + +func TestAccessTierReservationsEnforceCaps(t *testing.T) { + state := newAccessTierState(t.Context()) + state.usageReady = true + state.baseUsage["a"] = 80 + state.baseTotal = 180 + + cfg := ilm.Config{AccessMaxSize: 200} + task := accessTierTask{ctx: t.Context(), bucket: "a", object: "one", bytes: 30} + if reason, ok := state.reservePromotion(task, cfg, 0); ok || reason != "max-size" { + t.Fatalf("max-size reserve = %q/%v", reason, ok) + } + + cfg.AccessMaxSize = 0 + if reason, ok := state.reservePromotion(task, cfg, 100); ok || reason != "quota" { + t.Fatalf("quota reserve = %q/%v", reason, ok) + } + + task.bytes = 20 + if reason, ok := state.reservePromotion(task, cfg, 100); !ok || reason != "" { + t.Fatalf("valid reserve = %q/%v", reason, ok) + } + if got := state.bucketUsageLocked("a"); got != 100 { + t.Fatalf("reserved bucket usage = %d, want 100", got) + } + state.releasePending(task, true) +} + +func TestDataMovementDestinationIsGated(t *testing.T) { + dst := 0 + if got := dataMovementDstPool(ObjectOptions{DstPoolIdx: &dst}); got != nil { + t.Fatal("destination honored without DataMovement") + } + if got := dataMovementDstPool(ObjectOptions{DataMovement: true, DstPoolIdx: &dst}); got == nil || *got != 0 { + t.Fatalf("destination = %v", got) + } +} + +func TestForcedDestinationPoolSelection(t *testing.T) { + z := &erasureServerPools{serverPools: make([]*erasureSets, 2)} + dst := 1 + if got, err := z.getPoolIdx(context.Background(), "bucket", "object", 1, &dst); err != nil || got != dst { + t.Fatalf("destination = %d, err = %v", got, err) + } + bad := 2 + if _, err := z.getPoolIdx(context.Background(), "bucket", "object", 1, &bad); !errors.Is(err, errInvalidArgument) { + t.Fatalf("out-of-range error = %v", err) + } + z.poolMeta.Pools = make([]PoolStatus, 2) + z.poolMeta.Pools[1].Decommission = &PoolDecommissionInfo{} + if _, err := z.getPoolIdx(context.Background(), "bucket", "object", 1, &dst); err == nil { + t.Fatal("suspended destination was accepted") + } +} + +func TestHotTierAccounting(t *testing.T) { + entry := dataUsageEntry{} + entry.addSizes(sizeSummary{totalSize: 100, hotTierSize: 60, versions: 1}) + entry.merge(dataUsageEntry{Size: 50, HotTierSize: 25}) + if entry.Size != 150 || entry.HotTierSize != 85 { + t.Fatalf("usage = size:%d hot:%d", entry.Size, entry.HotTierSize) + } +} + +func TestScannerCyclesApart(t *testing.T) { + tests := []struct { + current, previous uint32 + want uint32 + }{ + {current: 12, previous: 10, want: 2}, + {current: 0, previous: ^uint32(0), want: 1}, + // A cycle counter reset is not evidence that a complete pass covered + // recent moves, so it must not release conservative deltas. + {current: 1, previous: 100, want: 0}, + } + for _, tt := range tests { + if got := scannerCyclesApart(tt.current, tt.previous); got != tt.want { + t.Fatalf("scannerCyclesApart(%d, %d) = %d, want %d", tt.current, tt.previous, got, tt.want) + } + } +} + +func TestDataUsageCacheV8Migration(t *testing.T) { + var encoded bytes.Buffer + if err := encoded.WriteByte(dataUsageCacheVerV8); err != nil { + t.Fatal(err) + } + zw, err := zstd.NewWriter(&encoded) + if err != nil { + t.Fatal(err) + } + mw := msgp.NewWriter(zw) + if err = mw.WriteMapHeader(2); err != nil { + t.Fatal(err) + } + if err = mw.WriteString("Info"); err != nil { + t.Fatal(err) + } + info := dataUsageCacheInfo{Name: dataUsageRoot, NextCycle: 17, LastUpdate: time.Now().UTC()} + if err = info.EncodeMsg(mw); err != nil { + t.Fatal(err) + } + if err = mw.WriteString("Cache"); err != nil { + t.Fatal(err) + } + if err = mw.WriteMapHeader(1); err != nil { + t.Fatal(err) + } + if err = mw.WriteString("entry"); err != nil { + t.Fatal(err) + } + // A v8 entry has no hts field. Encoding only populated fields also + // verifies that its map decoder retains normal msgpack compatibility. + if err = mw.WriteMapHeader(2); err != nil { + t.Fatal(err) + } + if err = mw.WriteString("sz"); err != nil { + t.Fatal(err) + } + if err = mw.WriteInt64(123); err != nil { + t.Fatal(err) + } + if err = mw.WriteString("os"); err != nil { + t.Fatal(err) + } + if err = mw.WriteUint64(2); err != nil { + t.Fatal(err) + } + if err = mw.Flush(); err != nil { + t.Fatal(err) + } + if err = zw.Close(); err != nil { + t.Fatal(err) + } + + var migrated dataUsageCache + if err = migrated.deserialize(bytes.NewReader(encoded.Bytes())); err != nil { + t.Fatal(err) + } + entry := migrated.Cache["entry"] + if entry.Size != 123 || entry.Objects != 2 || entry.HotTierSize != 0 { + t.Fatalf("migrated entry = size:%d objects:%d hot:%d", entry.Size, entry.Objects, entry.HotTierSize) + } + if migrated.Info.NextCycle != 17 || !migrated.Info.LastUpdate.Equal(info.LastUpdate) { + t.Fatalf("migrated cache info = %+v", migrated.Info) + } +} + +// The stamp written on every moved version and the stamp the rollback path +// matches against must agree, otherwise rollback silently skips its own work. +func TestAccessTierStampRoundTrip(t *testing.T) { + movedAt := time.Now().UnixNano() + stamp := accessTierStamp(3, movedAt) + + pool, at, ok := parseAccessTierStamp(map[string]string{accessTierMetadataKey: stamp}) + if !ok { + t.Fatalf("stamp %q did not parse", stamp) + } + if pool != 3 { + t.Fatalf("pool = %d, want 3", pool) + } + if at.UnixNano() != movedAt { + t.Fatalf("movedAt = %d, want %d", at.UnixNano(), movedAt) + } + // A different move of the same object must not match, so a concurrent + // client overwrite is never mistaken for our own copy. + if stamp == accessTierStamp(3, movedAt+1) { + t.Fatal("stamps from distinct moves collided") + } + if stamp == accessTierStamp(4, movedAt) { + t.Fatal("stamps from distinct pools collided") + } +} + +func TestAccessHitsMightPromote(t *testing.T) { + lc := accessLifecycleForTest(t) + hot := accessEntry{Bins: []uint32{100}, HeadAt: 0} + cold := accessEntry{Bins: []uint32{1}, HeadAt: 0} + if !accessHitsMightPromote(lc, "object", hot, 60) { + t.Fatal("object above promote threshold was rejected") + } + if accessHitsMightPromote(lc, "object", cold, 60) { + t.Fatal("object below promote threshold was accepted") + } + + xml := "logsEnabled" + + "logs/" + + "10m100" + + "51h" + + "" + prefixed, err := lifecycle.ParseLifecycleConfig(strings.NewReader(xml)) + if err != nil { + t.Fatal(err) + } + if accessHitsMightPromote(prefixed, "data/object", hot, 60) { + t.Fatal("object outside the rule prefix was accepted") + } + if !accessHitsMightPromote(prefixed, "logs/object", hot, 60) { + t.Fatal("object inside the rule prefix was rejected") + } +} diff --git a/cmd/ilm-access-tracker.go b/cmd/ilm-access-tracker.go new file mode 100644 index 0000000000000..11a8c5bdaffd2 --- /dev/null +++ b/cmd/ilm-access-tracker.go @@ -0,0 +1,598 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "cmp" + "context" + "fmt" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/minio/minio/internal/config/ilm" + "github.com/zeebo/xxh3" +) + +//go:generate msgp -file=$GOFILE -unexported +//msgp:ignore accessTracker mergedAccess + +const ( + // accessTrackerPrefix is where each node publishes its own counters. + // One object per node, merged by every node on a timer. + accessTrackerPrefix = minioConfigPrefix + "/ilm/access" + + // accessQueueSize bounds the GET -> tracker handoff. Overflow drops + // samples rather than slowing down reads. + accessQueueSize = 100000 + + // accessMaxDemoteCandidates bounds what the scanner may hand over in a + // single flush interval. + accessMaxDemoteCandidates = 100000 + + // accessShardStaleFactor multiplies the flush interval to decide when a + // peer's counters are too old to trust, e.g. after a node is removed. + accessShardStaleFactor = 5 +) + +func accessShardFresh(now int64, shard accessShard, stale int64) bool { + if shard.UpdatedAt <= 0 { + return false + } + if stale <= 0 { + return true + } + age := now - shard.UpdatedAt + return age >= -stale && age <= stale +} + +// accessEntry is a rolling hit counter for one object. Bins[0] is the current +// bin and each subsequent bin is one bin-width older, so a rule asking for +// "100 hits in 10 minutes" sums the newest ceil(10m/binWidth) bins. +// +// A fixed window is used rather than an exponentially decayed score because +// the rule is stated to operators in exactly those terms. +type accessEntry struct { + Bins []uint32 `msg:"b"` + HeadAt int64 `msg:"h"` // unix seconds at the start of Bins[0] + LastAt int64 `msg:"l"` // unix seconds of the most recent hit +} + +// demoteCandidate is an object the scanner found sitting on a fast pool with +// no recent reads. Candidates ride the node's own counter shard so they reach +// the leader without a new peer RPC. +type demoteCandidate struct { + Bucket string `msg:"b"` + Object string `msg:"o"` + Pool int `msg:"p"` +} + +// accessShard is what one node publishes. BinWidth is carried so a peer that +// has not yet picked up a configuration change is ignored rather than merged +// with mismatched bins. +type accessShard struct { + UpdatedAt int64 `msg:"u"` + BinWidth int64 `msg:"bw"` + Entries map[string]accessEntry `msg:"e"` + Demote []demoteCandidate `msg:"d"` +} + +// binStart truncates a unix timestamp to the start of its bin. +func binStart(now, binWidth int64) int64 { + if binWidth <= 0 { + return now + } + return now - now%binWidth +} + +// rollTo advances the counter to now, zeroing the bins that elapsed since the +// last update and resizing if the configured bin count changed. +func (e *accessEntry) rollTo(now, binWidth int64, nbins int) { + if nbins <= 0 || binWidth <= 0 { + return + } + if len(e.Bins) != nbins { + resized := make([]uint32, nbins) + copy(resized, e.Bins) + e.Bins = resized + } + head := binStart(now, binWidth) + if e.HeadAt == 0 { + e.HeadAt = head + return + } + steps := (head - e.HeadAt) / binWidth + if steps <= 0 { + return + } + if steps >= int64(nbins) { + clear(e.Bins) + } else { + copy(e.Bins[steps:], e.Bins[:nbins-int(steps)]) + clear(e.Bins[:steps]) + } + e.HeadAt = head +} + +// hits returns the number of accesses recorded over the newest bins covering +// window. A window longer than the configured history is clamped to it. +// +// The newest bin is partial, so the covered span is between window-binWidth +// and window. Operators tune resolution with ilm access_bin_width. +func (e accessEntry) hits(window time.Duration, binWidth int64) uint64 { + if binWidth <= 0 || len(e.Bins) == 0 { + return 0 + } + n := int((int64(window/time.Second) + binWidth - 1) / binWidth) + if n < 1 { + n = 1 + } + if n > len(e.Bins) { + n = len(e.Bins) + } + var total uint64 + for _, v := range e.Bins[:n] { + total += uint64(v) + } + return total +} + +// total is the whole retained history, used to decide what to evict. +func (e accessEntry) total() uint64 { + var t uint64 + for _, v := range e.Bins { + t += uint64(v) + } + return t +} + +// mergeFrom adds another node's counters for the same object. Both sides must +// already be rolled to the same head. +func (e *accessEntry) mergeFrom(o accessEntry) { + for i := range e.Bins { + if i < len(o.Bins) { + total := uint64(e.Bins[i]) + uint64(o.Bins[i]) + if total > uint64(^uint32(0)) { + total = uint64(^uint32(0)) + } + e.Bins[i] = uint32(total) + } + } + if o.LastAt > e.LastAt { + e.LastAt = o.LastAt + } +} + +// mergedAccess is an immutable cluster-wide snapshot. Readers take it from an +// atomic pointer, so the hot scanner and sweep paths never take a lock. +type mergedAccess struct { + entries map[string]accessEntry + binWidth int64 + at int64 +} + +func (m *mergedAccess) hits(key string, window time.Duration) uint64 { + if m == nil { + return 0 + } + e, ok := m.entries[key] + if !ok { + return 0 + } + return e.hits(window, m.binWidth) +} + +func (m *mergedAccess) lastAccess(key string) int64 { + if m == nil { + return 0 + } + return m.entries[key].LastAt +} + +// accessTracker records how often each object is read. +// +// Ownership is deliberately narrow: the live counter map is touched only by +// run()'s goroutine, so it needs no lock. Everything read from elsewhere goes +// through the immutable merged snapshot. +type accessTracker struct { + ch chan string + enabled atomic.Bool + merged atomic.Pointer[mergedAccess] + + // Demote candidates arrive from scanner goroutines, so this one does + // need a lock. It is small: only objects we previously promoted. + demoteMu sync.Mutex + demote map[string]demoteCandidate + + dropped atomic.Uint64 +} + +var globalAccessTracker = newAccessTracker() + +func newAccessTracker() *accessTracker { + return &accessTracker{ + ch: make(chan string, accessQueueSize), + demote: make(map[string]demoteCandidate), + } +} + +// accessKey is the tracker's map key. Bucket names cannot contain '/', so the +// join is unambiguous. +func accessKey(bucket, object string) string { + return bucket + "/" + object +} + +func splitAccessKey(key string) (bucket, object string, ok bool) { + bucket, object, ok = strings.Cut(key, "/") + if !ok || bucket == "" || object == "" { + return "", "", false + } + return bucket, object, true +} + +// note records one read. It is called from the GET path and must never block +// or allocate meaningfully: on a full queue the sample is dropped. +func (t *accessTracker) note(bucket, object string) { + if t == nil || !t.enabled.Load() { + return + } + select { + case t.ch <- accessKey(bucket, object): + default: + t.dropped.Add(1) + } +} + +// noteDemoteCandidate is called by the scanner for an object it found on a +// fast pool that has gone quiet. The leader picks these up on the next merge. +func (t *accessTracker) noteDemoteCandidate(bucket, object string, pool int) { + if t == nil || !t.enabled.Load() { + return + } + t.demoteMu.Lock() + defer t.demoteMu.Unlock() + if len(t.demote) >= accessMaxDemoteCandidates { + return + } + t.demote[accessKey(bucket, object)] = demoteCandidate{Bucket: bucket, Object: object, Pool: pool} +} + +// hits reports cluster-wide accesses to an object over window. +func (t *accessTracker) hits(bucket, object string, window time.Duration) uint64 { + if t == nil { + return 0 + } + return t.merged.Load().hits(accessKey(bucket, object), window) +} + +// lastAccess reports the cluster-wide time an object was last read. A zero +// time means "no read on record", which for demotion purposes is idle. +func (t *accessTracker) lastAccess(bucket, object string) time.Time { + if t == nil { + return time.Time{} + } + sec := t.merged.Load().lastAccess(accessKey(bucket, object)) + if sec == 0 { + return time.Time{} + } + return time.Unix(sec, 0) +} + +// snapshot returns the current merged view, or nil if none has been published. +func (t *accessTracker) snapshot() *mergedAccess { + if t == nil { + return nil + } + return t.merged.Load() +} + +// takeDemoteCandidates drains and returns the pending candidates. +func (t *accessTracker) takeDemoteCandidates() []demoteCandidate { + t.demoteMu.Lock() + defer t.demoteMu.Unlock() + if len(t.demote) == 0 { + return nil + } + out := make([]demoteCandidate, 0, len(t.demote)) + for _, c := range t.demote { + out = append(out, c) + } + t.demote = make(map[string]demoteCandidate) + return out +} + +// restoreDemoteCandidates puts candidates back when the publisher is still +// busy. Dropping access samples is acceptable; dropping the only scanner +// discovery of an idle promoted object would delay demotion by a full scan. +func (t *accessTracker) restoreDemoteCandidates(candidates []demoteCandidate) { + if len(candidates) == 0 { + return + } + t.demoteMu.Lock() + defer t.demoteMu.Unlock() + for _, c := range candidates { + if len(t.demote) >= accessMaxDemoteCandidates { + return + } + t.demote[accessKey(c.Bucket, c.Object)] = c + } +} + +// shardName is this node's counter object. The node name is hashed so that +// host:port never has to be escaped into an object key. +func (t *accessTracker) shardName() string { + return fmt.Sprintf("%s/%016x.bin", accessTrackerPrefix, xxh3.HashString(globalLocalNodeName)) +} + +// run owns the live counter map. It drains reads, and on every flush interval +// hands a marshaled shard to a background publisher. +// +// Everything here is best effort: this is accounting for a background data +// movement decision, not a durability path. +func (t *accessTracker) run(ctx context.Context, objAPI ObjectLayer) { + cfg := globalILMConfig.accessCfg() + t.enabled.Store(cfg.AccessTiering) + + live := make(map[string]accessEntry) + if cfg.AccessTiering { + if buf, err := readConfig(ctx, objAPI, t.shardName()); err == nil { + var previous accessShard + if _, err = previous.UnmarshalMsg(buf); err == nil && previous.BinWidth == int64(cfg.AccessBinWidth/time.Second) { + now := time.Now().Unix() + for key, entry := range previous.Entries { + entry.rollTo(now, previous.BinWidth, cfg.AccessBins) + if entry.total() != 0 { + live[key] = entry + } + } + } + } + } + ticker := time.NewTicker(cfg.AccessFlush) + defer ticker.Stop() + + // One publisher goroutine keeps object-layer I/O off the drain loop. + type pub struct { + shard accessShard + cfg ilm.Config + } + pubCh := make(chan pub, 1) + go func() { + for p := range pubCh { + t.publish(ctx, objAPI, p.shard, p.cfg) + } + }() + defer close(pubCh) + + for { + select { + case <-ctx.Done(): + return + + case key := <-t.ch: + now := time.Now().Unix() + e := live[key] + e.rollTo(now, int64(cfg.AccessBinWidth/time.Second), cfg.AccessBins) + if len(e.Bins) > 0 { + if e.Bins[0] != ^uint32(0) { + e.Bins[0]++ + } + } + e.LastAt = now + live[key] = e + + case <-ticker.C: + newCfg := globalILMConfig.accessCfg() + t.enabled.Store(newCfg.AccessTiering) + if newCfg.AccessFlush != cfg.AccessFlush && newCfg.AccessFlush > 0 { + ticker.Reset(newCfg.AccessFlush) + } + cfg = newCfg + if !cfg.AccessTiering { + // Feature turned off: release the counters rather than + // holding a stale working set for the process lifetime. + clear(live) + t.merged.Store(nil) + continue + } + + now := time.Now().Unix() + binWidth := int64(cfg.AccessBinWidth / time.Second) + t.evict(live, now, binWidth, cfg) + + shard := accessShard{ + UpdatedAt: now, + BinWidth: binWidth, + Entries: make(map[string]accessEntry, len(live)), + Demote: t.takeDemoteCandidates(), + } + for k, e := range live { + e.Bins = slices.Clone(e.Bins) + shard.Entries[k] = e + } + select { + case pubCh <- pub{shard: shard, cfg: cfg}: + default: + // Previous publish still running; skip this round + // rather than queueing work we cannot keep up with. + t.restoreDemoteCandidates(shard.Demote) + } + } + } +} + +// evict rolls every counter forward and drops the ones with no hits left in +// the retained history, then enforces the tracked-object cap. +// +// ponytail: amortized O(n) sweep on the flush tick; a heap would only pay off +// past ~10M tracked keys. +func (t *accessTracker) evict(live map[string]accessEntry, now, binWidth int64, cfg ilm.Config) { + for k, e := range live { + e.rollTo(now, binWidth, cfg.AccessBins) + if e.total() == 0 { + delete(live, k) + continue + } + live[k] = e + } + if len(live) <= cfg.AccessMaxTracked { + return + } + // Over the cap: keep the hottest and drop the rest. Objects that fall + // out are cold by construction, which is exactly what the demotion path + // already assumes about anything missing from the map. + type kt struct { + key string + total uint64 + } + all := make([]kt, 0, len(live)) + for k, e := range live { + all = append(all, kt{k, e.total()}) + } + slices.SortFunc(all, func(a, b kt) int { return cmp.Compare(b.total, a.total) }) + for _, x := range all[cfg.AccessMaxTracked:] { + delete(live, x.key) + } +} + +// publish writes this node's shard and rebuilds the merged snapshot from every +// node's shard. Both halves are best effort. +func (t *accessTracker) publish(ctx context.Context, objAPI ObjectLayer, shard accessShard, cfg ilm.Config) { + buf, err := shard.MarshalMsg(nil) + if err != nil { + ilmLogIf(ctx, err) + return + } + if err := saveConfig(ctx, objAPI, t.shardName(), buf); err != nil { + ilmLogIf(ctx, err) + // Still rebuild the snapshot below: our own counters are already + // in hand and a stale peer view beats no view. + } + t.merged.Store(t.mergeShards(ctx, objAPI, shard, cfg)) +} + +// mergeShards sums every live node's counters, including our own in-memory +// shard so this node's most recent reads are never a flush behind. +func (t *accessTracker) mergeShards(ctx context.Context, objAPI ObjectLayer, own accessShard, cfg ilm.Config) *mergedAccess { + now := time.Now().Unix() + binWidth := int64(cfg.AccessBinWidth / time.Second) + out := &mergedAccess{ + entries: make(map[string]accessEntry, len(own.Entries)), + binWidth: binWidth, + at: now, + } + + add := func(s accessShard) { + if s.BinWidth != binWidth { + // A peer has not yet picked up a bin-width change; merging + // its bins would silently mis-scale the counts. + return + } + for k, e := range s.Entries { + e.rollTo(now, binWidth, cfg.AccessBins) + cur, ok := out.entries[k] + if !ok { + cur = accessEntry{Bins: make([]uint32, cfg.AccessBins)} + } + cur.mergeFrom(e) + out.entries[k] = cur + } + } + + add(own) + + ownName := t.shardName() + stale := int64(cfg.AccessFlush/time.Second) * accessShardStaleFactor + for _, name := range t.listShards(ctx, objAPI) { + if name == ownName { + continue + } + buf, err := readConfig(ctx, objAPI, name) + if err != nil { + continue + } + var s accessShard + if _, err := s.UnmarshalMsg(buf); err != nil { + continue + } + if !accessShardFresh(now, s, stale) { + continue // node is gone or wedged + } + add(s) + } + return out +} + +func (t *accessTracker) listShards(ctx context.Context, objAPI ObjectLayer) []string { + res, err := objAPI.ListObjects(ctx, minioMetaBucket, accessTrackerPrefix+"/", "", "", maxObjectList) + if err != nil { + return nil + } + names := make([]string, 0, len(res.Objects)) + for _, o := range res.Objects { + if strings.HasSuffix(o.Name, ".bin") { + names = append(names, o.Name) + } + } + return names +} + +// collectDemoteCandidates returns every node's pending demote candidates. Only +// the leader calls this, right before running a demotion pass. +// +// Our own shard is read back rather than skipped: run() drains the local map +// into the published shard, so anything found since the last leader pass lives +// there, not in memory. The local map is still drained here to pick up +// candidates recorded since that publish. +func (t *accessTracker) collectDemoteCandidates(ctx context.Context, objAPI ObjectLayer, cfg ilm.Config) []demoteCandidate { + seen := make(map[string]struct{}) + var out []demoteCandidate + + for _, c := range t.takeDemoteCandidates() { + key := accessKey(c.Bucket, c.Object) + seen[key] = struct{}{} + out = append(out, c) + } + + now := time.Now().Unix() + stale := int64(cfg.AccessFlush/time.Second) * accessShardStaleFactor + for _, name := range t.listShards(ctx, objAPI) { + buf, err := readConfig(ctx, objAPI, name) + if err != nil { + continue + } + var s accessShard + if _, err := s.UnmarshalMsg(buf); err != nil { + continue + } + if !accessShardFresh(now, s, stale) { + continue + } + for _, c := range s.Demote { + key := accessKey(c.Bucket, c.Object) + if _, dup := seen[key]; dup { + continue + } + seen[key] = struct{}{} + out = append(out, c) + } + } + return out +} diff --git a/cmd/ilm-access-tracker_gen.go b/cmd/ilm-access-tracker_gen.go new file mode 100644 index 0000000000000..e533e3cacfa01 --- /dev/null +++ b/cmd/ilm-access-tracker_gen.go @@ -0,0 +1,742 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *accessEntry) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "b": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Bins") + return + } + if cap(z.Bins) >= int(zb0002) { + z.Bins = (z.Bins)[:zb0002] + } else { + z.Bins = make([]uint32, zb0002) + } + for za0001 := range z.Bins { + z.Bins[za0001], err = dc.ReadUint32() + if err != nil { + err = msgp.WrapError(err, "Bins", za0001) + return + } + } + case "h": + z.HeadAt, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "HeadAt") + return + } + case "l": + z.LastAt, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "LastAt") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *accessEntry) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "b" + err = en.Append(0x83, 0xa1, 0x62) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Bins))) + if err != nil { + err = msgp.WrapError(err, "Bins") + return + } + for za0001 := range z.Bins { + err = en.WriteUint32(z.Bins[za0001]) + if err != nil { + err = msgp.WrapError(err, "Bins", za0001) + return + } + } + // write "h" + err = en.Append(0xa1, 0x68) + if err != nil { + return + } + err = en.WriteInt64(z.HeadAt) + if err != nil { + err = msgp.WrapError(err, "HeadAt") + return + } + // write "l" + err = en.Append(0xa1, 0x6c) + if err != nil { + return + } + err = en.WriteInt64(z.LastAt) + if err != nil { + err = msgp.WrapError(err, "LastAt") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *accessEntry) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "b" + o = append(o, 0x83, 0xa1, 0x62) + o = msgp.AppendArrayHeader(o, uint32(len(z.Bins))) + for za0001 := range z.Bins { + o = msgp.AppendUint32(o, z.Bins[za0001]) + } + // string "h" + o = append(o, 0xa1, 0x68) + o = msgp.AppendInt64(o, z.HeadAt) + // string "l" + o = append(o, 0xa1, 0x6c) + o = msgp.AppendInt64(o, z.LastAt) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *accessEntry) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "b": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bins") + return + } + if cap(z.Bins) >= int(zb0002) { + z.Bins = (z.Bins)[:zb0002] + } else { + z.Bins = make([]uint32, zb0002) + } + for za0001 := range z.Bins { + z.Bins[za0001], bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bins", za0001) + return + } + } + case "h": + z.HeadAt, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "HeadAt") + return + } + case "l": + z.LastAt, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "LastAt") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *accessEntry) Msgsize() (s int) { + s = 1 + 2 + msgp.ArrayHeaderSize + (len(z.Bins) * (msgp.Uint32Size)) + 2 + msgp.Int64Size + 2 + msgp.Int64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *accessShard) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "u": + z.UpdatedAt, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "UpdatedAt") + return + } + case "bw": + z.BinWidth, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "BinWidth") + return + } + case "e": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Entries") + return + } + if z.Entries == nil { + z.Entries = make(map[string]accessEntry, zb0002) + } else if len(z.Entries) > 0 { + clear(z.Entries) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Entries") + return + } + var za0002 accessEntry + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Entries", za0001) + return + } + z.Entries[za0001] = za0002 + } + case "d": + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Demote") + return + } + if cap(z.Demote) >= int(zb0003) { + z.Demote = (z.Demote)[:zb0003] + } else { + z.Demote = make([]demoteCandidate, zb0003) + } + for za0003 := range z.Demote { + var zb0004 uint32 + zb0004, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Demote", za0003) + return + } + for zb0004 > 0 { + zb0004-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Demote", za0003) + return + } + switch msgp.UnsafeString(field) { + case "b": + z.Demote[za0003].Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Demote", za0003, "Bucket") + return + } + case "o": + z.Demote[za0003].Object, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Demote", za0003, "Object") + return + } + case "p": + z.Demote[za0003].Pool, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Demote", za0003, "Pool") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Demote", za0003) + return + } + } + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *accessShard) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "u" + err = en.Append(0x84, 0xa1, 0x75) + if err != nil { + return + } + err = en.WriteInt64(z.UpdatedAt) + if err != nil { + err = msgp.WrapError(err, "UpdatedAt") + return + } + // write "bw" + err = en.Append(0xa2, 0x62, 0x77) + if err != nil { + return + } + err = en.WriteInt64(z.BinWidth) + if err != nil { + err = msgp.WrapError(err, "BinWidth") + return + } + // write "e" + err = en.Append(0xa1, 0x65) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Entries))) + if err != nil { + err = msgp.WrapError(err, "Entries") + return + } + for za0001, za0002 := range z.Entries { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Entries") + return + } + err = za0002.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Entries", za0001) + return + } + } + // write "d" + err = en.Append(0xa1, 0x64) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Demote))) + if err != nil { + err = msgp.WrapError(err, "Demote") + return + } + for za0003 := range z.Demote { + // map header, size 3 + // write "b" + err = en.Append(0x83, 0xa1, 0x62) + if err != nil { + return + } + err = en.WriteString(z.Demote[za0003].Bucket) + if err != nil { + err = msgp.WrapError(err, "Demote", za0003, "Bucket") + return + } + // write "o" + err = en.Append(0xa1, 0x6f) + if err != nil { + return + } + err = en.WriteString(z.Demote[za0003].Object) + if err != nil { + err = msgp.WrapError(err, "Demote", za0003, "Object") + return + } + // write "p" + err = en.Append(0xa1, 0x70) + if err != nil { + return + } + err = en.WriteInt(z.Demote[za0003].Pool) + if err != nil { + err = msgp.WrapError(err, "Demote", za0003, "Pool") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *accessShard) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "u" + o = append(o, 0x84, 0xa1, 0x75) + o = msgp.AppendInt64(o, z.UpdatedAt) + // string "bw" + o = append(o, 0xa2, 0x62, 0x77) + o = msgp.AppendInt64(o, z.BinWidth) + // string "e" + o = append(o, 0xa1, 0x65) + o = msgp.AppendMapHeader(o, uint32(len(z.Entries))) + for za0001, za0002 := range z.Entries { + o = msgp.AppendString(o, za0001) + o, err = za0002.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Entries", za0001) + return + } + } + // string "d" + o = append(o, 0xa1, 0x64) + o = msgp.AppendArrayHeader(o, uint32(len(z.Demote))) + for za0003 := range z.Demote { + // map header, size 3 + // string "b" + o = append(o, 0x83, 0xa1, 0x62) + o = msgp.AppendString(o, z.Demote[za0003].Bucket) + // string "o" + o = append(o, 0xa1, 0x6f) + o = msgp.AppendString(o, z.Demote[za0003].Object) + // string "p" + o = append(o, 0xa1, 0x70) + o = msgp.AppendInt(o, z.Demote[za0003].Pool) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *accessShard) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "u": + z.UpdatedAt, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "UpdatedAt") + return + } + case "bw": + z.BinWidth, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "BinWidth") + return + } + case "e": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Entries") + return + } + if z.Entries == nil { + z.Entries = make(map[string]accessEntry, zb0002) + } else if len(z.Entries) > 0 { + clear(z.Entries) + } + for zb0002 > 0 { + var za0002 accessEntry + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Entries") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Entries", za0001) + return + } + z.Entries[za0001] = za0002 + } + case "d": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Demote") + return + } + if cap(z.Demote) >= int(zb0003) { + z.Demote = (z.Demote)[:zb0003] + } else { + z.Demote = make([]demoteCandidate, zb0003) + } + for za0003 := range z.Demote { + var zb0004 uint32 + zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Demote", za0003) + return + } + for zb0004 > 0 { + zb0004-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Demote", za0003) + return + } + switch msgp.UnsafeString(field) { + case "b": + z.Demote[za0003].Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Demote", za0003, "Bucket") + return + } + case "o": + z.Demote[za0003].Object, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Demote", za0003, "Object") + return + } + case "p": + z.Demote[za0003].Pool, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Demote", za0003, "Pool") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Demote", za0003) + return + } + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *accessShard) Msgsize() (s int) { + s = 1 + 2 + msgp.Int64Size + 3 + msgp.Int64Size + 2 + msgp.MapHeaderSize + if z.Entries != nil { + for za0001, za0002 := range z.Entries { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + s += 2 + msgp.ArrayHeaderSize + for za0003 := range z.Demote { + s += 1 + 2 + msgp.StringPrefixSize + len(z.Demote[za0003].Bucket) + 2 + msgp.StringPrefixSize + len(z.Demote[za0003].Object) + 2 + msgp.IntSize + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *demoteCandidate) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "b": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "o": + z.Object, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "p": + z.Pool, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Pool") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z demoteCandidate) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "b" + err = en.Append(0x83, 0xa1, 0x62) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + // write "o" + err = en.Append(0xa1, 0x6f) + if err != nil { + return + } + err = en.WriteString(z.Object) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + // write "p" + err = en.Append(0xa1, 0x70) + if err != nil { + return + } + err = en.WriteInt(z.Pool) + if err != nil { + err = msgp.WrapError(err, "Pool") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z demoteCandidate) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "b" + o = append(o, 0x83, 0xa1, 0x62) + o = msgp.AppendString(o, z.Bucket) + // string "o" + o = append(o, 0xa1, 0x6f) + o = msgp.AppendString(o, z.Object) + // string "p" + o = append(o, 0xa1, 0x70) + o = msgp.AppendInt(o, z.Pool) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *demoteCandidate) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "b": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "o": + z.Object, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "p": + z.Pool, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Pool") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z demoteCandidate) Msgsize() (s int) { + s = 1 + 2 + msgp.StringPrefixSize + len(z.Bucket) + 2 + msgp.StringPrefixSize + len(z.Object) + 2 + msgp.IntSize + return +} diff --git a/cmd/ilm-access-tracker_gen_test.go b/cmd/ilm-access-tracker_gen_test.go new file mode 100644 index 0000000000000..85ab06a4ed723 --- /dev/null +++ b/cmd/ilm-access-tracker_gen_test.go @@ -0,0 +1,349 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalaccessEntry(t *testing.T) { + v := accessEntry{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgaccessEntry(b *testing.B) { + v := accessEntry{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgaccessEntry(b *testing.B) { + v := accessEntry{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalaccessEntry(b *testing.B) { + v := accessEntry{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeaccessEntry(t *testing.T) { + v := accessEntry{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeaccessEntry Msgsize() is inaccurate") + } + + vn := accessEntry{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeaccessEntry(b *testing.B) { + v := accessEntry{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeaccessEntry(b *testing.B) { + v := accessEntry{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalaccessShard(t *testing.T) { + v := accessShard{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgaccessShard(b *testing.B) { + v := accessShard{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgaccessShard(b *testing.B) { + v := accessShard{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalaccessShard(b *testing.B) { + v := accessShard{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeaccessShard(t *testing.T) { + v := accessShard{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeaccessShard Msgsize() is inaccurate") + } + + vn := accessShard{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeaccessShard(b *testing.B) { + v := accessShard{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeaccessShard(b *testing.B) { + v := accessShard{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshaldemoteCandidate(t *testing.T) { + v := demoteCandidate{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgdemoteCandidate(b *testing.B) { + v := demoteCandidate{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgdemoteCandidate(b *testing.B) { + v := demoteCandidate{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshaldemoteCandidate(b *testing.B) { + v := demoteCandidate{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodedemoteCandidate(t *testing.T) { + v := demoteCandidate{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodedemoteCandidate Msgsize() is inaccurate") + } + + vn := demoteCandidate{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodedemoteCandidate(b *testing.B) { + v := demoteCandidate{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodedemoteCandidate(b *testing.B) { + v := demoteCandidate{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/ilm-access-tracker_test.go b/cmd/ilm-access-tracker_test.go new file mode 100644 index 0000000000000..32955ecb3439a --- /dev/null +++ b/cmd/ilm-access-tracker_test.go @@ -0,0 +1,95 @@ +// Copyright (c) 2015-2026 MinIO, Inc. + +package cmd + +import ( + "math" + "testing" + "time" + + "github.com/minio/minio/internal/config/ilm" +) + +func TestAccessEntryRollAndHits(t *testing.T) { + entry := accessEntry{Bins: []uint32{3, 2, 1}, HeadAt: 100, LastAt: 107} + entry.rollTo(120, 10, 3) + want := []uint32{0, 0, 3} + for i := range want { + if entry.Bins[i] != want[i] { + t.Fatalf("bins = %v, want %v", entry.Bins, want) + } + } + if entry.HeadAt != 120 || entry.LastAt != 107 { + t.Fatalf("head/last = %d/%d", entry.HeadAt, entry.LastAt) + } + + entry = accessEntry{Bins: []uint32{10, 20, 30}, HeadAt: 120} + if got := entry.hits(20*time.Second, 10); got != 30 { + t.Fatalf("20s hits = %d, want 30", got) + } + if got := entry.hits(21*time.Second, 10); got != 60 { + t.Fatalf("21s hits = %d, want 60", got) + } + entry.rollTo(200, 10, 3) + if got := entry.total(); got != 0 { + t.Fatalf("expired total = %d, want 0", got) + } +} + +func TestAccessEntryMergeSaturates(t *testing.T) { + entry := accessEntry{Bins: []uint32{math.MaxUint32 - 1}} + entry.mergeFrom(accessEntry{Bins: []uint32{10}, LastAt: 50}) + if entry.Bins[0] != math.MaxUint32 || entry.LastAt != 50 { + t.Fatalf("merged entry = %+v", entry) + } +} + +func TestAccessTrackerEvictsColdest(t *testing.T) { + tracker := newAccessTracker() + live := map[string]accessEntry{ + "a": {Bins: []uint32{1}, HeadAt: 100}, + "b": {Bins: []uint32{5}, HeadAt: 100}, + "c": {Bins: []uint32{3}, HeadAt: 100}, + } + tracker.evict(live, 100, 10, ilm.Config{AccessBins: 1, AccessMaxTracked: 2}) + if len(live) != 2 { + t.Fatalf("len = %d, want 2", len(live)) + } + if _, ok := live["a"]; ok { + t.Fatal("coldest entry was retained") + } +} + +func TestAccessKeyRoundTrip(t *testing.T) { + key := accessKey("bucket", "a/b/c") + bucket, object, ok := splitAccessKey(key) + if !ok || bucket != "bucket" || object != "a/b/c" { + t.Fatalf("split %q = %q/%q/%v", key, bucket, object, ok) + } +} + +func TestAccessShardFresh(t *testing.T) { + const now = int64(1000) + if !accessShardFresh(now, accessShard{UpdatedAt: 950}, 100) { + t.Fatal("fresh shard rejected") + } + if accessShardFresh(now, accessShard{UpdatedAt: 899}, 100) { + t.Fatal("stale shard accepted") + } + if accessShardFresh(now, accessShard{UpdatedAt: 1101}, 100) { + t.Fatal("far-future shard accepted") + } + if accessShardFresh(now, accessShard{}, 100) { + t.Fatal("zero timestamp accepted") + } +} + +func TestAccessTrackerRestoresDemoteCandidates(t *testing.T) { + tracker := newAccessTracker() + candidate := demoteCandidate{Bucket: "bucket", Object: "object", Pool: 1} + tracker.restoreDemoteCandidates([]demoteCandidate{candidate}) + got := tracker.takeDemoteCandidates() + if len(got) != 1 || got[0] != candidate { + t.Fatalf("restored candidates = %+v, want %+v", got, candidate) + } +} diff --git a/cmd/ilm-config.go b/cmd/ilm-config.go index 83e7e305b392c..03b4d0d7c1ffa 100644 --- a/cmd/ilm-config.go +++ b/cmd/ilm-config.go @@ -19,6 +19,7 @@ package cmd import ( "sync" + "time" "github.com/minio/minio/internal/config/ilm" ) @@ -27,6 +28,16 @@ var globalILMConfig = ilmConfig{ cfg: ilm.Config{ ExpirationWorkers: 100, TransitionWorkers: 100, + // Access tiering stays off until configured, but the counter + // geometry must be sane from the start: the tracker divides by + // AccessBinWidth before any config is loaded. + AccessPromoteWatermark: 85, + AccessBinWidth: time.Minute, + AccessBins: 12, + AccessFlush: time.Minute, + AccessMinResidency: 24 * time.Hour, + AccessWorkers: 10, + AccessMaxTracked: 1000000, }, } @@ -49,6 +60,25 @@ func (c *ilmConfig) getTransitionWorkers() int { return c.cfg.TransitionWorkers } +// accessCfg returns a copy of the access tiering settings. Callers take the +// whole struct rather than one getter per field because the promotion and +// demotion paths need a consistent view of several knobs at once. +func (c *ilmConfig) accessCfg() ilm.Config { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.cfg +} + +// accessTieringEnabled is the cheap gate used on the scanner path. +func (c *ilmConfig) accessTieringEnabled() bool { + c.mu.RLock() + defer c.mu.RUnlock() + + _, ok := c.cfg.HotPool() + return ok +} + func (c *ilmConfig) update(cfg ilm.Config) { c.mu.Lock() defer c.mu.Unlock() diff --git a/cmd/lceventsrc_string.go b/cmd/lceventsrc_string.go index 3d859dc405125..17cda8a14b978 100644 --- a/cmd/lceventsrc_string.go +++ b/cmd/lceventsrc_string.go @@ -19,11 +19,12 @@ func _() { _ = x[lcEventSrc_s3PutObject-8] _ = x[lcEventSrc_s3CopyObject-9] _ = x[lcEventSrc_s3CompleteMultipartUpload-10] + _ = x[lcEventSrc_AccessTier-11] } -const _lcEventSrc_name = "NoneHealScannerDecomRebals3HeadObjects3GetObjects3ListObjectss3PutObjects3CopyObjects3CompleteMultipartUpload" +const _lcEventSrc_name = "NoneHealScannerDecomRebals3HeadObjects3GetObjects3ListObjectss3PutObjects3CopyObjects3CompleteMultipartUploadAccessTier" -var _lcEventSrc_index = [...]uint8{0, 4, 8, 15, 20, 25, 37, 48, 61, 72, 84, 109} +var _lcEventSrc_index = [...]uint8{0, 4, 8, 15, 20, 25, 37, 48, 61, 72, 84, 109, 119} func (i lcEventSrc) String() string { idx := int(i) - 0 diff --git a/cmd/metrics-v3-ilm.go b/cmd/metrics-v3-ilm.go index 604beaae78cb2..f4beed6820455 100644 --- a/cmd/metrics-v3-ilm.go +++ b/cmd/metrics-v3-ilm.go @@ -26,6 +26,17 @@ const ( transitionActiveTasks = "transition_active_tasks" transitionPendingTasks = "transition_pending_tasks" transitionMissedImmediateTasks = "transition_missed_immediate_tasks" + accessTierActiveTasks = "access_tier_active_tasks" + accessTierPendingTasks = "access_tier_pending_tasks" + accessTierPromotionsTotal = "access_tier_promotions_total" + accessTierDemotionsTotal = "access_tier_demotions_total" + accessTierBytesMovedTotal = "access_tier_bytes_moved_total" + accessTierFailuresTotal = "access_tier_failures_total" + accessTierSkippedWatermark = "access_tier_skipped_watermark_total" + accessTierSkippedMaxSize = "access_tier_skipped_max_size_total" + accessTierSkippedQuota = "access_tier_skipped_bucket_quota_total" + accessTierHotBytes = "access_tier_hot_bytes" + accessTierSamplesDropped = "access_tier_samples_dropped_total" versionsScanned = "versions_scanned" ) @@ -34,6 +45,17 @@ var ( ilmTransitionActiveTasksMD = NewGaugeMD(transitionActiveTasks, "Number of active ILM transition tasks") ilmTransitionPendingTasksMD = NewGaugeMD(transitionPendingTasks, "Number of pending ILM transition tasks in the queue") ilmTransitionMissedImmediateTasksMD = NewCounterMD(transitionMissedImmediateTasks, "Number of missed immediate ILM transition tasks") + ilmAccessTierActiveTasksMD = NewGaugeMD(accessTierActiveTasks, "Number of active access-tier pool moves") + ilmAccessTierPendingTasksMD = NewGaugeMD(accessTierPendingTasks, "Number of pending access-tier pool moves") + ilmAccessTierPromotionsTotalMD = NewCounterMD(accessTierPromotionsTotal, "Total objects promoted by access-tier ILM") + ilmAccessTierDemotionsTotalMD = NewCounterMD(accessTierDemotionsTotal, "Total objects demoted by access-tier ILM") + ilmAccessTierBytesMovedTotalMD = NewCounterMD(accessTierBytesMovedTotal, "Total logical bytes moved by access-tier ILM") + ilmAccessTierFailuresTotalMD = NewCounterMD(accessTierFailuresTotal, "Total failed access-tier ILM moves") + ilmAccessTierSkippedWatermarkMD = NewCounterMD(accessTierSkippedWatermark, "Promotions skipped because the hot pool reached its watermark") + ilmAccessTierSkippedMaxSizeMD = NewCounterMD(accessTierSkippedMaxSize, "Promotions skipped because the cluster hot-tier size cap was reached") + ilmAccessTierSkippedQuotaMD = NewCounterMD(accessTierSkippedQuota, "Promotions skipped because the bucket hot-tier quota was reached") + ilmAccessTierHotBytesMD = NewGaugeMD(accessTierHotBytes, "Logical bytes currently accounted to the hot tier", "bucket") + ilmAccessTierSamplesDroppedMD = NewCounterMD(accessTierSamplesDropped, "GET samples dropped because the access tracker queue was full") ilmVersionsScannedMD = NewCounterMD(versionsScanned, "Total number of object versions checked for ILM actions since server start") ) @@ -47,6 +69,21 @@ func loadILMMetrics(_ context.Context, m MetricValues, _ *metricsCache) error { m.Set(transitionPendingTasks, float64(globalTransitionState.PendingTasks())) m.Set(transitionMissedImmediateTasks, float64(globalTransitionState.MissedImmediateTasks())) } + if globalAccessTierState != nil { + m.Set(accessTierActiveTasks, float64(globalAccessTierState.ActiveTasks())) + m.Set(accessTierPendingTasks, float64(globalAccessTierState.PendingTasks())) + m.Set(accessTierPromotionsTotal, float64(globalAccessTierState.promotions.Load())) + m.Set(accessTierDemotionsTotal, float64(globalAccessTierState.demotions.Load())) + m.Set(accessTierBytesMovedTotal, float64(globalAccessTierState.bytesMoved.Load())) + m.Set(accessTierFailuresTotal, float64(globalAccessTierState.failures.Load())) + m.Set(accessTierSkippedWatermark, float64(globalAccessTierState.skippedWatermark.Load())) + m.Set(accessTierSkippedMaxSize, float64(globalAccessTierState.skippedMaxSize.Load())) + m.Set(accessTierSkippedQuota, float64(globalAccessTierState.skippedQuota.Load())) + for bucket, bytes := range globalAccessTierState.hotUsageSnapshot() { + m.Set(accessTierHotBytes, float64(bytes), "bucket", bucket) + } + } + m.Set(accessTierSamplesDropped, float64(globalAccessTracker.dropped.Load())) m.Set(versionsScanned, float64(globalScannerMetrics.lifetime(scannerMetricILM))) return nil diff --git a/cmd/metrics-v3.go b/cmd/metrics-v3.go index 93749258cb11a..5b578448d4364 100644 --- a/cmd/metrics-v3.go +++ b/cmd/metrics-v3.go @@ -390,6 +390,17 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection { ilmTransitionActiveTasksMD, ilmTransitionPendingTasksMD, ilmTransitionMissedImmediateTasksMD, + ilmAccessTierActiveTasksMD, + ilmAccessTierPendingTasksMD, + ilmAccessTierPromotionsTotalMD, + ilmAccessTierDemotionsTotalMD, + ilmAccessTierBytesMovedTotalMD, + ilmAccessTierFailuresTotalMD, + ilmAccessTierSkippedWatermarkMD, + ilmAccessTierSkippedMaxSizeMD, + ilmAccessTierSkippedQuotaMD, + ilmAccessTierHotBytesMD, + ilmAccessTierSamplesDroppedMD, ilmVersionsScannedMD, }, loadILMMetrics, diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go index 06c73741078d3..565b10fd134b7 100644 --- a/cmd/object-api-interface.go +++ b/cmd/object-api-interface.go @@ -117,6 +117,10 @@ type ObjectOptions struct { SkipRebalancing bool SrcPoolIdx int // set by PutObject/CompleteMultipart operations due to rebalance; used to prevent rebalance src, dst pools to be the same + // DstPoolIdx forces a data-movement write onto a specific server pool. + // It is ignored unless DataMovement is true; a pointer keeps pool zero + // distinguishable from the unset value. + DstPoolIdx *int DataMovement bool // indicates an going decommisionning or rebalacing diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 296817ed5352e..4639833c9095c 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -563,6 +563,8 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj return } + globalAccessTracker.note(bucket, object) + // Notify object accessed via a GET request. sendEvent(eventArgs{ EventName: event.ObjectAccessedGet, diff --git a/cmd/server-main.go b/cmd/server-main.go index a547581f78a67..60b1ce29dc26e 100644 --- a/cmd/server-main.go +++ b/cmd/server-main.go @@ -500,6 +500,7 @@ func initAllSubsystems(ctx context.Context) { globalTierConfigMgr = NewTierConfigMgr() globalTransitionState = newTransitionState(GlobalContext) + globalAccessTierState = newAccessTierState(GlobalContext) globalSiteResyncMetrics = newSiteResyncMetrics(GlobalContext) } @@ -1057,6 +1058,10 @@ func serverMain(ctx *cli.Context) { bootstrapTrace("globalTransitionState.Init", func() { globalTransitionState.Init(newObject) }) + bootstrapTrace("globalAccessTierState.Init", func() { + globalAccessTierState.Init(newObject) + go globalAccessTracker.run(GlobalContext, newObject) + }) go func() { // Initialize transition tier configuration manager diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index f3a6231c3daab..4049f7dc55c1f 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -583,6 +583,7 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates } poolIdx, setIdx, _ := s.GetDiskLoc() + hotPool, hotPoolOK := globalILMConfig.accessCfg().HotPool() disks, err := objAPI.GetDisks(poolIdx, setIdx) if err != nil { @@ -592,6 +593,7 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates cache.Info.updates = updates dataUsageInfo, err := scanDataFolder(ctx, disks, s, cache, func(item scannerItem) (sizeSummary, error) { + item.poolIdx = poolIdx // Look for `xl.meta/xl.json' at the leaf. if !strings.HasSuffix(item.Path, SlashSeparator+xlStorageFormatFile) && !strings.HasSuffix(item.Path, SlashSeparator+xlStorageFormatFileV1) { @@ -655,6 +657,9 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates sizeS.versions++ } sizeS.totalSize += sz + if hotPoolOK && poolIdx == hotPool { + sizeS.hotTierSize += sz + } // Skip tier accounting if object version is a delete-marker or a free-version // tracking deleted transitioned objects diff --git a/docs/bucket/lifecycle/README.md b/docs/bucket/lifecycle/README.md index de63333695612..3e5f5d5dd49dc 100644 --- a/docs/bucket/lifecycle/README.md +++ b/docs/bucket/lifecycle/README.md @@ -226,6 +226,128 @@ aws s3api restore-object --bucket srcbucket \ Note that transition event notification is a Silo extension. +## 5. Access-based tiering between server pools + +Silo can move frequently read objects to a faster server pool and return them +to a slower pool after they become idle. This is different from remote ILM +transition: the object remains a native local object and all versions move +together. + +Access tiering requires at least two server pools. Pool indices follow the +order on the server command line: + +~~~sh +silo server /srv/nvme{1...4} /srv/hdd{1...8} +# pool 0 (fast) pool 1 (slow) +~~~ + +Server pools are erasure-coding expansion units, not individual drives. Each +pool should consist of internally homogeneous media. + +The feature is disabled by default. Configure the topology and safety limits +with `mc admin config set`; ILM is a dynamic subsystem, so this applies without +a restart. Environment variables (`MINIO_ILM_ACCESS_TIERING`, +`MINIO_ILM_ACCESS_POOLS`, `MINIO_ILM_ACCESS_MAX_SIZE`, +`MINIO_ILM_ACCESS_PROMOTE_WATERMARK`, `MINIO_ILM_ACCESS_BIN_WIDTH`, +`MINIO_ILM_ACCESS_BINS`, `MINIO_ILM_ACCESS_FLUSH`, +`MINIO_ILM_ACCESS_MIN_RESIDENCY`, `MINIO_ILM_ACCESS_WORKERS`, +`MINIO_ILM_ACCESS_MAX_TRACKED`) are read at process start and override the +stored config. + +~~~sh +mc admin config set local ilm \ + access_tiering=on \ + access_pools="0,1" \ + access_max_size="2TiB" \ + access_promote_watermark=85 \ + access_bin_width=1m \ + access_bins=12 \ + access_flush=1m \ + access_min_residency=24h \ + access_workers=10 \ + access_max_tracked=1000000 +~~~ + +The pool list is ordered hottest to coldest. With three or more pools, +promotion always targets the first index and demotion always targets the last; +intermediate pools are not hop targets. An access_max_size value of zero means +no cluster-wide logical-byte cap. Promotion also stops when the hottest pool +reaches access_promote_watermark. + +New PUTs still land via the usual free-space pool picker; they are not steered +onto the cold pool. Size the capacity pool larger than the hot pool so new +objects tend to land there. + +Add an AccessTransition to the bucket lifecycle XML: + +~~~xml + + 500GiB + + hot-logs + Enabled + + + logs/ + 65536 + + + + 10m + 100 + 5 + 24h + + + +~~~ + +This promotes a matching object after 100 successful GETs in 10 minutes. An +object becomes eligible to return to the coldest configured pool only after +access tiering has already moved it (the `x-minio-internal-ilm-atier` stamp), +it has stayed put for the server-wide minimum residency, it has been idle at +least 24 hours, and it has no more than 5 GETs in the window. Objects that +landed on the hot pool via a normal PUT never demote. Prefix, tag, and +object-size lifecycle filters are honored. + +Access-based moves are a parallel path: they are not lifecycle `Eval` actions +and do not appear in S3 prediction headers. If the same object is also due +for age-based remote `Transition` or expiry, that scanner action wins and +demotion discovery is skipped for that pass; promotions still run from the +GET tracker. Site replication copies expiry rules only, same as remote +Transition, so AccessTransition stays local to the cluster. + +AccessTierQuota is an optional bucket-wide cap. Promotion checks, in order: + +1. hot-pool used percentage; +2. cluster-wide access_max_size; +3. bucket AccessTierQuota. + +Demotion is not blocked by these caps and is processed before promotion. +Access moves pause during rebalance or decommission, never target a suspended +pool, skip remotely transitioned objects and objects with excessive version +counts, and recheck eligibility while holding the object namespace lock. + +The hit counter is intentionally best effort. Only successfully served GET +requests count; HEAD requests do not. Counters are merged across nodes and +bounded by access_max_tracked. A rule window longer than +access_bin_width multiplied by access_bins is clamped to retained history. + +AccessTransition and AccessTierQuota are Silo lifecycle extensions. A stock +AWS SDK that reads and rewrites the lifecycle configuration may discard +unknown fields. Use a raw signed S3 PUT lifecycle request, such as +[setup_ilm_access_tiering.sh](setup_ilm_access_tiering.sh), when installing +the rule. Save the XML above as `rule.xml`, then run: + +~~~sh +AWS_ACCESS_KEY_ID=minioadmin AWS_SECRET_ACCESS_KEY=minioadmin \ + ./setup_ilm_access_tiering.sh http://127.0.0.1:9000 testbucket us-east-1 rule.xml +~~~ + +Access-tier activity is exposed under /minio/metrics/v3/ilm, including move +counts, moved bytes, queue depth, hot bytes per bucket, failed moves, dropped +GET samples, and separate skip counters for each capacity limit. + ## Explore Further - [MinIO Go client API reference (S3-compatible SDK)](https://pkg.go.dev/github.com/minio/minio-go/v7) diff --git a/docs/bucket/lifecycle/setup_ilm_access_tiering.sh b/docs/bucket/lifecycle/setup_ilm_access_tiering.sh new file mode 100755 index 0000000000000..250a10f882392 --- /dev/null +++ b/docs/bucket/lifecycle/setup_ilm_access_tiering.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +# Install a lifecycle XML document without an SDK normalizing away Silo's +# AccessTransition and AccessTierQuota extension elements. +set -eu + +if [ "$#" -ne 4 ]; then + echo "usage: AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... $0 ENDPOINT BUCKET REGION LIFECYCLE_XML" >&2 + exit 2 +fi + +endpoint=$1 +bucket=$2 +region=$3 +lifecycle_file=$4 + +: "$AWS_ACCESS_KEY_ID" +: "$AWS_SECRET_ACCESS_KEY" + +if [ ! -r "$lifecycle_file" ]; then + echo "cannot read lifecycle document: $lifecycle_file" >&2 + exit 2 +fi + +content_md5=$(openssl dgst -md5 -binary "$lifecycle_file" | openssl base64) +endpoint=$(printf '%s' "$endpoint" | sed 's:/*$::') + +curl --fail-with-body --silent --show-error \ + --request PUT \ + --aws-sigv4 "aws:amz:$region:s3" \ + --user "$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY" \ + --header "Content-MD5: $content_md5" \ + --header "Content-Type: application/xml" \ + --data-binary "@$lifecycle_file" \ + "$endpoint/$bucket?lifecycle" + +echo "installed lifecycle configuration on $bucket" diff --git a/docs/metrics/prometheus/list.md b/docs/metrics/prometheus/list.md index ec8cd6c208543..050e2577bc10f 100644 --- a/docs/metrics/prometheus/list.md +++ b/docs/metrics/prometheus/list.md @@ -225,6 +225,17 @@ For deployments with [bucket](https://silo.pgsty.com/administration/bucket-repli | `minio_node_ilm_transition_active_tasks` | Number of active ILM transition tasks. | | `minio_node_ilm_transition_pending_tasks` | Number of pending ILM transition tasks in the queue. | | `minio_node_ilm_transition_missed_immediate_tasks` | Number of missed immediate ILM transition tasks. | +| `minio_node_ilm_access_tier_active_tasks` | Number of active access-tier pool moves. | +| `minio_node_ilm_access_tier_pending_tasks` | Number of pending access-tier pool moves. | +| `minio_node_ilm_access_tier_promotions_total` | Total objects promoted by access-tier ILM. | +| `minio_node_ilm_access_tier_demotions_total` | Total objects demoted by access-tier ILM. | +| `minio_node_ilm_access_tier_bytes_moved_total` | Total logical bytes moved by access-tier ILM. | +| `minio_node_ilm_access_tier_failures_total` | Total failed access-tier ILM moves. | +| `minio_node_ilm_access_tier_skipped_watermark_total` | Promotions skipped because the hot pool reached its watermark. | +| `minio_node_ilm_access_tier_skipped_max_size_total` | Promotions skipped because the cluster hot-tier size cap was reached. | +| `minio_node_ilm_access_tier_skipped_bucket_quota_total` | Promotions skipped because the bucket hot-tier quota was reached. | +| `minio_node_ilm_access_tier_hot_bytes` | Logical bytes currently accounted to the hot tier, labeled by bucket. | +| `minio_node_ilm_access_tier_samples_dropped_total` | GET samples dropped because the access tracker queue was full. | | `minio_node_ilm_versions_scanned` | Total number of object versions checked for ilm actions since server start. | | `minio_node_ilm_action_count_delete_action` | Total action outcome of lifecycle checks since server start for deleting object | | `minio_node_ilm_action_count_delete_version_action` | Total action outcome of lifecycle checks since server start for deleting a version | diff --git a/internal/bucket/lifecycle/access-transition.go b/internal/bucket/lifecycle/access-transition.go new file mode 100644 index 0000000000000..ebdf9f325f499 --- /dev/null +++ b/internal/bucket/lifecycle/access-transition.go @@ -0,0 +1,129 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "encoding/xml" + "time" +) + +var ( + errAccessInvalidDuration = Errorf("Window and DemoteAfterIdle must be valid Go durations, e.g. 10m or 24h") + errAccessInvalidWindow = Errorf("Window must be a positive duration with AccessTransition") + errAccessInvalidPromote = Errorf("PromoteAfterAccesses must be a positive integer with AccessTransition") + errAccessInvalidDemote = Errorf("DemoteAfterAccesses must be smaller than PromoteAfterAccesses and 0 or greater") + errAccessInvalidIdle = Errorf("DemoteAfterIdle must be a positive duration no shorter than Window") + errAccessInvalidQuotaSize = Errorf("AccessTierQuota must be a valid size, e.g. 500GiB") +) + +// Duration is a time.Duration that marshals to and from an XML element +// holding a Go duration string, e.g. 10m. +type Duration time.Duration + +// UnmarshalXML parses a duration string such as "10m" or "24h". +func (d *Duration) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) error { + var s string + if err := dec.DecodeElement(&s, &start); err != nil { + return err + } + dur, err := time.ParseDuration(s) + if err != nil { + return errAccessInvalidDuration + } + *d = Duration(dur) + return nil +} + +// MarshalXML encodes a non-zero duration, and nothing otherwise. +func (d Duration) MarshalXML(enc *xml.Encoder, start xml.StartElement) error { + if d == 0 { + return nil + } + return enc.EncodeElement(time.Duration(d).String(), start) +} + +// D returns the value as a time.Duration. +func (d Duration) D() time.Duration { + return time.Duration(d) +} + +// AccessTransition is a Silo extension to the S3 lifecycle rule. It relocates +// an object between server pools based on how often it is read, rather than on +// its age: an object read at least PromoteAfterAccesses times within Window +// moves to the fastest configured pool, and moves back once it has been idle +// for DemoteAfterIdle and its windowed hit count has fallen to +// DemoteAfterAccesses or below. +// +// The gap between the two thresholds, together with DemoteAfterIdle and the +// server-side access_min_residency, is what keeps an object from oscillating +// between pools. +type AccessTransition struct { + XMLName xml.Name `xml:"AccessTransition"` + Window Duration `xml:"Window,omitempty"` + PromoteAfterAccesses int `xml:"PromoteAfterAccesses,omitempty"` + DemoteAfterAccesses int `xml:"DemoteAfterAccesses,omitempty"` + DemoteAfterIdle Duration `xml:"DemoteAfterIdle,omitempty"` + + set bool +} + +// IsNull returns true if no usable access transition is configured. +func (a AccessTransition) IsNull() bool { + return !a.set || a.PromoteAfterAccesses <= 0 +} + +// MarshalXML encodes an AccessTransition element, and nothing if unset. +func (a AccessTransition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error { + if !a.set { + return nil + } + type accessTransitionWrapper AccessTransition + return enc.EncodeElement(accessTransitionWrapper(a), start) +} + +// UnmarshalXML decodes an AccessTransition element. +func (a *AccessTransition) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) error { + type accessTransitionWrapper AccessTransition + var atw accessTransitionWrapper + if err := dec.DecodeElement(&atw, &start); err != nil { + return err + } + *a = AccessTransition(atw) + a.set = true + return nil +} + +// Validate checks the AccessTransition element. +func (a AccessTransition) Validate() error { + if !a.set { + return nil + } + if a.Window <= 0 { + return errAccessInvalidWindow + } + if a.PromoteAfterAccesses <= 0 { + return errAccessInvalidPromote + } + if a.DemoteAfterAccesses < 0 || a.DemoteAfterAccesses >= a.PromoteAfterAccesses { + return errAccessInvalidDemote + } + if a.DemoteAfterIdle <= 0 || a.DemoteAfterIdle < a.Window { + return errAccessInvalidIdle + } + return nil +} diff --git a/internal/bucket/lifecycle/access-transition_test.go b/internal/bucket/lifecycle/access-transition_test.go new file mode 100644 index 0000000000000..dd54146108069 --- /dev/null +++ b/internal/bucket/lifecycle/access-transition_test.go @@ -0,0 +1,219 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "bytes" + "encoding/xml" + "strings" + "testing" + "time" + + "github.com/minio/minio/internal/bucket/object/lock" +) + +const accessTieringXML = ` + 500GiB + + hot-logs + Enabled + logs/65536 + + 10m + 100 + 5 + 24h + + +` + +func TestAccessTransitionParse(t *testing.T) { + lc, err := ParseLifecycleConfig(strings.NewReader(accessTieringXML)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if err := lc.Validate(lock.Retention{}); err != nil { + t.Fatalf("validate: %v", err) + } + if got := lc.AccessQuotaBytes(); got != 500*1024*1024*1024 { + t.Fatalf("quota = %d, want %d", got, 500*1024*1024*1024) + } + if !lc.HasAccessTransition() { + t.Fatal("HasAccessTransition = false, want true") + } + at := lc.Rules[0].AccessTransition + if at.Window.D() != 10*time.Minute { + t.Fatalf("window = %v, want 10m", at.Window.D()) + } + if at.DemoteAfterIdle.D() != 24*time.Hour { + t.Fatalf("idle = %v, want 24h", at.DemoteAfterIdle.D()) + } + if at.PromoteAfterAccesses != 100 || at.DemoteAfterAccesses != 5 { + t.Fatalf("thresholds = %d/%d, want 100/5", at.PromoteAfterAccesses, at.DemoteAfterAccesses) + } +} + +// A round trip through Marshal must preserve both the rule element and the +// bucket-wide quota, since PutBucketLifecycle stores whatever we re-encode. +func TestAccessTransitionRoundTrip(t *testing.T) { + lc, err := ParseLifecycleConfig(strings.NewReader(accessTieringXML)) + if err != nil { + t.Fatalf("parse: %v", err) + } + buf, err := xml.Marshal(lc) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if !bytes.Contains(buf, []byte("500GiB")) { + t.Fatalf("quota lost in round trip: %s", buf) + } + got, err := ParseLifecycleConfig(bytes.NewReader(buf)) + if err != nil { + t.Fatalf("reparse: %v", err) + } + if got.AccessQuotaBytes() != lc.AccessQuotaBytes() { + t.Fatalf("quota %d != %d", got.AccessQuotaBytes(), lc.AccessQuotaBytes()) + } + if got.Rules[0].AccessTransition != lc.Rules[0].AccessTransition { + t.Fatalf("rule %+v != %+v", got.Rules[0].AccessTransition, lc.Rules[0].AccessTransition) + } +} + +// A rule with no AccessTransition must not emit an empty element - otherwise +// every existing lifecycle config would change shape on rewrite. +func TestAccessTransitionUnsetNotMarshalled(t *testing.T) { + lc, err := ParseLifecycleConfig(strings.NewReader(` + oldEnableda/ + 3`)) + if err != nil { + t.Fatalf("parse: %v", err) + } + buf, err := xml.Marshal(lc) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if bytes.Contains(buf, []byte("AccessTransition")) || bytes.Contains(buf, []byte("AccessTierQuota")) { + t.Fatalf("unset elements emitted: %s", buf) + } + if lc.HasAccessTransition() { + t.Fatal("HasAccessTransition = true for a plain expiry rule") + } +} + +func TestAccessTransitionValidate(t *testing.T) { + tests := []struct { + name string + at AccessTransition + err error + }{ + {"ok", AccessTransition{Window: Duration(10 * time.Minute), PromoteAfterAccesses: 100, DemoteAfterAccesses: 5, DemoteAfterIdle: Duration(time.Hour), set: true}, nil}, + {"unset", AccessTransition{}, nil}, + {"zero window", AccessTransition{PromoteAfterAccesses: 100, DemoteAfterIdle: Duration(time.Hour), set: true}, errAccessInvalidWindow}, + {"zero promote", AccessTransition{Window: Duration(time.Minute), DemoteAfterIdle: Duration(time.Hour), set: true}, errAccessInvalidPromote}, + {"demote >= promote", AccessTransition{Window: Duration(time.Minute), PromoteAfterAccesses: 5, DemoteAfterAccesses: 5, DemoteAfterIdle: Duration(time.Hour), set: true}, errAccessInvalidDemote}, + {"negative demote", AccessTransition{Window: Duration(time.Minute), PromoteAfterAccesses: 5, DemoteAfterAccesses: -1, DemoteAfterIdle: Duration(time.Hour), set: true}, errAccessInvalidDemote}, + {"idle shorter than window", AccessTransition{Window: Duration(time.Hour), PromoteAfterAccesses: 5, DemoteAfterIdle: Duration(time.Minute), set: true}, errAccessInvalidIdle}, + {"zero idle", AccessTransition{Window: Duration(time.Minute), PromoteAfterAccesses: 5, set: true}, errAccessInvalidIdle}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if err := tc.at.Validate(); err != tc.err { + t.Fatalf("err = %v, want %v", err, tc.err) + } + }) + } +} + +func TestAccessTierQuotaInvalid(t *testing.T) { + lc, err := ParseLifecycleConfig(strings.NewReader(` + not-a-size + rEnabled3 + `)) + if err != nil { + t.Fatalf("parse: %v", err) + } + if err := lc.Validate(lock.Retention{}); err != errAccessInvalidQuotaSize { + t.Fatalf("err = %v, want %v", err, errAccessInvalidQuotaSize) + } + // An invalid quota that somehow reached the evaluator means "unlimited", + // never "zero bytes allowed". + if got := lc.AccessQuotaBytes(); got != 0 { + t.Fatalf("quota = %d, want 0 (unlimited)", got) + } +} + +func TestAccessTransitionBadDuration(t *testing.T) { + _, err := ParseLifecycleConfig(strings.NewReader(` + rEnabled + ten minutes1 + `)) + if err == nil { + t.Fatal("expected a parse error for a malformed duration") + } +} + +// AccessRule must reuse the standard rule filtering: prefix, tags, size and +// Status all have to be honored. +func TestAccessRuleFiltering(t *testing.T) { + lc, err := ParseLifecycleConfig(strings.NewReader(accessTieringXML)) + if err != nil { + t.Fatalf("parse: %v", err) + } + tests := []struct { + name string + obj ObjectOpts + want bool + }{ + {"match", ObjectOpts{Name: "logs/a.log", Size: 1 << 20, IsLatest: true}, true}, + {"wrong prefix", ObjectOpts{Name: "data/a.log", Size: 1 << 20, IsLatest: true}, false}, + {"too small", ObjectOpts{Name: "logs/a.log", Size: 1024, IsLatest: true}, false}, + {"no name", ObjectOpts{Size: 1 << 20}, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, id, ok := lc.AccessRule(tc.obj) + if ok != tc.want { + t.Fatalf("ok = %v, want %v", ok, tc.want) + } + if ok && id != "hot-logs" { + t.Fatalf("ruleID = %q, want hot-logs", id) + } + }) + } + + lc.Rules[0].Status = Disabled + if _, _, ok := lc.AccessRule(ObjectOpts{Name: "logs/a.log", Size: 1 << 20, IsLatest: true}); ok { + t.Fatal("disabled rule still matched") + } + if lc.HasAccessTransition() { + t.Fatal("HasAccessTransition = true with only a disabled rule") + } +} + +func TestAccessTransitionCountsAsActiveRule(t *testing.T) { + lc, err := ParseLifecycleConfig(strings.NewReader(accessTieringXML)) + if err != nil { + t.Fatal(err) + } + if !lc.HasActiveRules("logs/2026") { + t.Fatal("access-only lifecycle rule was not active for its prefix") + } + if lc.HasActiveRules("data/") { + t.Fatal("access rule was active outside its prefix") + } +} diff --git a/internal/bucket/lifecycle/lifecycle.go b/internal/bucket/lifecycle/lifecycle.go index 97c4200f27762..5ee9e9e3723d5 100644 --- a/internal/bucket/lifecycle/lifecycle.go +++ b/internal/bucket/lifecycle/lifecycle.go @@ -26,6 +26,7 @@ import ( "strings" "time" + "github.com/dustin/go-humanize" "github.com/google/uuid" "github.com/minio/minio/internal/bucket/object/lock" "github.com/minio/minio/internal/bucket/replication" @@ -101,11 +102,56 @@ func (a Action) Delete() bool { // Lifecycle - Configuration for bucket lifecycle. type Lifecycle struct { - XMLName xml.Name `xml:"LifecycleConfiguration"` + XMLName xml.Name `xml:"LifecycleConfiguration"` + // AccessTierQuota caps how many bytes of this bucket access-based ILM + // may keep on the fastest pool, e.g. "500GiB". Empty means unlimited. + // It is bucket-wide rather than per-rule so it cannot be declared + // inconsistently by two rules matching the same object. + AccessTierQuota string `xml:"AccessTierQuota,omitempty"` Rules []Rule `xml:"Rule"` ExpiryUpdatedAt *time.Time `xml:"ExpiryUpdatedAt,omitempty"` } +// HasAccessTransition returns 'true' if any enabled rule carries a usable +// AccessTransition. Used as a cheap per-bucket gate before consulting the +// access tracker. +func (lc Lifecycle) HasAccessTransition() bool { + for _, rule := range lc.Rules { + if rule.Status == Disabled { + continue + } + if !rule.AccessTransition.IsNull() { + return true + } + } + return false +} + +// AccessRule returns the first enabled rule matching obj that carries a usable +// AccessTransition, along with its rule ID. +func (lc Lifecycle) AccessRule(obj ObjectOpts) (AccessTransition, string, bool) { + for _, rule := range lc.FilterRules(obj) { + if !rule.AccessTransition.IsNull() { + return rule.AccessTransition, rule.ID, true + } + } + return AccessTransition{}, "", false +} + +// AccessQuotaBytes returns the bucket-wide fast-pool byte cap, 0 meaning +// unlimited. The value is validated at PUT time, so a parse failure here is +// treated as unlimited rather than as an error. +func (lc Lifecycle) AccessQuotaBytes() uint64 { + if lc.AccessTierQuota == "" { + return 0 + } + sz, err := humanize.ParseBytes(lc.AccessTierQuota) + if err != nil { + return 0 + } + return sz +} + // HasTransition returns 'true' if lifecycle document has Transition enabled. func (lc Lifecycle) HasTransition() bool { for _, rule := range lc.Rules { @@ -158,6 +204,12 @@ func (lc *Lifecycle) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err e return err } lc.ExpiryUpdatedAt = &t + case "AccessTierQuota": + var q string + if err = d.DecodeElement(&q, &se); err != nil { + return err + } + lc.AccessTierQuota = q default: return xml.UnmarshalError(fmt.Sprintf("expected element type but have <%s>", se.Name.Local)) } @@ -208,6 +260,9 @@ func (lc Lifecycle) HasActiveRules(prefix string) bool { if !rule.Transition.IsNull() { // this allows for Transition.Days to be zero. return true } + if !rule.AccessTransition.IsNull() { + return true + } } return false } @@ -248,6 +303,12 @@ func (lc Lifecycle) Validate(lr lock.Retention) error { return errLifecycleNoRule } + if lc.AccessTierQuota != "" { + if _, err := humanize.ParseBytes(lc.AccessTierQuota); err != nil { + return errAccessInvalidQuotaSize + } + } + // Validate all the rules in the lifecycle config for _, r := range lc.Rules { if err := r.Validate(); err != nil { diff --git a/internal/bucket/lifecycle/rule.go b/internal/bucket/lifecycle/rule.go index 2fb006066ca05..0ca5669a3959c 100644 --- a/internal/bucket/lifecycle/rule.go +++ b/internal/bucket/lifecycle/rule.go @@ -41,6 +41,7 @@ type Rule struct { Expiration Expiration `xml:"Expiration,omitempty"` Transition Transition `xml:"Transition,omitempty"` DelMarkerExpiration DelMarkerExpiration `xml:"DelMarkerExpiration,omitempty"` + AccessTransition AccessTransition `xml:"AccessTransition,omitempty"` // FIXME: add a type to catch unsupported AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"` NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"` NoncurrentVersionTransition NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty"` @@ -171,10 +172,13 @@ func (r Rule) Validate() error { if err := r.validateNoncurrentTransition(); err != nil { return err } + if err := r.AccessTransition.Validate(); err != nil { + return err + } if (!r.Filter.Tag.IsEmpty() || len(r.Filter.And.Tags) != 0) && !r.DelMarkerExpiration.Empty() { return errInvalidRuleDelMarkerExpiration } - if !r.Expiration.set && !r.Transition.set && !r.NoncurrentVersionExpiration.set && !r.NoncurrentVersionTransition.set && r.DelMarkerExpiration.Empty() { + if !r.Expiration.set && !r.Transition.set && !r.NoncurrentVersionExpiration.set && !r.NoncurrentVersionTransition.set && r.DelMarkerExpiration.Empty() && !r.AccessTransition.set { return errXMLNotWellFormed } return nil diff --git a/internal/config/ilm/help.go b/internal/config/ilm/help.go index 9cbf30b157679..ceec4bb035015 100644 --- a/internal/config/ilm/help.go +++ b/internal/config/ilm/help.go @@ -22,10 +22,43 @@ import "github.com/minio/minio/internal/config" const ( transitionWorkers = "transition_workers" expirationWorkers = "expiration_workers" + + accessTiering = "access_tiering" + accessPools = "access_pools" + accessMaxSize = "access_max_size" + accessPromoteWatermark = "access_promote_watermark" + accessBinWidth = "access_bin_width" + accessBins = "access_bins" + accessFlush = "access_flush" + accessMinResidency = "access_min_residency" + accessWorkers = "access_workers" + accessMaxTracked = "access_max_tracked" + // EnvILMTransitionWorkers env variable to configure number of transition workers EnvILMTransitionWorkers = "MINIO_ILM_TRANSITION_WORKERS" // EnvILMExpirationWorkers env variable to configure number of expiration workers EnvILMExpirationWorkers = "MINIO_ILM_EXPIRATION_WORKERS" + + // EnvILMAccessTiering env variable to enable access based tiering + EnvILMAccessTiering = "MINIO_ILM_ACCESS_TIERING" + // EnvILMAccessPools env variable listing pool indices hottest first + EnvILMAccessPools = "MINIO_ILM_ACCESS_POOLS" + // EnvILMAccessMaxSize env variable capping bytes held on the hottest pool + EnvILMAccessMaxSize = "MINIO_ILM_ACCESS_MAX_SIZE" + // EnvILMAccessPromoteWatermark env variable for the hottest pool fill limit + EnvILMAccessPromoteWatermark = "MINIO_ILM_ACCESS_PROMOTE_WATERMARK" + // EnvILMAccessBinWidth env variable for the access counter resolution + EnvILMAccessBinWidth = "MINIO_ILM_ACCESS_BIN_WIDTH" + // EnvILMAccessBins env variable for the number of access counter bins + EnvILMAccessBins = "MINIO_ILM_ACCESS_BINS" + // EnvILMAccessFlush env variable for the counter publish and sweep interval + EnvILMAccessFlush = "MINIO_ILM_ACCESS_FLUSH" + // EnvILMAccessMinResidency env variable for the anti-thrash floor + EnvILMAccessMinResidency = "MINIO_ILM_ACCESS_MIN_RESIDENCY" + // EnvILMAccessWorkers env variable to configure number of access tiering workers + EnvILMAccessWorkers = "MINIO_ILM_ACCESS_WORKERS" + // EnvILMAccessMaxTracked env variable capping tracked objects per node + EnvILMAccessMaxTracked = "MINIO_ILM_ACCESS_MAX_TRACKED" ) var ( @@ -48,5 +81,65 @@ var ( Description: `set the number of expiration workers` + defaultHelpPostfix(expirationWorkers), Optional: true, }, + config.HelpKV{ + Key: accessTiering, + Type: "on|off", + Description: `move objects between fast and slow server pools based on how often they are read` + defaultHelpPostfix(accessTiering), + Optional: true, + }, + config.HelpKV{ + Key: accessPools, + Type: "string", + Description: `server pool indices for access tiering, hottest first, e.g. "0,1"` + defaultHelpPostfix(accessPools), + Optional: true, + }, + config.HelpKV{ + Key: accessMaxSize, + Type: "string", + Description: `cap total bytes access tiering keeps on the fastest pool, e.g. "2TiB", 0 for unlimited` + defaultHelpPostfix(accessMaxSize), + Optional: true, + }, + config.HelpKV{ + Key: accessPromoteWatermark, + Type: "number", + Description: `stop promoting once the fastest pool is this percent full` + defaultHelpPostfix(accessPromoteWatermark), + Optional: true, + }, + config.HelpKV{ + Key: accessBinWidth, + Type: "duration", + Description: `resolution of the access counter` + defaultHelpPostfix(accessBinWidth), + Optional: true, + }, + config.HelpKV{ + Key: accessBins, + Type: "number", + Description: `number of access counter bins; bins times bin width caps the rule Window` + defaultHelpPostfix(accessBins), + Optional: true, + }, + config.HelpKV{ + Key: accessFlush, + Type: "duration", + Description: `how often access counters are published and a promotion sweep runs` + defaultHelpPostfix(accessFlush), + Optional: true, + }, + config.HelpKV{ + Key: accessMinResidency, + Type: "duration", + Description: `minimum time an object stays on a pool after access tiering moved it` + defaultHelpPostfix(accessMinResidency), + Optional: true, + }, + config.HelpKV{ + Key: accessWorkers, + Type: "number", + Description: `set the number of access tiering workers` + defaultHelpPostfix(accessWorkers), + Optional: true, + }, + config.HelpKV{ + Key: accessMaxTracked, + Type: "number", + Description: `maximum number of objects each node keeps access counters for` + defaultHelpPostfix(accessMaxTracked), + Optional: true, + }, } ) diff --git a/internal/config/ilm/ilm.go b/internal/config/ilm/ilm.go index 3ecf68fae63e4..cb88e69954ba1 100644 --- a/internal/config/ilm/ilm.go +++ b/internal/config/ilm/ilm.go @@ -18,12 +18,29 @@ package ilm import ( + "errors" "strconv" + "strings" + "time" + "github.com/dustin/go-humanize" "github.com/minio/minio/internal/config" "github.com/minio/pkg/v3/env" ) +// Errors returned when the access tiering configuration is unusable. +var ( + ErrAccessPoolsInvalid = errors.New("ilm access_pools must be a comma separated list of distinct pool indices, hottest first, e.g. \"0,1\"") + ErrAccessPoolsTooFew = errors.New("ilm access_pools needs at least two pools to move objects between") + ErrAccessWatermarkInvalid = errors.New("ilm access_promote_watermark must be between 1 and 100") + ErrAccessBinsInvalid = errors.New("ilm access_bins must be between 2 and 64") + ErrAccessBinWidthInvalid = errors.New("ilm access_bin_width must be at least 1s") + ErrAccessFlushInvalid = errors.New("ilm access_flush must be at least 1s") + ErrAccessResidencyInvalid = errors.New("ilm access_min_residency cannot be negative") + ErrAccessWorkersInvalid = errors.New("ilm access_workers must be a positive integer") + ErrAccessTrackedInvalid = errors.New("ilm access_max_tracked must be a positive integer") +) + // DefaultKVS default configuration values for ILM subsystem var DefaultKVS = config.KVS{ config.KV{ @@ -34,12 +51,124 @@ var DefaultKVS = config.KVS{ Key: expirationWorkers, Value: "100", }, + config.KV{ + Key: accessTiering, + Value: config.EnableOff, + }, + config.KV{ + Key: accessPools, + Value: "", + }, + config.KV{ + Key: accessMaxSize, + Value: "0", + }, + config.KV{ + Key: accessPromoteWatermark, + Value: "85", + }, + config.KV{ + Key: accessBinWidth, + Value: "1m", + }, + config.KV{ + Key: accessBins, + Value: "12", + }, + config.KV{ + Key: accessFlush, + Value: "1m", + }, + config.KV{ + Key: accessMinResidency, + Value: "24h", + }, + config.KV{ + Key: accessWorkers, + Value: "10", + }, + config.KV{ + Key: accessMaxTracked, + Value: "1000000", + }, } // Config represents the different configuration values for ILM subsystem type Config struct { TransitionWorkers int ExpirationWorkers int + + // AccessTiering enables access-frequency driven relocation of objects + // between server pools. Off by default: it moves data. + AccessTiering bool + // AccessPools lists pool indices hottest first, e.g. []int{0, 1}. + AccessPools []int + // AccessMaxSize caps total bytes held on the hottest pool, 0 == unlimited. + AccessMaxSize uint64 + // AccessPromoteWatermark stops promotion once the hottest pool is this + // percentage full. + AccessPromoteWatermark int + // AccessBinWidth and AccessBins size the rolling hit counter. Their + // product is the longest rule Window that can be evaluated. + AccessBinWidth time.Duration + AccessBins int + // AccessFlush is how often each node publishes its counters and the + // leader merges them and runs a promotion sweep. + AccessFlush time.Duration + // AccessMinResidency is the minimum time an object stays put after a + // move, regardless of what the counters say. + AccessMinResidency time.Duration + AccessWorkers int + // AccessMaxTracked caps how many objects each node keeps counters for. + AccessMaxTracked int +} + +// HotPool returns the index of the hottest configured pool and whether access +// tiering is usable at all. +func (c Config) HotPool() (int, bool) { + if !c.AccessTiering || len(c.AccessPools) < 2 { + return -1, false + } + return c.AccessPools[0], true +} + +// ColdPool returns the index of the coldest configured pool, i.e. where +// demoted objects go. +func (c Config) ColdPool() (int, bool) { + if !c.AccessTiering || len(c.AccessPools) < 2 { + return -1, false + } + return c.AccessPools[len(c.AccessPools)-1], true +} + +// HistoryWindow is the longest window the rolling counter can answer for. +func (c Config) HistoryWindow() time.Duration { + return time.Duration(c.AccessBins) * c.AccessBinWidth +} + +// parseAccessPools parses "0,1" into []int{0, 1}, rejecting duplicates and +// negative indices. An empty string yields no pools, which disables the +// feature rather than erroring - operators enable the switch before they +// configure the topology. +func parseAccessPools(s string) ([]int, error) { + s = strings.TrimSpace(s) + if s == "" { + return nil, nil + } + var pools []int + seen := make(map[int]struct{}) + for _, f := range strings.Split(s, ",") { + idx, err := strconv.Atoi(strings.TrimSpace(f)) + if err != nil || idx < 0 { + return nil, ErrAccessPoolsInvalid + } + if _, dup := seen[idx]; dup { + return nil, ErrAccessPoolsInvalid + } + seen[idx] = struct{}{} + pools = append(pools, idx) + } + return pools, nil } // LookupConfig - lookup ilm config and override with valid environment settings if any. @@ -65,5 +194,79 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) { cfg.TransitionWorkers = tw cfg.ExpirationWorkers = ew + + if err := cfg.lookupAccess(kvs); err != nil { + return cfg, err + } return cfg, nil } + +func (c *Config) lookupAccess(kvs config.KVS) (err error) { + c.AccessTiering, err = config.ParseBool(env.Get(EnvILMAccessTiering, kvs.GetWithDefault(accessTiering, DefaultKVS))) + if err != nil { + return err + } + + if c.AccessPools, err = parseAccessPools(env.Get(EnvILMAccessPools, kvs.GetWithDefault(accessPools, DefaultKVS))); err != nil { + return err + } + // Enabling the feature without a usable topology is a configuration + // error worth surfacing at set time rather than silently doing nothing. + if c.AccessTiering && len(c.AccessPools) < 2 { + return ErrAccessPoolsTooFew + } + + maxSize := env.Get(EnvILMAccessMaxSize, kvs.GetWithDefault(accessMaxSize, DefaultKVS)) + if maxSize == "" || maxSize == "0" { + c.AccessMaxSize = 0 + } else if c.AccessMaxSize, err = humanize.ParseBytes(maxSize); err != nil { + return err + } + + if c.AccessPromoteWatermark, err = strconv.Atoi(env.Get(EnvILMAccessPromoteWatermark, kvs.GetWithDefault(accessPromoteWatermark, DefaultKVS))); err != nil { + return err + } + if c.AccessPromoteWatermark < 1 || c.AccessPromoteWatermark > 100 { + return ErrAccessWatermarkInvalid + } + + if c.AccessBinWidth, err = time.ParseDuration(env.Get(EnvILMAccessBinWidth, kvs.GetWithDefault(accessBinWidth, DefaultKVS))); err != nil { + return err + } + if c.AccessBinWidth < time.Second { + return ErrAccessBinWidthInvalid + } + if c.AccessBins, err = strconv.Atoi(env.Get(EnvILMAccessBins, kvs.GetWithDefault(accessBins, DefaultKVS))); err != nil { + return err + } + if c.AccessBins < 2 || c.AccessBins > 64 { + return ErrAccessBinsInvalid + } + + if c.AccessFlush, err = time.ParseDuration(env.Get(EnvILMAccessFlush, kvs.GetWithDefault(accessFlush, DefaultKVS))); err != nil { + return err + } + if c.AccessFlush < time.Second { + return ErrAccessFlushInvalid + } + if c.AccessMinResidency, err = time.ParseDuration(env.Get(EnvILMAccessMinResidency, kvs.GetWithDefault(accessMinResidency, DefaultKVS))); err != nil { + return err + } + if c.AccessMinResidency < 0 { + return ErrAccessResidencyInvalid + } + + if c.AccessWorkers, err = strconv.Atoi(env.Get(EnvILMAccessWorkers, kvs.GetWithDefault(accessWorkers, DefaultKVS))); err != nil { + return err + } + if c.AccessWorkers < 1 { + return ErrAccessWorkersInvalid + } + if c.AccessMaxTracked, err = strconv.Atoi(env.Get(EnvILMAccessMaxTracked, kvs.GetWithDefault(accessMaxTracked, DefaultKVS))); err != nil { + return err + } + if c.AccessMaxTracked < 1 { + return ErrAccessTrackedInvalid + } + return nil +} diff --git a/internal/config/ilm/ilm_test.go b/internal/config/ilm/ilm_test.go new file mode 100644 index 0000000000000..35e8689ff1b78 --- /dev/null +++ b/internal/config/ilm/ilm_test.go @@ -0,0 +1,109 @@ +// Copyright (c) 2015-2026 MinIO, Inc. + +package ilm + +import ( + "errors" + "reflect" + "testing" + "time" + + "github.com/minio/minio/internal/config" +) + +func clearAccessEnv(t *testing.T) { + t.Helper() + for _, name := range []string{ + EnvILMAccessTiering, EnvILMAccessPools, EnvILMAccessMaxSize, + EnvILMAccessPromoteWatermark, EnvILMAccessBinWidth, EnvILMAccessBins, + EnvILMAccessFlush, EnvILMAccessMinResidency, EnvILMAccessWorkers, + EnvILMAccessMaxTracked, + } { + // The env helper treats an empty value as unset. t.Setenv restores the + // caller's exact value automatically when the test finishes. + t.Setenv(name, "") + } +} + +func TestLookupAccessDefaults(t *testing.T) { + clearAccessEnv(t) + cfg, err := LookupConfig(DefaultKVS.Clone()) + if err != nil { + t.Fatal(err) + } + if cfg.AccessTiering { + t.Fatal("access tiering must default off") + } + if cfg.AccessBinWidth != time.Minute || cfg.AccessBins != 12 || cfg.AccessFlush != time.Minute { + t.Fatalf("unexpected counter defaults: width=%s bins=%d flush=%s", cfg.AccessBinWidth, cfg.AccessBins, cfg.AccessFlush) + } + if cfg.AccessWorkers != 10 || cfg.AccessMaxTracked != 1000000 { + t.Fatalf("unexpected worker/map defaults: %d/%d", cfg.AccessWorkers, cfg.AccessMaxTracked) + } +} + +func TestLookupAccessEnabled(t *testing.T) { + clearAccessEnv(t) + kvs := DefaultKVS.Clone() + kvs.Set(accessTiering, config.EnableOn) + kvs.Set(accessPools, "2, 0, 1") + kvs.Set(accessMaxSize, "2GiB") + kvs.Set(accessPromoteWatermark, "90") + kvs.Set(accessBinWidth, "30s") + kvs.Set(accessBins, "20") + kvs.Set(accessFlush, "15s") + kvs.Set(accessMinResidency, "2h") + kvs.Set(accessWorkers, "7") + kvs.Set(accessMaxTracked, "1234") + + cfg, err := LookupConfig(kvs) + if err != nil { + t.Fatal(err) + } + if !cfg.AccessTiering || !reflect.DeepEqual(cfg.AccessPools, []int{2, 0, 1}) { + t.Fatalf("unexpected topology: enabled=%v pools=%v", cfg.AccessTiering, cfg.AccessPools) + } + if cfg.AccessMaxSize != 2<<30 || cfg.HistoryWindow() != 10*time.Minute { + t.Fatalf("unexpected size/history: %d/%s", cfg.AccessMaxSize, cfg.HistoryWindow()) + } + if hot, ok := cfg.HotPool(); !ok || hot != 2 { + t.Fatalf("HotPool = %d/%v", hot, ok) + } + if cold, ok := cfg.ColdPool(); !ok || cold != 1 { + t.Fatalf("ColdPool = %d/%v", cold, ok) + } +} + +func TestLookupAccessRejectsInvalidValues(t *testing.T) { + clearAccessEnv(t) + tests := []struct { + key, value string + want error + }{ + {accessPools, "0,0", ErrAccessPoolsInvalid}, + {accessPromoteWatermark, "0", ErrAccessWatermarkInvalid}, + {accessBinWidth, "500ms", ErrAccessBinWidthInvalid}, + {accessBins, "1", ErrAccessBinsInvalid}, + {accessFlush, "0s", ErrAccessFlushInvalid}, + {accessMinResidency, "-1s", ErrAccessResidencyInvalid}, + {accessWorkers, "0", ErrAccessWorkersInvalid}, + {accessMaxTracked, "0", ErrAccessTrackedInvalid}, + } + for _, tc := range tests { + t.Run(tc.key, func(t *testing.T) { + kvs := DefaultKVS.Clone() + kvs.Set(tc.key, tc.value) + _, err := LookupConfig(kvs) + if !errors.Is(err, tc.want) { + t.Fatalf("error = %v, want %v", err, tc.want) + } + }) + } + + kvs := DefaultKVS.Clone() + kvs.Set(accessTiering, config.EnableOn) + kvs.Set(accessPools, "0") + if _, err := LookupConfig(kvs); !errors.Is(err, ErrAccessPoolsTooFew) { + t.Fatalf("error = %v, want %v", err, ErrAccessPoolsTooFew) + } +}