diff --git a/README.md b/README.md index e6e47a0..01505e1 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,9 @@ panel issue is the source of truth for the wire field set. "version": "2.0.0", "git": "abc1234", "dev": false, - "theme": "default" + "theme": "default", + "region": "eu", + "exported_30d": true }, "env": { "php": "8.2", @@ -69,7 +71,25 @@ panel issue is the source of truth for the wire field set. "comms_active": 412, "comms_total": 5108, "submissions_30d": 23, - "protests_30d": 0 + "protests_30d": 0, + "bans_30d": 311, + "admins_active_30d": 5, + "ban_history_months": 74, + "web_bans_majority_30d": false, + "games": { + "tf": true, + "cstrike": false, + "csgo": true, + "garrysmod": false, + "left4dead2": false, + "other": true + }, + "perms": { + "web_groups": true, + "server_groups": true, + "overrides": false, + "immunity": true + } }, "features": { "submit": true, @@ -121,6 +141,11 @@ the load-bearing trust contract from `request.cf.colo` (the edge node id) is allowed — it identifies our edge, not the client. +`panel.region` is not a geolocation of the request. The panel derives it from +its own configured `config.timezone` and sends only the continent code (`af`, +`as`, `eu`, `na`, `oc`, `sa`) or omits it; the Worker never reads +`request.cf.timezone` or any other `request.cf` location field to fill it. + The Worker is structured so the AE data point is built _only_ from the validated body. Headers and `request.cf` are never read on the ingest path. That makes "no IP data path" the default, not an opt-in. The IP-stripping @@ -168,7 +193,8 @@ edit policy): `lock.blobs` / `lock.doubles` at the next free position. Until promotion, queries reach it via `json_extract(blob, '$.new_field')`. 2. **Slot exhaustion.** Schema-1 reserves 20 blob slots and 20 double slots. - Currently 11/20 blobs and 13/20 doubles are committed. Once an addition + Currently 12/20 blobs and 16/20 doubles are committed, and 39/53 bits of + `panel_features_bits`. Once an addition would push past the cap, the field lives permanently in `extras` until a schema bump. **We never reshuffle existing slot positions** — AE indexes are positional, and historical rows already use the current layout. @@ -201,7 +227,8 @@ deep-equality both directions. "env.web_server", "env.db_engine", "env.db_version", - "extras" + "extras", + "panel.region" ], "doubles": [ "schema", @@ -216,7 +243,10 @@ deep-equality both directions. "scale.comms_active", "scale.comms_total", "scale.submissions_30d", - "scale.protests_30d" + "scale.protests_30d", + "scale.bans_30d", + "scale.admins_active_30d", + "scale.ban_history_months" ], "bits": [ "panel.dev", @@ -245,7 +275,19 @@ deep-equality both directions. "env.host_panel_plesk", "env.host_panel_directadmin", "env.docroot_user_home", - "env.sapi_per_user" + "env.sapi_per_user", + "scale.games.tf", + "scale.games.cstrike", + "scale.games.csgo", + "scale.games.garrysmod", + "scale.games.left4dead2", + "scale.games.other", + "scale.perms.web_groups", + "scale.perms.server_groups", + "scale.perms.overrides", + "scale.perms.immunity", + "scale.web_bans_majority_30d", + "panel.exported_30d" ] } ``` @@ -261,16 +303,20 @@ deep-equality both directions. `panel.version`, which gives bounded cardinality and is the field most queries filter on. - `bits[i]` is bit `i` (LSB = 0) of the `panel_features_bits` double. - 27 booleans pack into one double (well under the 53-bit safe-integer - ceiling), leaving 7/20 doubles free for future scale dimensions. + 39 booleans pack into one double (under the 53-bit safe-integer + ceiling), leaving 4/20 doubles free for future scale dimensions. - Missing typed strings → `null` in the corresponding blob. Missing scale numbers → `null` in the corresponding double (so analysts can distinguish "not sent" from "zero"). Missing booleans → `0` bits in `panel_features_bits`. The `panel_features_bits` double is always present. -- `extras` (last blob) is `null` when there are no unknown top-level keys, +- `extras` (`blob11`) is `null` when there are no unknown top-level keys, otherwise a JSON-stringified object of every unknown top-level key. AE stores nothing rather than `{}` so analysts don't have to coalesce empty - objects out. + objects out. It is no longer the last blob: slots promoted after it (e.g. + `panel.region`) append behind it, because positions never move. +- Nested objects flatten to dotted names: `scale.games.tf` reads + `payload.scale.games.tf`. Rows written before a slot existed read as + `null` (doubles / blobs) or `0` (bits), the same as "not sent". ## `featureFlag(name)` SQL macro for AE @@ -447,9 +493,9 @@ curl -i http://127.0.0.1:8787/v1/ping \ -d '{ "schema": 1, "instance_id": "test-instance-0000000000000000000000000000000000000000000000000000", - "panel": {"version":"2.0.0","git":"abc1234","dev":false,"theme":"default"}, + "panel": {"version":"2.0.0","git":"abc1234","dev":false,"theme":"default","region":"eu","exported_30d":false}, "env": {"php":"8.2","sapi":"fpm-fcgi","db_engine":"mariadb","db_version":"10.11","web_server":"litespeed","os_family":"linux","memory_limit_mb":256,"max_execution_time":30,"disable_functions_count":7,"zts":false,"php_64bit":true,"open_basedir_set":true,"allow_url_fopen":true,"opcache_loaded":true,"suhosin_loaded":false,"posix_available":true,"host_panel_cpanel":true,"host_panel_plesk":false,"host_panel_directadmin":false,"docroot_user_home":true,"sapi_per_user":true}, - "scale": {"admins":1,"servers_enabled":1,"bans_active":0,"bans_total":0,"comms_active":0,"comms_total":0,"submissions_30d":0,"protests_30d":0}, + "scale": {"admins":1,"servers_enabled":1,"bans_active":0,"bans_total":0,"comms_active":0,"comms_total":0,"submissions_30d":0,"protests_30d":0,"bans_30d":0,"admins_active_30d":1,"ban_history_months":0,"web_bans_majority_30d":false,"games":{"tf":true,"cstrike":false,"csgo":false,"garrysmod":false,"left4dead2":false,"other":false},"perms":{"web_groups":false,"server_groups":false,"overrides":false,"immunity":false}}, "features": {"submit":true,"protest":false,"comms":false,"kickit":false,"exportpublic":false,"publiccomments":false,"steamlogin":true,"normallogin":true,"groupbanning":false,"friendsbanning":false,"adminrehashing":false,"smtp_configured":false,"steam_api_key_set":false,"geoip_present":false} }' ``` diff --git a/schema/1.lock.json b/schema/1.lock.json index 84cc151..c3bb4a4 100644 --- a/schema/1.lock.json +++ b/schema/1.lock.json @@ -10,7 +10,8 @@ "env.web_server", "env.db_engine", "env.db_version", - "extras" + "extras", + "panel.region" ], "doubles": [ "schema", @@ -25,7 +26,10 @@ "scale.comms_active", "scale.comms_total", "scale.submissions_30d", - "scale.protests_30d" + "scale.protests_30d", + "scale.bans_30d", + "scale.admins_active_30d", + "scale.ban_history_months" ], "bits": [ "panel.dev", @@ -54,6 +58,18 @@ "env.host_panel_plesk", "env.host_panel_directadmin", "env.docroot_user_home", - "env.sapi_per_user" + "env.sapi_per_user", + "scale.games.tf", + "scale.games.cstrike", + "scale.games.csgo", + "scale.games.garrysmod", + "scale.games.left4dead2", + "scale.games.other", + "scale.perms.web_groups", + "scale.perms.server_groups", + "scale.perms.overrides", + "scale.perms.immunity", + "scale.web_bans_majority_30d", + "panel.exported_30d" ] } diff --git a/src/schema.ts b/src/schema.ts index a7fc53d..d974a76 100644 --- a/src/schema.ts +++ b/src/schema.ts @@ -19,6 +19,8 @@ const schema1Panel = z git: z.string().optional(), dev: z.boolean().optional(), theme: z.string().optional(), + region: z.string().optional(), + exported_30d: z.boolean().optional(), }) .passthrough(); @@ -48,6 +50,26 @@ const schema1Env = z }) .passthrough(); +const schema1ScaleGames = z + .object({ + tf: z.boolean().optional(), + cstrike: z.boolean().optional(), + csgo: z.boolean().optional(), + garrysmod: z.boolean().optional(), + left4dead2: z.boolean().optional(), + other: z.boolean().optional(), + }) + .passthrough(); + +const schema1ScalePerms = z + .object({ + web_groups: z.boolean().optional(), + server_groups: z.boolean().optional(), + overrides: z.boolean().optional(), + immunity: z.boolean().optional(), + }) + .passthrough(); + const schema1Scale = z .object({ admins: z.number().optional(), @@ -58,6 +80,12 @@ const schema1Scale = z comms_total: z.number().optional(), submissions_30d: z.number().optional(), protests_30d: z.number().optional(), + bans_30d: z.number().optional(), + admins_active_30d: z.number().optional(), + ban_history_months: z.number().optional(), + web_bans_majority_30d: z.boolean().optional(), + games: schema1ScaleGames.optional(), + perms: schema1ScalePerms.optional(), }) .passthrough(); diff --git a/test/__snapshots__/ae.test.ts.snap b/test/__snapshots__/ae.test.ts.snap index 637453f..2c1a991 100644 --- a/test/__snapshots__/ae.test.ts.snap +++ b/test/__snapshots__/ae.test.ts.snap @@ -14,10 +14,11 @@ exports[`mapDataPoint > maps the canonical payload to the snapshot data point 1` "mariadb", "10.11", null, + "eu", ], "doubles": [ 1, - 107968910, + 374441212302, 256, 30, 7, @@ -29,6 +30,9 @@ exports[`mapDataPoint > maps the canonical payload to the snapshot data point 1` 5108, 23, 0, + 311, + 5, + 74, ], "indexes": [ "2.0.0", diff --git a/test/ae.test.ts b/test/ae.test.ts index bea3f47..f5d66ab 100644 --- a/test/ae.test.ts +++ b/test/ae.test.ts @@ -186,3 +186,117 @@ describe("env.* shared-host fingerprint round-trip (issue #3)", () => { expect(bits).toBe(Math.pow(2, lock.bits.indexOf("env.sapi_per_user"))); }); }); + +describe("scale activity / game mix / permission shape / region / export round-trip (issue #6)", () => { + // Same contract as the env.* block above: every new slot reads through the + // existing readPath / bit-pack path, including the two-level nesting under + // scale.games / scale.perms. These tests pin each slot to its lock-derived + // position and pin the sparse-payload defaults. + + const NEW_DOUBLES = ["scale.bans_30d", "scale.admins_active_30d", "scale.ban_history_months"]; + + const NEW_BITS = [ + "scale.games.tf", + "scale.games.cstrike", + "scale.games.csgo", + "scale.games.garrysmod", + "scale.games.left4dead2", + "scale.games.other", + "scale.perms.web_groups", + "scale.perms.server_groups", + "scale.perms.overrides", + "scale.perms.immunity", + "scale.web_bans_majority_30d", + "panel.exported_30d", + ]; + + const bitSet = (bits: number, name: string): boolean => + (Math.floor(bits / Math.pow(2, lock.bits.indexOf(name))) & 1) === 1; + + it("appends the new slots after every pre-existing slot", () => { + // Append-only: the new names occupy the tail of each lock array, in the + // order the issue lists them. + expect(lock.blobs.slice(-1)).toEqual(["panel.region"]); + expect(lock.doubles.slice(-NEW_DOUBLES.length)).toEqual(NEW_DOUBLES); + expect(lock.bits.slice(-NEW_BITS.length)).toEqual(NEW_BITS); + expect(lock.blobs.indexOf("extras")).toBe(lock.blobs.length - 2); + }); + + it("round-trips every new signal to its lock-derived position", () => { + const dp = mapDataPoint(canonicalPayload); + + expect(dp.blobs[lock.blobs.indexOf("panel.region")]).toBe("eu"); + + expect(dp.doubles[lock.doubles.indexOf("scale.bans_30d")]).toBe(311); + expect(dp.doubles[lock.doubles.indexOf("scale.admins_active_30d")]).toBe(5); + expect(dp.doubles[lock.doubles.indexOf("scale.ban_history_months")]).toBe(74); + + const bits = dp.doubles[lock.doubles.indexOf("panel_features_bits")] ?? 0; + expect(bitSet(bits, "scale.games.tf")).toBe(true); + expect(bitSet(bits, "scale.games.cstrike")).toBe(false); + expect(bitSet(bits, "scale.games.csgo")).toBe(true); + expect(bitSet(bits, "scale.games.garrysmod")).toBe(false); + expect(bitSet(bits, "scale.games.left4dead2")).toBe(false); + expect(bitSet(bits, "scale.games.other")).toBe(true); + expect(bitSet(bits, "scale.perms.web_groups")).toBe(true); + expect(bitSet(bits, "scale.perms.server_groups")).toBe(true); + expect(bitSet(bits, "scale.perms.overrides")).toBe(false); + expect(bitSet(bits, "scale.perms.immunity")).toBe(true); + expect(bitSet(bits, "scale.web_bans_majority_30d")).toBe(false); + expect(bitSet(bits, "panel.exported_30d")).toBe(true); + + // The nested games / perms objects are typed, not unknown top-level keys. + expect(dp.blobs[lock.blobs.indexOf("extras")]).toBeNull(); + }); + + it.each(NEW_BITS)("places %s alone at its lock-derived bit", (name) => { + const payload: Record = { schema: 1, instance_id: "abc" }; + // Build the nested object for `name` (e.g. scale.games.tf -> { scale: { games: { tf: true } } }). + const segments = name.split("."); + let cursor = payload; + for (const seg of segments.slice(0, -1)) { + const next: Record = {}; + cursor[seg] = next; + cursor = next; + } + cursor[segments[segments.length - 1]!] = true; + + const dp = mapDataPoint(payload); + const bits = dp.doubles[lock.doubles.indexOf("panel_features_bits")] ?? 0; + expect(bits).toBe(Math.pow(2, lock.bits.indexOf(name))); + }); + + it("emits null/0 for the new signals when the panel sends an older payload", () => { + const older = { + schema: 1 as const, + instance_id: canonicalPayload.instance_id, + panel: { version: "2.0.0", git: "abc1234", dev: false, theme: "default" }, + scale: { + admins: 12, + servers_enabled: 7, + bans_active: 2847, + bans_total: 18394, + comms_active: 412, + comms_total: 5108, + submissions_30d: 23, + protests_30d: 0, + }, + }; + const dp = mapDataPoint(older); + + expect(dp.blobs[lock.blobs.indexOf("panel.region")]).toBeNull(); + expect(dp.blobs[lock.blobs.indexOf("extras")]).toBeNull(); + + for (const name of NEW_DOUBLES) { + expect(dp.doubles[lock.doubles.indexOf(name)], `expected ${name} to be null`).toBeNull(); + } + + const bits = dp.doubles[lock.doubles.indexOf("panel_features_bits")] ?? 0; + for (const name of NEW_BITS) { + expect(bitSet(bits, name), `expected bit ${name} to be 0 for older payload`).toBe(false); + } + + // Legacy scale doubles still serialise. + expect(dp.doubles[lock.doubles.indexOf("scale.bans_total")]).toBe(18394); + }); +}); diff --git a/test/fixtures.ts b/test/fixtures.ts index f2abd85..a8e7efe 100644 --- a/test/fixtures.ts +++ b/test/fixtures.ts @@ -15,6 +15,8 @@ export const canonicalPayload = { git: "abc1234", dev: false, theme: "default", + region: "eu", + exported_30d: true, }, env: { php: "8.2", @@ -48,6 +50,24 @@ export const canonicalPayload = { comms_total: 5108, submissions_30d: 23, protests_30d: 0, + bans_30d: 311, + admins_active_30d: 5, + ban_history_months: 74, + web_bans_majority_30d: false, + games: { + tf: true, + cstrike: false, + csgo: true, + garrysmod: false, + left4dead2: false, + other: true, + }, + perms: { + web_groups: true, + server_groups: true, + overrides: false, + immunity: true, + }, }, features: { submit: true, diff --git a/test/schema.test.ts b/test/schema.test.ts index 6ba4cf5..7a61d14 100644 --- a/test/schema.test.ts +++ b/test/schema.test.ts @@ -62,6 +62,22 @@ describe("validate (dispatch + zod boundary cases)", () => { expect(() => validate({ schema: 1, instance_id: "" })).toThrow(ZodError); }); + it("accepts the nested scale.games / scale.perms objects and rejects wrong-typed leaves", () => { + expect(() => + validate({ + schema: 1, + instance_id: "abc", + scale: { games: { tf: true }, perms: { immunity: false } }, + }), + ).not.toThrow(); + expect(() => + validate({ schema: 1, instance_id: "abc", scale: { games: { tf: "yes" } } }), + ).toThrow(ZodError); + expect(() => validate({ schema: 1, instance_id: "abc", scale: { bans_30d: "311" } })).toThrow( + ZodError, + ); + }); + it("rejects wrong-typed required scalar fields", () => { expect(() => validate({ schema: 1, instance_id: 42 })).toThrow(ZodError); });