Skip to content

Commit 787ca8e

Browse files
carderneTrigger.dev RepoOps
authored andcommitted
perf(clickhouse): drop the task event attributes JSON column
Removes the native JSON attributes column from the task events table; attributes are stored as serialized text. Mono-RevId: e4c1daf4eb684f03e7edb16318b41c7ab1e3f0c9
1 parent c9a1b99 commit 787ca8e

3 files changed

Lines changed: 25 additions & 21 deletions

File tree

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
-- +goose Up
2+
3+
-- Writers stopped populating attributes several releases ago. Apply once the
4+
-- table's TTL has aged out the stored values, so nothing readable is lost.
5+
-- Metadata-only. The parts are rewritten by normal merges.
6+
ALTER TABLE trigger_dev.task_events_v2
7+
DROP COLUMN IF EXISTS attributes;
8+
9+
-- +goose Down
10+
11+
ALTER TABLE trigger_dev.task_events_v2
12+
ADD COLUMN IF NOT EXISTS attributes JSON CODEC(ZSTD(1))
13+
AFTER status;

‎internal-packages/clickhouse/src/taskEvents.test.ts‎

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -39,17 +39,12 @@ function readAttributesText(ch: ClickHouse) {
3939
return ch.reader.query({
4040
name: "read-task-event-attributes",
4141
query: `SELECT attributes_text,
42-
toJSONString(attributes) AS attributes_json,
4342
toUInt8(inserted_at > toDateTime64('2020-01-01 00:00:00', 3)) AS has_inserted_at
4443
FROM trigger_dev.task_events_v2
4544
WHERE environment_id = {environmentId: String}
4645
AND span_id = {spanId: String}`,
4746
params: z.object({ environmentId: z.string(), spanId: z.string() }),
48-
schema: z.object({
49-
attributes_text: z.string(),
50-
attributes_json: z.string(),
51-
has_inserted_at: z.number(),
52-
}),
47+
schema: z.object({ attributes_text: z.string(), has_inserted_at: z.number() }),
5348
});
5449
}
5550

@@ -93,24 +88,24 @@ describe("task events v2", () => {
9388
expect(rows).toEqual([
9489
{
9590
attributes_text: '{"z":1,"a":"hello","nested":{"enabled":true}}',
96-
// The JSON column is no longer written; only the text is stored.
97-
attributes_json: "{}",
9891
has_inserted_at: 1,
9992
},
10093
]);
10194
}
10295
);
10396

104-
clickhouseTest("attributes_text is a plain stored column", async ({ clickhouseContainer }) => {
105-
const ch = new ClickHouse({ url: clickhouseContainer.getConnectionUrl(), name: "test" });
97+
clickhouseTest(
98+
"attributes_text is the only attributes column",
99+
async ({ clickhouseContainer }) => {
100+
const ch = new ClickHouse({ url: clickhouseContainer.getConnectionUrl(), name: "test" });
106101

107-
const [columnError, columns] = await readColumnKinds(ch)({});
108-
expect(columnError).toBeNull();
109-
expect(columns).toEqual([
110-
{ name: "attributes", default_kind: "", default_expression: "" },
111-
{ name: "attributes_text", default_kind: "", default_expression: "" },
112-
]);
113-
});
102+
const [columnError, columns] = await readColumnKinds(ch)({});
103+
expect(columnError).toBeNull();
104+
expect(columns).toEqual([
105+
{ name: "attributes_text", default_kind: "", default_expression: "" },
106+
]);
107+
}
108+
);
114109

115110
clickhouseTest("has no attributes text indexes", async ({ clickhouseContainer }) => {
116111
const ch = new ClickHouse({ url: clickhouseContainer.getConnectionUrl(), name: "test" });

‎internal-packages/clickhouse/src/taskEvents.ts‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -258,11 +258,7 @@ export function insertTaskEventsV2(
258258
table: "trigger_dev.task_events_v2",
259259
columns: TASK_EVENT_V2_INSERT_COLUMNS,
260260
settings: {
261-
enable_json_type: 1,
262-
type_json_skip_duplicated_paths: 1,
263-
input_format_json_infer_array_of_dynamic_from_array_of_different_types: 1,
264261
input_format_json_throw_on_bad_escape_sequence: 0,
265-
input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects: 1,
266262
...settings,
267263
},
268264
});

0 commit comments

Comments
 (0)