diff --git a/packages/cli/src/commands/lineage/dependents.ts b/packages/cli/src/commands/lineage/dependents.ts new file mode 100644 index 0000000..bc42b8b --- /dev/null +++ b/packages/cli/src/commands/lineage/dependents.ts @@ -0,0 +1,107 @@ +import { ConfigError } from "@metabase/client/errors"; +import { + type EntityRef, + LineageType, + type LineageType as LineageTypeValue, + walkDependents, +} from "../../core/lineage"; +import { renderList } from "../../output/render"; +import { warn } from "../../output/notice"; +import { listEnvelopeSchema } from "../../output/types"; +import { dependentView, DependentCompact } from "../../output/views/dependent"; +import { windowList } from "../../output/window"; +import { connectionFlags, listFlags, outputFlags, profileFlag } from "../flags"; +import { defineMetabaseCommand } from "../runtime"; + +export const DependentsEnvelope = listEnvelopeSchema(DependentCompact); + +export default defineMetabaseCommand({ + meta: { name: "dependents", description: "List Metabase entities downstream of a table" }, + details: + "Finds a physical table by schema and name, then walks every downstream dependency. --types filters the returned entities without pruning intermediate paths.", + capabilities: { minVersion: 61, tokenFeature: "dependencies" }, + args: { + ...outputFlags, + ...listFlags, + ...profileFlag, + ...connectionFlags, + database: { type: "string", description: "Metabase database id or name (optional)" }, + schema: { type: "string", description: "Physical table schema", required: true }, + table: { type: "string", description: "Physical table name", required: true }, + types: { + type: "string", + description: "Returned dependency types, comma separated (default: all)", + }, + }, + outputSchema: DependentsEnvelope, + examples: [ + "mb lineage dependents --schema reporting --table orders", + "mb lineage dependents --database Analytics --schema reporting --table orders --types question,dashboard", + ], + async run({ args, ctx, getClient }) { + const requestedTypes = parseTypes(args.types); + const client = await getClient(); + const databases = (await client.database.list()).data.filter((database) => + matchesDatabase(database.id, database.name, args.database), + ); + if (databases.length === 0) { + throw new ConfigError(`no Metabase database matches --database "${args.database}"`); + } + + const candidates = ( + await Promise.all( + databases.map((database) => client.database.schemaTables(database.id, args.schema)), + ) + ).flatMap(({ data }) => data.filter((table) => equalName(table.name, args.table))); + if (candidates.length === 0) { + throw new ConfigError(`table not found: ${args.schema}.${args.table}`); + } + if (candidates.length > 1) { + const ids = candidates.map((table) => `${table.db_id}:${table.id}`).join(", "); + throw new ConfigError( + `multiple Metabase tables match ${args.schema}.${args.table} (${ids}); pass --database`, + ); + } + + const table = candidates[0]; + if (table === undefined) { + throw new ConfigError(`table not found: ${args.schema}.${args.table}`); + } + const root: EntityRef = { type: "table", id: table.id }; + const dependents = await walkDependents(client.dependency.dependents, root); + const filtered = dependents.filter((dependent) => requestedTypes.has(dependent.type)); + + const status = await client.dependency.backfillStatus(); + if (!status.complete) { + warn("Metabase is still indexing dependencies; dependents may be incomplete."); + } + renderList(windowList(filtered, ctx.range), dependentView, ctx); + }, +}); + +function parseTypes(raw: string | undefined): Set { + if (raw === undefined) { + return new Set(LineageType); + } + const values = raw.split(",").map((value) => value.trim()); + const invalid = values.filter((value) => !isLineageType(value)); + if (invalid.length > 0 || values.length === 0) { + throw new ConfigError(`invalid --types value: "${raw}" (expected: ${LineageType.join(", ")})`); + } + return new Set(values.filter(isLineageType)); +} + +function isLineageType(value: string): value is LineageTypeValue { + return LineageType.some((candidate) => candidate === value); +} + +function equalName(left: string, right: string): boolean { + return left.localeCompare(right, undefined, { sensitivity: "accent" }) === 0; +} + +function matchesDatabase(id: number, name: string, filter: string | undefined): boolean { + if (filter === undefined) { + return true; + } + return String(id) === filter || equalName(name, filter); +} diff --git a/packages/cli/src/commands/lineage/index.ts b/packages/cli/src/commands/lineage/index.ts new file mode 100644 index 0000000..9bf9ad1 --- /dev/null +++ b/packages/cli/src/commands/lineage/index.ts @@ -0,0 +1,9 @@ +import { defineCommandGroup } from "../group"; + +export default defineCommandGroup({ + name: "lineage", + description: "Inspect dependencies between Metabase and warehouse entities", + subCommands: { + dependents: () => import("./dependents").then((mod) => mod.default), + }, +}); diff --git a/packages/cli/src/core/lineage.test.ts b/packages/cli/src/core/lineage.test.ts new file mode 100644 index 0000000..abcaad0 --- /dev/null +++ b/packages/cli/src/core/lineage.test.ts @@ -0,0 +1,45 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { DependencyNode } from "@metabase/client/domain/dependency"; + +import { lineageType, walkDependents } from "./lineage"; + +function node(id: number, type: DependencyNode["type"], data: DependencyNode["data"] = {}) { + return { id, type, data }; +} + +describe("lineage", () => { + it("uses formal card subtypes for questions, models, and metrics", () => { + expect(lineageType(node(1, "card", { type: "model" }))).toBe("model"); + expect(lineageType(node(2, "card", { type: "metric" }))).toBe("metric"); + expect(lineageType(node(3, "card", { type: "question" }))).toBe("question"); + }); + + it("walks breadth-first, keeps shortest paths, and breaks cycles", async () => { + const read = vi.fn(async (type: DependencyNode["type"], id: number) => { + const graph: Record = { + "table:1": [ + node(2, "card", { type: "model", name: "Orders model" }), + node(3, "card", { name: "Question" }), + ], + "card:2": [node(4, "dashboard", { name: "Operations" })], + "card:3": [node(4, "dashboard", { name: "Operations" })], + "dashboard:4": [node(2, "card", { type: "model", name: "Orders model" })], + }; + return graph[`${type}:${id}`] ?? []; + }); + + const result = await walkDependents(read, { type: "table", id: 1 }); + + expect(result.map(({ type, id, distance }) => ({ type, id, distance }))).toEqual([ + { type: "model", id: 2, distance: 1 }, + { type: "question", id: 3, distance: 1 }, + { type: "dashboard", id: 4, distance: 2 }, + ]); + expect(result[2]?.path).toEqual([ + { type: "table", id: 1 }, + { type: "card", id: 2 }, + { type: "dashboard", id: 4 }, + ]); + }); +}); diff --git a/packages/cli/src/core/lineage.ts b/packages/cli/src/core/lineage.ts new file mode 100644 index 0000000..ab4bcbe --- /dev/null +++ b/packages/cli/src/core/lineage.ts @@ -0,0 +1,91 @@ +import type { + DependencyCardType, + DependencyEntityType, + DependencyNode, +} from "@metabase/client/domain/dependency"; + +export const LineageType = [ + "question", + "model", + "metric", + "table", + "transform", + "snippet", + "dashboard", + "document", + "sandbox", + "segment", + "measure", +] as const; +export type LineageType = (typeof LineageType)[number]; + +export interface EntityRef { + id: number; + type: DependencyEntityType; +} + +export interface Dependent { + id: number; + type: LineageType; + name: string | null; + distance: number; + path: EntityRef[]; +} + +export interface DependencyReader { + (type: EntityRef["type"], id: number): Promise; +} + +export async function walkDependents(read: DependencyReader, root: EntityRef) { + const queue: Array<{ entity: EntityRef; path: EntityRef[] }> = [{ entity: root, path: [root] }]; + const seen = new Set([entityKey(root)]); + const dependents: Dependent[] = []; + + while (queue.length > 0) { + const current = queue.shift(); + if (current === undefined) { + break; + } + const direct = await read(current.entity.type, current.entity.id); + for (const node of direct) { + const entity: EntityRef = { type: node.type, id: node.id }; + const key = entityKey(entity); + if (seen.has(key)) { + continue; + } + seen.add(key); + const path = [...current.path, entity]; + dependents.push(buildDependent(node, path)); + queue.push({ entity, path }); + } + } + return dependents; +} + +export function entityKey(entity: EntityRef): string { + return `${entity.type}:${entity.id}`; +} + +export function lineageType(node: DependencyNode): LineageType { + if (node.type !== "card") { + return node.type; + } + const cardType = node.data["type"]; + return isCardType(cardType) ? cardType : "question"; +} + +export function buildDependent(node: DependencyNode, path: EntityRef[]): Dependent { + const rawName = + node.type === "table" ? (node.data["display_name"] ?? node.data["name"]) : node.data["name"]; + return { + id: node.id, + type: lineageType(node), + name: typeof rawName === "string" ? rawName : null, + distance: path.length - 1, + path, + }; +} + +function isCardType(value: unknown): value is DependencyCardType { + return value === "question" || value === "model" || value === "metric"; +} diff --git a/packages/cli/src/main.ts b/packages/cli/src/main.ts index 8765d50..b074fa4 100644 --- a/packages/cli/src/main.ts +++ b/packages/cli/src/main.ts @@ -15,6 +15,7 @@ const main: CommandDef = defineCommand({ auth: () => import("./commands/auth").then((mod) => mod.default), db: () => import("./commands/db").then((mod) => mod.default), table: () => import("./commands/table").then((mod) => mod.default), + lineage: () => import("./commands/lineage").then((mod) => mod.default), field: () => import("./commands/field").then((mod) => mod.default), upload: () => import("./commands/upload").then((mod) => mod.default), card: () => import("./commands/card").then((mod) => mod.default), diff --git a/packages/cli/src/output/views/dependent.ts b/packages/cli/src/output/views/dependent.ts new file mode 100644 index 0000000..bdb2c85 --- /dev/null +++ b/packages/cli/src/output/views/dependent.ts @@ -0,0 +1,46 @@ +import { z } from "zod"; +import { DependencyEntityType } from "@metabase/client/domain/dependency"; + +import type { Dependent } from "../../core/lineage"; +import { LineageType } from "../../core/lineage"; +import type { ResourceView } from "../view"; + +const EntityRef = z.object({ id: z.number().int().positive(), type: DependencyEntityType }); + +export const DependentCompact = z.object({ + id: z.number().int().positive(), + type: z.enum(LineageType), + name: z.string().nullable(), + distance: z.number().int().positive(), + path: z.array(EntityRef), +}); + +export const dependentView: ResourceView = { + compactPick: DependentCompact, + tableColumns: [ + { key: "type", label: "Type" }, + { key: "id", label: "ID" }, + { key: "name", label: "Name" }, + { key: "distance", label: "Distance" }, + { + key: "path", + label: "Path", + format: (value) => + Array.isArray(value) + ? value + .filter((item): item is { type: string; id: number } => + Boolean( + typeof item === "object" && + item !== null && + "type" in item && + "id" in item && + typeof item.type === "string" && + typeof item.id === "number", + ), + ) + .map((item) => `${item.type}:${item.id}`) + .join(" -> ") + : "", + }, + ], +}; diff --git a/packages/cli/src/runtime/command-help.test.ts b/packages/cli/src/runtime/command-help.test.ts index 4faecb4..4283959 100644 --- a/packages/cli/src/runtime/command-help.test.ts +++ b/packages/cli/src/runtime/command-help.test.ts @@ -316,6 +316,7 @@ const ALL_COMMANDS = [ "table get", "table fields", "table update", + "lineage dependents", "field get", "field values", "field summary", diff --git a/packages/client/src/client.ts b/packages/client/src/client.ts index 6301dac..bec8edd 100644 --- a/packages/client/src/client.ts +++ b/packages/client/src/client.ts @@ -4,6 +4,7 @@ import { collectionResource } from "./resources/collection"; import { dashboardResource } from "./resources/dashboard"; import { databaseResource } from "./resources/database"; import { datasetResource } from "./resources/dataset"; +import { dependencyResource } from "./resources/dependency"; import { documentResource } from "./resources/document"; import { eidTranslationResource } from "./resources/eid-translation"; import { fieldResource } from "./resources/field"; @@ -37,6 +38,7 @@ export function createClient(config: ClientCredentials, options: ClientOptions) dashboard: dashboardResource(transport), database: databaseResource(transport), dataset: datasetResource(transport), + dependency: dependencyResource(transport), document: documentResource(transport), eidTranslation: eidTranslationResource(transport), field: fieldResource(transport), diff --git a/packages/client/src/domain/dependency.ts b/packages/client/src/domain/dependency.ts new file mode 100644 index 0000000..91ba28c --- /dev/null +++ b/packages/client/src/domain/dependency.ts @@ -0,0 +1,28 @@ +import { z } from "zod"; + +export const DependencyEntityType = z.enum([ + "card", + "table", + "transform", + "snippet", + "dashboard", + "document", + "sandbox", + "segment", + "measure", +]); +export type DependencyEntityType = z.infer; + +export const DependencyCardType = z.enum(["question", "model", "metric"]); +export type DependencyCardType = z.infer; + +export const DependencyNode = z.object({ + id: z.number().int().positive(), + type: DependencyEntityType, + data: z.record(z.string(), z.unknown()), + dependents_count: z.record(z.string(), z.number().int().nonnegative()).nullable().optional(), +}); +export type DependencyNode = z.infer; + +export const DependencyBackfillStatus = z.object({ complete: z.boolean() }); +export type DependencyBackfillStatus = z.infer; diff --git a/packages/client/src/resources/dependency.test.ts b/packages/client/src/resources/dependency.test.ts new file mode 100644 index 0000000..af4507f --- /dev/null +++ b/packages/client/src/resources/dependency.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; + +import { createClient } from "../client"; +import type { ClientCredentials } from "../http/transport"; +import { captureFetch, jsonResponse, TEST_USER_AGENT } from "../testing/fetch-capture"; + +const CREDENTIALS: ClientCredentials = { + url: "https://mb.example.com", + credential: { kind: "apiKey", apiKey: "mb_test_key" }, +}; + +const HEADERS = { + accept: "application/json", + "user-agent": TEST_USER_AGENT, + "x-api-key": "mb_test_key", +}; + +function clientOver(body: unknown) { + const capture = captureFetch([jsonResponse(body)]); + const client = createClient(CREDENTIALS, { + userAgent: TEST_USER_AGENT, + fetchImpl: capture.fetch, + }); + return { client, capture }; +} + +describe("dependency resource wire requests", () => { + it("requests direct dependents for an entity", async () => { + const response = [{ id: 12, type: "card", data: { name: "Orders", type: "question" } }]; + const { client, capture } = clientOver(response); + + expect(await client.dependency.dependents("table", 7)).toEqual(response); + expect(capture.calls).toEqual([ + { + url: "https://mb.example.com/api/ee/dependencies/graph/dependents?type=table&id=7&include-personal-collections=true", + method: "GET", + headers: HEADERS, + body: null, + }, + ]); + }); + + it("requests the dependency backfill status", async () => { + const { client, capture } = clientOver({ complete: true }); + + expect(await client.dependency.backfillStatus()).toEqual({ complete: true }); + expect(capture.calls).toEqual([ + { + url: "https://mb.example.com/api/ee/dependencies/backfill-status", + method: "GET", + headers: HEADERS, + body: null, + }, + ]); + }); +}); diff --git a/packages/client/src/resources/dependency.ts b/packages/client/src/resources/dependency.ts new file mode 100644 index 0000000..7d1ee23 --- /dev/null +++ b/packages/client/src/resources/dependency.ts @@ -0,0 +1,35 @@ +import { z } from "zod"; + +import { + DependencyBackfillStatus, + DependencyNode, + type DependencyEntityType, +} from "../domain/dependency"; +import type { RequestOptions, Transport } from "../http/transport"; + +const DependencyNodeList = z.array(DependencyNode); + +export function dependencyResource(transport: Transport) { + /** List the direct dependents of an entity. */ + async function dependents( + type: DependencyEntityType, + id: number, + options: RequestOptions = {}, + ): Promise { + return transport.requestParsed(DependencyNodeList, "/api/ee/dependencies/graph/dependents", { + ...options, + query: { type, id, "include-personal-collections": true }, + }); + } + + /** Report whether Metabase has finished indexing dependencies. */ + async function backfillStatus(options: RequestOptions = {}): Promise { + return transport.requestParsed( + DependencyBackfillStatus, + "/api/ee/dependencies/backfill-status", + { ...options }, + ); + } + + return { dependents, backfillStatus }; +}