From 778641acc1e588188eeb2c39f96fe47cfde37ae8 Mon Sep 17 00:00:00 2001 From: Mohit Varikuti Date: Thu, 25 Jun 2026 04:10:51 -0700 Subject: [PATCH] feat(ai-twelvelabs): add TwelveLabs Pegasus video understanding adapter Adds @tanstack/ai-twelvelabs, an opt-in provider adapter that brings TwelveLabs Pegasus video understanding to the chat() activity. A video content part (URL, inline base64, or a pre-uploaded asset id) plus a text prompt is analyzed by Pegasus and streamed back as text. - Native streaming via TwelveLabs analyzeStream (NDJSON), mapped to the standard RUN_STARTED / TEXT_MESSAGE_* / RUN_FINISHED event lifecycle. - Structured output via the non-streaming analyze call with a json_schema response format. - Provider options for temperature, maxTokens, clip windowing (startTime/endTime), and assetId. - model-meta for pegasus1.5 / pegasus1.2 (and marengo3.0 for reference). - Deterministic no-network unit tests plus a TWELVELABS_API_KEY-gated live test. Non-breaking; no existing defaults change. --- .changeset/twelvelabs-adapter.md | 5 + README.md | 1 + docs/adapters/twelvelabs.md | 153 +++++++ docs/config.json | 5 + packages/ai-twelvelabs/LICENSE | 21 + packages/ai-twelvelabs/README.md | 120 ++++++ packages/ai-twelvelabs/package.json | 68 +++ packages/ai-twelvelabs/src/adapters/text.ts | 395 ++++++++++++++++++ packages/ai-twelvelabs/src/index.ts | 44 ++ packages/ai-twelvelabs/src/message-types.ts | 30 ++ packages/ai-twelvelabs/src/model-meta.ts | 67 +++ .../src/text/text-provider-options.ts | 42 ++ packages/ai-twelvelabs/src/utils/client.ts | 48 +++ packages/ai-twelvelabs/src/utils/index.ts | 6 + packages/ai-twelvelabs/tests/live.test.ts | 49 +++ .../ai-twelvelabs/tests/model-meta.test.ts | 16 + .../ai-twelvelabs/tests/text-adapter.test.ts | 186 +++++++++ packages/ai-twelvelabs/tsconfig.json | 8 + packages/ai-twelvelabs/vite.config.ts | 37 ++ pnpm-lock.yaml | 70 +++- 20 files changed, 1360 insertions(+), 11 deletions(-) create mode 100644 .changeset/twelvelabs-adapter.md create mode 100644 docs/adapters/twelvelabs.md create mode 100644 packages/ai-twelvelabs/LICENSE create mode 100644 packages/ai-twelvelabs/README.md create mode 100644 packages/ai-twelvelabs/package.json create mode 100644 packages/ai-twelvelabs/src/adapters/text.ts create mode 100644 packages/ai-twelvelabs/src/index.ts create mode 100644 packages/ai-twelvelabs/src/message-types.ts create mode 100644 packages/ai-twelvelabs/src/model-meta.ts create mode 100644 packages/ai-twelvelabs/src/text/text-provider-options.ts create mode 100644 packages/ai-twelvelabs/src/utils/client.ts create mode 100644 packages/ai-twelvelabs/src/utils/index.ts create mode 100644 packages/ai-twelvelabs/tests/live.test.ts create mode 100644 packages/ai-twelvelabs/tests/model-meta.test.ts create mode 100644 packages/ai-twelvelabs/tests/text-adapter.test.ts create mode 100644 packages/ai-twelvelabs/tsconfig.json create mode 100644 packages/ai-twelvelabs/vite.config.ts diff --git a/.changeset/twelvelabs-adapter.md b/.changeset/twelvelabs-adapter.md new file mode 100644 index 000000000..355ec8029 --- /dev/null +++ b/.changeset/twelvelabs-adapter.md @@ -0,0 +1,5 @@ +--- +'@tanstack/ai-twelvelabs': minor +--- + +Add `@tanstack/ai-twelvelabs`, a TwelveLabs provider adapter for video understanding. `twelvelabsText('pegasus1.5')` analyzes a video supplied as a URL, inline base64, or a pre-uploaded asset id and streams prompt-guided text via Pegasus through the standard `chat()` activity. Supports native streaming (`analyzeStream`), structured output (`json_schema` response format), clip windowing, and token usage. Opt-in and non-breaking. diff --git a/README.md b/README.md index 7b2ac19aa..5fa0546e2 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,7 @@ Official adapters include: | [`@tanstack/ai-groq`](https://tanstack.com/ai/latest/docs/adapters/groq) | Groq low-latency inference | | [`@tanstack/ai-elevenlabs`](https://tanstack.com/ai/latest/docs/adapters/elevenlabs) | ElevenLabs realtime voice, speech, transcription, music, and sound effects | | [`@tanstack/ai-fal`](https://tanstack.com/ai/latest/docs/adapters/fal) | fal.ai image, video, audio, speech, and transcription models | +| [`@tanstack/ai-twelvelabs`](https://tanstack.com/ai/latest/docs/adapters/twelvelabs) | TwelveLabs Pegasus video understanding — summaries, Q&A, and structured output | The adapter system is tree-shakeable by activity. Import `openaiText` for chat, `openaiImage` for images, `falVideo` for video, `geminiSpeech` for TTS, and so diff --git a/docs/adapters/twelvelabs.md b/docs/adapters/twelvelabs.md new file mode 100644 index 000000000..8987293f5 --- /dev/null +++ b/docs/adapters/twelvelabs.md @@ -0,0 +1,153 @@ +--- +title: TwelveLabs +id: twelvelabs-adapter +order: 9 +description: "Understand video with TwelveLabs Pegasus in TanStack AI — summaries, Q&A, chapters, and structured output via the @tanstack/ai-twelvelabs adapter." +keywords: + - tanstack ai + - twelvelabs + - pegasus + - video understanding + - multimodal + - adapter +--- + +The TwelveLabs adapter brings video understanding to TanStack AI. [TwelveLabs](https://twelvelabs.io) builds video-native foundation models; **Pegasus** reasons over a video and returns prompt-guided text — summaries, Q&A, chapters, and highlights. The adapter plugs Pegasus into the standard `chat()` and `summarize()` activities: include a video content part in a message, ask a question, and stream back text. + +## Installation + +```bash +npm install @tanstack/ai-twelvelabs +``` + +Set `TWELVELABS_API_KEY` in your environment, or pass the key explicitly. You can grab a free API key at [twelvelabs.io](https://twelvelabs.io) — there's a generous free tier. + +## Basic Usage + +```typescript +import { chat } from "@tanstack/ai"; +import { twelvelabsText } from "@tanstack/ai-twelvelabs"; + +const stream = chat({ + adapter: twelvelabsText("pegasus1.5"), + messages: [ + { + role: "user", + content: [ + { type: "text", content: "Summarize this video in one paragraph." }, + { + type: "video", + source: { type: "url", value: "https://example.com/clip.mp4" }, + }, + ], + }, + ], +}); + +for await (const chunk of stream) { + if (chunk.type === "TEXT_MESSAGE_CONTENT") { + process.stdout.write(chunk.delta); + } +} +``` + +## Supplying the Video + +A video can be supplied three ways: + +- **URL** — `{ type: "video", source: { type: "url", value } }`. Use a direct link to a raw media file; video-hosting-platform and cloud-storage sharing links are not supported. +- **Inline base64** — `{ type: "video", source: { type: "data", value, mimeType } }`. Max 30 MB. +- **Pre-uploaded asset** — set `modelOptions.assetId`. It takes precedence over any inline video part in the messages. + +## Custom API Key + +```typescript +import { chat } from "@tanstack/ai"; +import { createTwelveLabsText } from "@tanstack/ai-twelvelabs"; + +const adapter = createTwelveLabsText("pegasus1.5", process.env.TWELVELABS_API_KEY!); + +const stream = chat({ + adapter, + messages: [ + { + role: "user", + content: [ + { type: "text", content: "What happens in this clip?" }, + { type: "video", source: { type: "url", value: "https://example.com/clip.mp4" } }, + ], + }, + ], +}); +``` + +## Provider Options + +```typescript +import { chat } from "@tanstack/ai"; +import { twelvelabsText } from "@tanstack/ai-twelvelabs"; + +const stream = chat({ + adapter: twelvelabsText("pegasus1.5"), + messages: [ + { + role: "user", + content: [ + { type: "text", content: "Describe what happens." }, + { type: "video", source: { type: "url", value: "https://example.com/clip.mp4" } }, + ], + }, + ], + modelOptions: { + temperature: 0.2, + maxTokens: 2048, + // Analyze only seconds 10–30 of the video (Pegasus 1.5). + startTime: 10, + endTime: 30, + }, +}); +``` + +| Option | Description | +| ------------- | ----------------------------------------------------------------- | +| `temperature` | Sampling temperature, `0`–`1`. Default `0.2`. | +| `maxTokens` | Maximum response length, in tokens. | +| `startTime` | Start of the analysis window, in seconds (Pegasus 1.5). | +| `endTime` | End of the analysis window, in seconds (Pegasus 1.5). | +| `assetId` | Analyze a previously uploaded TwelveLabs asset instead of inline. | + +## Structured Output + +Pegasus supports a `json_schema` response format. Pass an `outputSchema` to `chat()` and the adapter constrains the model's output to it: + +```typescript +import { chat } from "@tanstack/ai"; +import { twelvelabsText } from "@tanstack/ai-twelvelabs"; +import { z } from "zod"; + +const result = await chat({ + adapter: twelvelabsText("pegasus1.5"), + messages: [ + { + role: "user", + content: [ + { type: "text", content: "Extract the summary and topics." }, + { type: "video", source: { type: "url", value: "https://example.com/clip.mp4" } }, + ], + }, + ], + outputSchema: z.object({ + summary: z.string(), + topics: z.array(z.string()), + }), +}); +``` + +## Models + +| Model | Use | +| ------------ | ----------------------------------------------------------------- | +| `pegasus1.5` | Video understanding with clip windowing and a larger token budget | +| `pegasus1.2` | General video understanding (legacy) | + +`TWELVELABS_EMBEDDING_MODELS` (`marengo3.0`) is also exported for reference — Marengo produces 512-dim multimodal embeddings over a shared text/image/audio/video space. TanStack AI does not yet expose an embeddings activity, so this adapter focuses on the Pegasus video-understanding path. diff --git a/docs/config.json b/docs/config.json index 86021af3f..86044fc21 100644 --- a/docs/config.json +++ b/docs/config.json @@ -458,6 +458,11 @@ "to": "adapters/fal", "addedAt": "2026-04-15" }, + { + "label": "TwelveLabs", + "to": "adapters/twelvelabs", + "addedAt": "2026-06-25" + }, { "label": "OpenRouter Adapter", "to": "adapters/openrouter", diff --git a/packages/ai-twelvelabs/LICENSE b/packages/ai-twelvelabs/LICENSE new file mode 100644 index 000000000..308cb68dc --- /dev/null +++ b/packages/ai-twelvelabs/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Tanner Linsley + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/ai-twelvelabs/README.md b/packages/ai-twelvelabs/README.md new file mode 100644 index 000000000..1e647ef44 --- /dev/null +++ b/packages/ai-twelvelabs/README.md @@ -0,0 +1,120 @@ +# @tanstack/ai-twelvelabs + +TwelveLabs adapter for TanStack AI — video understanding with Pegasus. + +[TwelveLabs](https://twelvelabs.io) builds video-native foundation models. +**Pegasus** reasons over a video and returns prompt-guided text (summaries, +Q&A, chapters, highlights). This adapter exposes Pegasus through the standard +TanStack AI `chat()` / `summarize()` activities: put a video content part in a +message, ask a question, and stream back text. + +## Installation + +```bash +npm install @tanstack/ai-twelvelabs @tanstack/ai +``` + +Set `TWELVELABS_API_KEY` in your environment (or pass the key explicitly). You +can grab a free API key at [twelvelabs.io](https://twelvelabs.io) — there's a +generous free tier. + +## Basic Usage + +```typescript +import { chat } from '@tanstack/ai' +import { twelvelabsText } from '@tanstack/ai-twelvelabs' + +const stream = chat({ + adapter: twelvelabsText('pegasus1.5'), + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: 'Summarize this video in one paragraph.' }, + { + type: 'video', + source: { type: 'url', value: 'https://example.com/clip.mp4' }, + }, + ], + }, + ], +}) + +for await (const chunk of stream) { + if (chunk.type === 'TEXT_MESSAGE_CONTENT') process.stdout.write(chunk.delta) +} +``` + +The video can be supplied three ways: + +- **URL** — `{ type: 'video', source: { type: 'url', value } }` (direct link to + a raw media file; hosting-platform links are not supported). +- **Inline base64** — `source: { type: 'data', value, mimeType }` (max 30 MB). +- **Pre-uploaded asset** — `modelOptions: { assetId }`, which takes precedence + over any inline video part. + +## Custom API Key + +```typescript +import { createTwelveLabsText } from '@tanstack/ai-twelvelabs' + +const adapter = createTwelveLabsText( + 'pegasus1.5', + process.env.TWELVELABS_API_KEY!, +) +``` + +## Provider Options + +```typescript +import { chat } from '@tanstack/ai' +import { twelvelabsText } from '@tanstack/ai-twelvelabs' + +const stream = chat({ + adapter: twelvelabsText('pegasus1.5'), + messages: [ + /* ... */ + ], + modelOptions: { + temperature: 0.2, + maxTokens: 2048, + startTime: 10, // analyze only seconds 10–30 (Pegasus 1.5) + endTime: 30, + }, +}) +``` + +## Structured Output + +Pegasus supports a `json_schema` response format. Pass a schema to `chat()` and +the adapter constrains the output: + +```typescript +import { chat } from '@tanstack/ai' +import { twelvelabsText } from '@tanstack/ai-twelvelabs' +import { z } from 'zod' + +const result = await chat({ + adapter: twelvelabsText('pegasus1.5'), + messages: [ + /* ... a video + prompt ... */ + ], + outputSchema: z.object({ summary: z.string(), topics: z.array(z.string()) }), +}) +``` + +## Models + +| Model | Use | +| ------------ | --------------------------------------------------------------- | +| `pegasus1.5` | Video understanding with clip windowing and larger token budget | +| `pegasus1.2` | General video understanding (legacy) | + +`TWELVELABS_EMBEDDING_MODELS` (`marengo3.0`) is also exported for reference — +Marengo produces 512-dim multimodal embeddings over a shared text/image/audio/video +space. TanStack AI does not yet expose an embeddings activity; this adapter +focuses on the Pegasus video-understanding path. + +## License + +MIT diff --git a/packages/ai-twelvelabs/package.json b/packages/ai-twelvelabs/package.json new file mode 100644 index 000000000..b427c27b1 --- /dev/null +++ b/packages/ai-twelvelabs/package.json @@ -0,0 +1,68 @@ +{ + "name": "@tanstack/ai-twelvelabs", + "version": "0.0.1", + "description": "TwelveLabs adapter for TanStack AI — video understanding with Pegasus (analyze/summarize) and multimodal embeddings with Marengo.", + "author": "Tanner Linsley", + "license": "MIT", + "homepage": "https://tanstack.com/ai", + "repository": { + "type": "git", + "url": "git+https://github.com/TanStack/ai.git", + "directory": "packages/ai-twelvelabs" + }, + "bugs": { + "url": "https://github.com/TanStack/ai/issues" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "keywords": [ + "ai", + "ai-sdk", + "typescript", + "tanstack", + "twelvelabs", + "adapter", + "video", + "video-understanding", + "multimodal", + "pegasus", + "marengo", + "embeddings" + ], + "type": "module", + "module": "./dist/esm/index.js", + "types": "./dist/esm/index.d.ts", + "exports": { + ".": { + "types": "./dist/esm/index.d.ts", + "import": "./dist/esm/index.js" + } + }, + "files": [ + "dist", + "src" + ], + "scripts": { + "build": "vite build", + "clean": "premove ./build ./dist", + "lint:fix": "eslint ./src --fix", + "test:build": "publint --strict", + "test:eslint": "eslint ./src", + "test:lib": "vitest --passWithNoTests", + "test:lib:dev": "pnpm test:lib --watch", + "test:types": "tsc" + }, + "dependencies": { + "@tanstack/ai-utils": "workspace:*", + "twelvelabs-js": "^1.2.8" + }, + "peerDependencies": { + "@tanstack/ai": "workspace:^" + }, + "devDependencies": { + "@tanstack/ai": "workspace:*", + "@vitest/coverage-v8": "4.0.14" + } +} diff --git a/packages/ai-twelvelabs/src/adapters/text.ts b/packages/ai-twelvelabs/src/adapters/text.ts new file mode 100644 index 000000000..dd4ee504c --- /dev/null +++ b/packages/ai-twelvelabs/src/adapters/text.ts @@ -0,0 +1,395 @@ +import { EventType, buildBaseUsage, normalizeSystemPrompts } from '@tanstack/ai' +import { toRunErrorRawEvent } from '@tanstack/ai/adapter-internals' +import { BaseTextAdapter } from '@tanstack/ai/adapters' +import { + createTwelveLabsClient, + generateId, + getTwelveLabsApiKeyFromEnv, +} from '../utils' +import type { + StructuredOutputOptions, + StructuredOutputResult, +} from '@tanstack/ai/adapters' +import type { + ContentPart, + Modality, + ModelMessage, + StreamChunk, + TextOptions, + TokenUsage, +} from '@tanstack/ai' +import type { TwelveLabs, TwelvelabsApi } from 'twelvelabs-js' +import type { + TWELVELABS_CHAT_MODELS, + TwelveLabsChatModelProviderOptionsByName, + TwelveLabsModelInputModalitiesByName, +} from '../model-meta' +import type { TwelveLabsMessageMetadataByModality } from '../message-types' +import type { TwelveLabsTextProviderOptions } from '../text/text-provider-options' +import type { TwelveLabsClientConfig } from '../utils' + +/** + * Configuration for the TwelveLabs text adapter. + */ +export interface TwelveLabsTextConfig extends TwelveLabsClientConfig {} + +export type { TwelveLabsTextProviderOptions } from '../text/text-provider-options' + +type TwelveLabsChatModel = (typeof TWELVELABS_CHAT_MODELS)[number] + +type ResolveProviderOptions = + TModel extends keyof TwelveLabsChatModelProviderOptionsByName + ? TwelveLabsChatModelProviderOptionsByName[TModel] + : TwelveLabsTextProviderOptions + +type ResolveInputModalities = + TModel extends keyof TwelveLabsModelInputModalitiesByName + ? TwelveLabsModelInputModalitiesByName[TModel] + : readonly ['text', 'video'] + +/** + * TwelveLabs Pegasus Text (Video Understanding) Adapter. + * + * Turns a chat turn that carries a video content part plus a text prompt into + * a Pegasus analysis. Pegasus is a video-understanding model: it reasons over + * the supplied video and returns prompt-guided text (summaries, Q&A, chapters, + * highlights). The video may be supplied as: + * + * - a URL video content part (`{ type: 'video', source: { type: 'url', value } }`), + * - an inline base64 video content part (`source.type === 'data'`), or + * - a previously uploaded TwelveLabs asset via `modelOptions.assetId`. + * + * Streaming uses TwelveLabs' native NDJSON `analyzeStream`; structured output + * uses the non-streaming `analyze` call with a `json_schema` response format. + */ +export class TwelveLabsTextAdapter< + TModel extends TwelveLabsChatModel, + TProviderOptions extends Record = ResolveProviderOptions, + TInputModalities extends ReadonlyArray = + ResolveInputModalities, +> extends BaseTextAdapter< + TModel, + TProviderOptions, + TInputModalities, + TwelveLabsMessageMetadataByModality +> { + override readonly kind = 'text' as const + readonly name = 'twelvelabs' as const + + private readonly client: TwelveLabs + + constructor(config: TwelveLabsTextConfig, model: TModel) { + super({}, model) + this.client = createTwelveLabsClient(config) + } + + async *chatStream( + options: TextOptions, + ): AsyncIterable { + const { logger } = options + const model = options.model + + const runId = options.runId ?? generateId(this.name) + const threadId = options.threadId ?? generateId(this.name) + const messageId = generateId(this.name) + + try { + const request = this.buildAnalyzeRequest(options) + logger.request( + `activity=chat provider=twelvelabs model=${model} messages=${options.messages.length} stream=true`, + { provider: 'twelvelabs', model }, + ) + + const stream = await this.client.analyzeStream(request) + + let accumulatedContent = '' + let hasEmittedTextStart = false + let usage: TokenUsage | undefined + let finishReason: 'stop' | 'length' = 'stop' + + yield { + type: EventType.RUN_STARTED, + runId, + threadId, + model, + timestamp: Date.now(), + parentRunId: options.parentRunId, + } + + for await (const event of stream) { + logger.provider(`provider=twelvelabs`, { event }) + + if (event.eventType === 'text_generation' && event.text) { + if (!hasEmittedTextStart) { + hasEmittedTextStart = true + yield { + type: EventType.TEXT_MESSAGE_START, + messageId, + model, + timestamp: Date.now(), + role: 'assistant', + } + } + accumulatedContent += event.text + yield { + type: EventType.TEXT_MESSAGE_CONTENT, + messageId, + model, + timestamp: Date.now(), + delta: event.text, + content: accumulatedContent, + } + } else if (event.eventType === 'stream_end') { + if (event.finishReason === 'length') { + finishReason = 'length' + } + const sdkUsage = event.metadata?.usage + if (sdkUsage) { + usage = buildBaseUsage({ + promptTokens: sdkUsage.inputTokens ?? 0, + completionTokens: sdkUsage.outputTokens, + totalTokens: (sdkUsage.inputTokens ?? 0) + sdkUsage.outputTokens, + }) + } + } + } + + if (hasEmittedTextStart) { + yield { + type: EventType.TEXT_MESSAGE_END, + messageId, + model, + timestamp: Date.now(), + } + } + + yield { + type: EventType.RUN_FINISHED, + runId, + threadId, + model, + timestamp: Date.now(), + finishReason, + ...(usage && { usage }), + } + } catch (error) { + const rawEvent = toRunErrorRawEvent(error) + logger.errors('twelvelabs.chatStream fatal', { + error, + source: 'twelvelabs.chatStream', + }) + yield { + type: EventType.RUN_ERROR, + runId, + model, + timestamp: Date.now(), + message: + error instanceof Error + ? error.message + : 'An unknown error occurred during the analyze stream.', + ...(rawEvent !== undefined && { rawEvent }), + error: { + message: + error instanceof Error + ? error.message + : 'An unknown error occurred during the analyze stream.', + }, + } + } + } + + /** + * Structured output via TwelveLabs' non-streaming `analyze` call with a + * `json_schema` response format. The schema arrives pre-converted to JSON + * Schema from the activity layer. + */ + async structuredOutput( + options: StructuredOutputOptions, + ): Promise> { + const { chatOptions, outputSchema } = options + const { logger } = chatOptions + + try { + const request = this.buildAnalyzeRequest(chatOptions) + logger.request( + `activity=chat provider=twelvelabs model=${chatOptions.model} messages=${chatOptions.messages.length} stream=false`, + { provider: 'twelvelabs', model: chatOptions.model }, + ) + + const result = await this.client.analyze({ + ...request, + responseFormat: { + type: 'json_schema', + jsonSchema: outputSchema, + }, + }) + + const rawText = result.data ?? '' + let parsed: unknown + try { + parsed = JSON.parse(rawText) + } catch { + throw new Error( + `Failed to parse structured output as JSON. Content: ${rawText.slice(0, 200)}${rawText.length > 200 ? '...' : ''}`, + ) + } + + return { + data: parsed, + rawText, + ...(result.usage && { + usage: buildBaseUsage({ + promptTokens: result.usage.inputTokens ?? 0, + completionTokens: result.usage.outputTokens, + totalTokens: + (result.usage.inputTokens ?? 0) + result.usage.outputTokens, + }), + }), + } + } catch (error) { + logger.errors('twelvelabs.structuredOutput fatal', { + error, + source: 'twelvelabs.structuredOutput', + }) + throw new Error( + error instanceof Error + ? error.message + : 'An unknown error occurred during structured output generation.', + ) + } + } + + /** + * Build the shared TwelveLabs analyze request from the framework's chat + * options. Extracts the video source and concatenates all text (system + * prompts + user text parts) into a single Pegasus prompt. + */ + private buildAnalyzeRequest( + options: TextOptions, + ): TwelvelabsApi.AnalyzeStreamRequest { + const modelOptions = options.modelOptions ?? {} + const prompt = this.buildPrompt(options) + + const request: TwelvelabsApi.AnalyzeStreamRequest = { + modelName: this.model, + prompt, + } + + // assetId (provider option) wins over an inline video content part. + if (modelOptions.assetId) { + request.video = { type: 'asset_id', assetId: modelOptions.assetId } + } else { + const video = this.extractVideo(options.messages) + if (!video) { + throw new Error( + 'TwelveLabs Pegasus requires a video. Provide a video content part in the messages or set modelOptions.assetId.', + ) + } + request.video = video + } + + if (modelOptions.temperature !== undefined) { + request.temperature = modelOptions.temperature + } + if (modelOptions.maxTokens !== undefined) { + request.maxTokens = modelOptions.maxTokens + } + if (modelOptions.startTime !== undefined) { + request.startTime = modelOptions.startTime + } + if (modelOptions.endTime !== undefined) { + request.endTime = modelOptions.endTime + } + + return request + } + + /** + * Concatenate system prompts and all text content parts into one prompt. + */ + private buildPrompt( + options: TextOptions, + ): string { + const parts: Array = [] + + for (const sp of normalizeSystemPrompts(options.systemPrompts)) { + if (sp.content) parts.push(sp.content) + } + + for (const msg of options.messages) { + if (msg.role === 'tool') continue + if (typeof msg.content === 'string') { + if (msg.content) parts.push(msg.content) + } else if (Array.isArray(msg.content)) { + for (const part of msg.content) { + if (part.type === 'text' && part.content) parts.push(part.content) + } + } + } + + return parts.join('\n') + } + + /** + * Find the first video content part in the message list and map it to a + * TwelveLabs `VideoContext`. + */ + private extractVideo( + messages: Array, + ): TwelvelabsApi.VideoContext | undefined { + for (const msg of messages) { + if (!Array.isArray(msg.content)) continue + for (const part of msg.content as Array) { + if (part.type === 'video') { + if (part.source.type === 'url') { + return { type: 'url', url: part.source.value } + } + return { type: 'base64_string', base64String: part.source.value } + } + } + } + return undefined + } +} + +/** + * Creates a TwelveLabs Pegasus text adapter with an explicit API key. + * + * @example + * ```typescript + * const adapter = createTwelveLabsText('pegasus1.5', 'tlk_...') + * ``` + */ +export function createTwelveLabsText( + model: TModel, + apiKey: string, +): TwelveLabsTextAdapter { + return new TwelveLabsTextAdapter({ apiKey }, model) +} + +/** + * Creates a TwelveLabs Pegasus text adapter, reading the API key from + * `TWELVELABS_API_KEY` (or `TWELVE_LABS_API_KEY`). + * + * @example + * ```typescript + * const adapter = twelvelabsText('pegasus1.5') + * const stream = chat({ + * adapter, + * messages: [ + * { + * role: 'user', + * content: [ + * { type: 'text', content: 'Summarize this video in one paragraph.' }, + * { type: 'video', source: { type: 'url', value: 'https://.../clip.mp4' } }, + * ], + * }, + * ], + * }) + * ``` + */ +export function twelvelabsText( + model: TModel, +): TwelveLabsTextAdapter { + return createTwelveLabsText(model, getTwelveLabsApiKeyFromEnv()) +} diff --git a/packages/ai-twelvelabs/src/index.ts b/packages/ai-twelvelabs/src/index.ts new file mode 100644 index 000000000..b74d95d04 --- /dev/null +++ b/packages/ai-twelvelabs/src/index.ts @@ -0,0 +1,44 @@ +// ============================================================================ +// Text (Pegasus video understanding) Adapter +// ============================================================================ + +export { + TwelveLabsTextAdapter, + createTwelveLabsText, + twelvelabsText, + type TwelveLabsTextConfig, + type TwelveLabsTextProviderOptions, +} from './adapters/text' + +// ============================================================================ +// Model Metadata +// ============================================================================ + +export { + TWELVELABS_CHAT_MODELS, + TWELVELABS_EMBEDDING_MODELS, + type TwelveLabsChatModel, + type TwelveLabsEmbeddingModel, + type TwelveLabsChatModelProviderOptionsByName, + type TwelveLabsModelInputModalitiesByName, +} from './model-meta' + +// ============================================================================ +// Message Metadata Types +// ============================================================================ + +export type { + TwelveLabsVideoMetadata, + TwelveLabsVideoMimeType, + TwelveLabsMessageMetadataByModality, +} from './message-types' + +// ============================================================================ +// Utilities +// ============================================================================ + +export { + createTwelveLabsClient, + getTwelveLabsApiKeyFromEnv, + type TwelveLabsClientConfig, +} from './utils' diff --git a/packages/ai-twelvelabs/src/message-types.ts b/packages/ai-twelvelabs/src/message-types.ts new file mode 100644 index 000000000..f147f2eeb --- /dev/null +++ b/packages/ai-twelvelabs/src/message-types.ts @@ -0,0 +1,30 @@ +import type { DefaultMessageMetadataByModality } from '@tanstack/ai' + +/** + * Supported video MIME types for TwelveLabs. + * + * @see https://docs.twelvelabs.io/v1.3/docs/concepts/supported-file-types + */ +export type TwelveLabsVideoMimeType = + | 'video/mp4' + | 'video/quicktime' + | 'video/webm' + | 'video/x-msvideo' + | 'video/mpeg' + +/** + * Metadata for TwelveLabs video content parts. The MIME type is optional — + * TwelveLabs infers it server-side from the supplied URL or asset. + */ +export interface TwelveLabsVideoMetadata { + mimeType?: TwelveLabsVideoMimeType +} + +/** + * Per-modality message metadata for TwelveLabs. Pegasus is a video + * understanding model, so only the `video` modality carries provider metadata; + * the rest fall back to the framework defaults. + */ +export interface TwelveLabsMessageMetadataByModality extends DefaultMessageMetadataByModality { + video: TwelveLabsVideoMetadata +} diff --git a/packages/ai-twelvelabs/src/model-meta.ts b/packages/ai-twelvelabs/src/model-meta.ts new file mode 100644 index 000000000..366e6fd4f --- /dev/null +++ b/packages/ai-twelvelabs/src/model-meta.ts @@ -0,0 +1,67 @@ +import type { TwelveLabsTextProviderOptions } from './text/text-provider-options' + +interface ModelMeta { + name: string + supports: { + input: Array<'text' | 'image' | 'audio' | 'video' | 'document'> + output: Array<'text' | 'image' | 'audio' | 'video'> + capabilities?: Array<'structured_output' | 'video_understanding'> + } + max_output_tokens?: number + /** Type-level description of which provider options this model supports. */ + providerOptions?: TProviderOptions +} + +const PEGASUS_1_5 = { + name: 'pegasus1.5', + max_output_tokens: 98_304, + supports: { + input: ['text', 'video'], + output: ['text'], + capabilities: ['structured_output', 'video_understanding'], + }, +} as const satisfies ModelMeta + +const PEGASUS_1_2 = { + name: 'pegasus1.2', + max_output_tokens: 4_096, + supports: { + input: ['text', 'video'], + output: ['text'], + capabilities: ['structured_output', 'video_understanding'], + }, +} as const satisfies ModelMeta + +/** + * TwelveLabs Pegasus chat / video-understanding models. Use with + * {@link createTwelveLabsText} / `twelvelabsText` and the `chat()` / + * `summarize()` activities. + */ +export const TWELVELABS_CHAT_MODELS = [ + PEGASUS_1_5.name, + PEGASUS_1_2.name, +] as const + +/** + * TwelveLabs Marengo multimodal embedding models. Marengo produces 512-dim + * embeddings for text, image, audio, and video over a single shared space. + */ +export const TWELVELABS_EMBEDDING_MODELS = ['marengo3.0'] as const + +export type TwelveLabsChatModel = (typeof TWELVELABS_CHAT_MODELS)[number] +export type TwelveLabsEmbeddingModel = + (typeof TWELVELABS_EMBEDDING_MODELS)[number] + +/** + * Per-model provider-option resolution. Both Pegasus models accept the same + * option surface today; the map keeps the door open for per-model divergence. + */ +export interface TwelveLabsChatModelProviderOptionsByName { + 'pegasus1.5': TwelveLabsTextProviderOptions + 'pegasus1.2': TwelveLabsTextProviderOptions +} + +export interface TwelveLabsModelInputModalitiesByName { + 'pegasus1.5': readonly ['text', 'video'] + 'pegasus1.2': readonly ['text', 'video'] +} diff --git a/packages/ai-twelvelabs/src/text/text-provider-options.ts b/packages/ai-twelvelabs/src/text/text-provider-options.ts new file mode 100644 index 000000000..bed7f9d62 --- /dev/null +++ b/packages/ai-twelvelabs/src/text/text-provider-options.ts @@ -0,0 +1,42 @@ +/** + * Provider-specific options for the TwelveLabs Pegasus text (video + * understanding) adapter. Threaded through `modelOptions` on a `chat()` / + * `summarize()` call. + * + * @see https://docs.twelvelabs.io/v1.3/api-reference/analyze-videos + */ +export interface TwelveLabsTextProviderOptions { + /** + * Sampling temperature, `0`–`1`. Lower values are more deterministic. + * + * **Default:** `0.2` + */ + temperature?: number + + /** + * Maximum response length, in tokens. + * + * Pegasus 1.2: `2`–`4096`. Pegasus 1.5: `512`–`98304`. Defaults to `4096`. + */ + maxTokens?: number + + /** + * Start of the analysis window, in seconds. Use with `endTime` to analyze + * only a portion of the video. Requires a Pegasus 1.5 model. The clip + * (`endTime - startTime`) must be at least 4 seconds. + */ + startTime?: number + + /** + * End of the analysis window, in seconds. Use with `startTime`. Requires a + * Pegasus 1.5 model. + */ + endTime?: number + + /** + * Analyze a previously uploaded TwelveLabs asset by id instead of supplying + * the video inline in the message. When set, it takes precedence over any + * video content part in the messages. + */ + assetId?: string +} diff --git a/packages/ai-twelvelabs/src/utils/client.ts b/packages/ai-twelvelabs/src/utils/client.ts new file mode 100644 index 000000000..04ad5d2b7 --- /dev/null +++ b/packages/ai-twelvelabs/src/utils/client.ts @@ -0,0 +1,48 @@ +import { generateId as _generateId, getApiKeyFromEnv } from '@tanstack/ai-utils' +import { TwelveLabs } from 'twelvelabs-js' + +/** + * Configuration for constructing a TwelveLabs client. + */ +export interface TwelveLabsClientConfig { + /** Your TwelveLabs API key. */ + apiKey: string +} + +/** + * Creates a TwelveLabs SDK client instance. + */ +export function createTwelveLabsClient( + config: TwelveLabsClientConfig, +): TwelveLabs { + return new TwelveLabs({ apiKey: config.apiKey }) +} + +/** + * Reads the TwelveLabs API key from the environment. + * + * Looks for `TWELVELABS_API_KEY` first, then the SDK-native + * `TWELVE_LABS_API_KEY` spelling. + * + * @throws Error if neither variable is set. + */ +export function getTwelveLabsApiKeyFromEnv(): string { + try { + return getApiKeyFromEnv('TWELVELABS_API_KEY') + } catch { + try { + return getApiKeyFromEnv('TWELVE_LABS_API_KEY') + } catch { + throw new Error( + 'TWELVELABS_API_KEY (or TWELVE_LABS_API_KEY) is not set. Please set one of these environment variables or pass the API key directly.', + ) + } + } +} + +/** + * Generates a unique ID with a prefix. + */ +export function generateId(prefix: string): string { + return _generateId(prefix) +} diff --git a/packages/ai-twelvelabs/src/utils/index.ts b/packages/ai-twelvelabs/src/utils/index.ts new file mode 100644 index 000000000..f8d3f54ef --- /dev/null +++ b/packages/ai-twelvelabs/src/utils/index.ts @@ -0,0 +1,6 @@ +export { + createTwelveLabsClient, + generateId, + getTwelveLabsApiKeyFromEnv, + type TwelveLabsClientConfig, +} from './client' diff --git a/packages/ai-twelvelabs/tests/live.test.ts b/packages/ai-twelvelabs/tests/live.test.ts new file mode 100644 index 000000000..40bcb21c7 --- /dev/null +++ b/packages/ai-twelvelabs/tests/live.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from 'vitest' +import { twelvelabsText } from '../src/adapters/text' +import type { StreamChunk } from '@tanstack/ai' + +const apiKey = process.env.TWELVELABS_API_KEY ?? process.env.TWELVE_LABS_API_KEY + +// Gated: only runs when a real API key is present. No key in CI → skipped. +const maybe = apiKey ? describe : describe.skip + +// A short, raw, TwelveLabs-ingestible sample clip (direct media URL). +const SAMPLE_VIDEO_URL = + 'https://test-videos.co.uk/vids/bigbuckbunny/mp4/h264/720/Big_Buck_Bunny_720_10s_1MB.mp4' + +function makeLogger() { + const noop = () => {} + return { request: noop, response: noop, provider: noop, errors: noop } as any +} + +maybe('TwelveLabs Pegasus live analyze', () => { + it('analyzes a public video and streams non-empty text', async () => { + const adapter = twelvelabsText('pegasus1.5') + const chunks: Array = [] + for await (const chunk of adapter.chatStream({ + model: 'pegasus1.5', + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: 'Describe this video in one sentence.' }, + { + type: 'video', + source: { type: 'url', value: SAMPLE_VIDEO_URL }, + }, + ], + }, + ], + modelOptions: { maxTokens: 512 }, + logger: makeLogger(), + })) { + chunks.push(chunk) + } + + const text = chunks + .filter((c) => c.type === 'TEXT_MESSAGE_CONTENT') + .map((c) => (c as { delta: string }).delta) + .join('') + expect(text.length).toBeGreaterThan(0) + }, 120_000) +}) diff --git a/packages/ai-twelvelabs/tests/model-meta.test.ts b/packages/ai-twelvelabs/tests/model-meta.test.ts new file mode 100644 index 000000000..9f7c9064d --- /dev/null +++ b/packages/ai-twelvelabs/tests/model-meta.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from 'vitest' +import { + TWELVELABS_CHAT_MODELS, + TWELVELABS_EMBEDDING_MODELS, +} from '../src/model-meta' + +describe('TwelveLabs model metadata', () => { + it('exposes the Pegasus chat models', () => { + expect(TWELVELABS_CHAT_MODELS).toContain('pegasus1.5') + expect(TWELVELABS_CHAT_MODELS).toContain('pegasus1.2') + }) + + it('exposes the Marengo embedding model', () => { + expect(TWELVELABS_EMBEDDING_MODELS).toContain('marengo3.0') + }) +}) diff --git a/packages/ai-twelvelabs/tests/text-adapter.test.ts b/packages/ai-twelvelabs/tests/text-adapter.test.ts new file mode 100644 index 000000000..0265327be --- /dev/null +++ b/packages/ai-twelvelabs/tests/text-adapter.test.ts @@ -0,0 +1,186 @@ +import { EventType } from '@tanstack/ai' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { StreamChunk, TextOptions } from '@tanstack/ai' + +const analyzeStreamMock = vi.fn() +const analyzeMock = vi.fn() + +vi.mock('twelvelabs-js', () => ({ + TwelveLabs: class { + analyzeStream = analyzeStreamMock + analyze = analyzeMock + }, +})) + +import { createTwelveLabsText } from '../src/adapters/text' + +function makeLogger() { + return { + request: vi.fn(), + response: vi.fn(), + provider: vi.fn(), + errors: vi.fn(), + } as unknown as TextOptions['logger'] +} + +async function* fakeStream( + events: Array>, +): AsyncIterable> { + for (const ev of events) yield ev +} + +function videoMessageOptions(): TextOptions { + return { + model: 'pegasus1.5', + messages: [ + { + role: 'user', + content: [ + { type: 'text', content: 'What happens in this video?' }, + { + type: 'video', + source: { type: 'url', value: 'https://example.com/clip.mp4' }, + }, + ], + }, + ], + logger: makeLogger(), + } +} + +async function collect( + iter: AsyncIterable, +): Promise> { + const out: Array = [] + for await (const chunk of iter) out.push(chunk) + return out +} + +describe('TwelveLabs Pegasus text adapter', () => { + beforeEach(() => { + analyzeStreamMock.mockReset() + analyzeMock.mockReset() + }) + + it('maps a video URL + text prompt to the analyzeStream request', async () => { + analyzeStreamMock.mockResolvedValue( + fakeStream([ + { eventType: 'stream_start', metadata: {} }, + { eventType: 'text_generation', text: 'A dog ' }, + { eventType: 'text_generation', text: 'runs.' }, + { + eventType: 'stream_end', + finishReason: 'stop', + metadata: { usage: { inputTokens: 100, outputTokens: 5 } }, + }, + ]), + ) + + const adapter = createTwelveLabsText('pegasus1.5', 'test-key') + await collect(adapter.chatStream(videoMessageOptions())) + + expect(analyzeStreamMock).toHaveBeenCalledTimes(1) + const req = analyzeStreamMock.mock.calls[0]![0] + expect(req).toMatchObject({ + modelName: 'pegasus1.5', + prompt: 'What happens in this video?', + video: { type: 'url', url: 'https://example.com/clip.mp4' }, + }) + }) + + it('emits RUN_STARTED → TEXT_MESSAGE_* → RUN_FINISHED with accumulated text and usage', async () => { + analyzeStreamMock.mockResolvedValue( + fakeStream([ + { eventType: 'text_generation', text: 'A dog ' }, + { eventType: 'text_generation', text: 'runs.' }, + { + eventType: 'stream_end', + finishReason: 'stop', + metadata: { usage: { inputTokens: 100, outputTokens: 5 } }, + }, + ]), + ) + + const adapter = createTwelveLabsText('pegasus1.5', 'test-key') + const chunks = await collect(adapter.chatStream(videoMessageOptions())) + const types = chunks.map((c) => c.type) + + expect(types[0]).toBe(EventType.RUN_STARTED) + expect(types).toContain(EventType.TEXT_MESSAGE_START) + expect(types).toContain(EventType.TEXT_MESSAGE_END) + expect(types[types.length - 1]).toBe(EventType.RUN_FINISHED) + + const content = chunks + .filter((c) => c.type === EventType.TEXT_MESSAGE_CONTENT) + .map((c) => (c as { delta: string }).delta) + .join('') + expect(content).toBe('A dog runs.') + + const finished = chunks[chunks.length - 1] as { + finishReason: string + usage?: { promptTokens?: number; completionTokens?: number } + } + expect(finished.finishReason).toBe('stop') + expect(finished.usage).toMatchObject({ + promptTokens: 100, + completionTokens: 5, + totalTokens: 105, + }) + }) + + it('prefers modelOptions.assetId over an inline video part', async () => { + analyzeStreamMock.mockResolvedValue(fakeStream([])) + const adapter = createTwelveLabsText('pegasus1.5', 'test-key') + + const opts = videoMessageOptions() + opts.modelOptions = { + assetId: 'asset_123', + temperature: 0.5, + maxTokens: 1024, + } + await collect(adapter.chatStream(opts)) + + const req = analyzeStreamMock.mock.calls[0]![0] + expect(req.video).toEqual({ type: 'asset_id', assetId: 'asset_123' }) + expect(req.temperature).toBe(0.5) + expect(req.maxTokens).toBe(1024) + }) + + it('emits RUN_ERROR when no video is supplied', async () => { + const adapter = createTwelveLabsText('pegasus1.5', 'test-key') + const chunks = await collect( + adapter.chatStream({ + model: 'pegasus1.5', + messages: [{ role: 'user', content: 'No video here' }], + logger: makeLogger(), + }), + ) + const errorChunk = chunks.find((c) => c.type === EventType.RUN_ERROR) as + | { message: string } + | undefined + expect(errorChunk).toBeDefined() + expect(errorChunk!.message).toMatch(/requires a video/i) + expect(analyzeStreamMock).not.toHaveBeenCalled() + }) + + it('structuredOutput sends a json_schema response format and parses the result', async () => { + analyzeMock.mockResolvedValue({ + data: '{"summary":"a dog runs"}', + usage: { inputTokens: 100, outputTokens: 6 }, + }) + const adapter = createTwelveLabsText('pegasus1.5', 'test-key') + + const result = await adapter.structuredOutput({ + chatOptions: videoMessageOptions(), + outputSchema: { + type: 'object', + properties: { summary: { type: 'string' } }, + }, + }) + + const req = analyzeMock.mock.calls[0]![0] + expect(req.responseFormat.type).toBe('json_schema') + expect(result.data).toEqual({ summary: 'a dog runs' }) + expect(result.rawText).toBe('{"summary":"a dog runs"}') + }) +}) diff --git a/packages/ai-twelvelabs/tsconfig.json b/packages/ai-twelvelabs/tsconfig.json new file mode 100644 index 000000000..c38689f4e --- /dev/null +++ b/packages/ai-twelvelabs/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": ["src", "tests"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/ai-twelvelabs/vite.config.ts b/packages/ai-twelvelabs/vite.config.ts new file mode 100644 index 000000000..11f5b20b7 --- /dev/null +++ b/packages/ai-twelvelabs/vite.config.ts @@ -0,0 +1,37 @@ +import { defineConfig, mergeConfig } from 'vitest/config' +import { tanstackViteConfig } from '@tanstack/vite-config' +import packageJson from './package.json' + +const config = defineConfig({ + test: { + name: packageJson.name, + dir: './', + watch: false, + + globals: true, + environment: 'node', + include: ['tests/**/*.test.ts'], + coverage: { + provider: 'v8', + reporter: ['text', 'json', 'html', 'lcov'], + exclude: [ + 'node_modules/', + 'dist/', + 'tests/', + '**/*.test.ts', + '**/*.config.ts', + '**/types.ts', + ], + include: ['src/**/*.ts'], + }, + }, +}) + +export default mergeConfig( + config, + tanstackViteConfig({ + entry: ['./src/index.ts'], + srcDir: './src', + cjs: false, + }), +) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 40f5181e4..8e8dfcf3a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -49,7 +49,7 @@ importers: version: 0.5.0 knip: specifier: ^5.70.2 - version: 5.73.4(@types/node@24.10.3)(typescript@5.9.3) + version: 5.73.4(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)(@types/node@24.10.3)(typescript@5.9.3) markdown-link-extractor: specifier: ^4.0.3 version: 4.0.3 @@ -1699,7 +1699,7 @@ importers: version: 1.9.10 tsdown: specifier: ^0.17.0-beta.6 - version: 0.17.3(oxc-resolver@11.15.0)(publint@0.3.16)(typescript@5.9.3) + version: 0.17.3(oxc-resolver@11.15.0(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1))(publint@0.3.16)(typescript@5.9.3) typescript: specifier: 5.9.3 version: 5.9.3 @@ -1781,6 +1781,22 @@ importers: specifier: ^7.3.3 version: 7.3.3(@types/node@24.10.3)(jiti@2.6.1)(less@4.6.6)(lightningcss@1.30.2)(sass@1.101.0)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2) + packages/ai-twelvelabs: + dependencies: + '@tanstack/ai-utils': + specifier: workspace:* + version: link:../ai-utils + twelvelabs-js: + specifier: ^1.2.8 + version: 1.2.8 + devDependencies: + '@tanstack/ai': + specifier: workspace:* + version: link:../ai + '@vitest/coverage-v8': + specifier: 4.0.14 + version: 4.0.14(vitest@4.1.4) + packages/ai-utils: devDependencies: '@types/node': @@ -1819,7 +1835,7 @@ importers: version: 27.3.0(postcss@8.5.15) tsdown: specifier: ^0.17.0-beta.6 - version: 0.17.3(oxc-resolver@11.15.0)(publint@0.3.16)(typescript@5.9.3) + version: 0.17.3(oxc-resolver@11.15.0(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1))(publint@0.3.16)(typescript@5.9.3) typescript: specifier: 5.9.3 version: 5.9.3 @@ -10102,6 +10118,10 @@ packages: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} + form-data-encoder@4.1.0: + resolution: {integrity: sha512-G6NsmEW15s0Uw9XnCg+33H3ViYRyiM0hMrMhhqQOR8NFc5GhYrI+6I3u7OTw7b91J2g8rtvMBZJDbcGb2YUniw==} + engines: {node: '>= 18'} + form-data@4.0.5: resolution: {integrity: sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==} engines: {node: '>= 6'} @@ -10111,6 +10131,10 @@ packages: engines: {node: '>=18.3.0'} hasBin: true + formdata-node@6.0.3: + resolution: {integrity: sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==} + engines: {node: '>= 18'} + formdata-polyfill@4.0.10: resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} engines: {node: '>=12.20.0'} @@ -13702,6 +13726,9 @@ packages: tw-animate-css@1.4.0: resolution: {integrity: sha512-7bziOlRqH0hJx80h/3mbicLW7o8qLsH5+RaLR2t+OHM3D0JlWGODQKQ4cxbK7WlvmUxpcj6Kgu6EKqjrGFe3QQ==} + twelvelabs-js@1.2.8: + resolution: {integrity: sha512-LFOIp0zUA1YmOGW2Q8ugav81ln0XTsyPpLki4iCGiJFfsnWx9by47FUbLf2g08EfNwZp8n0G44cTtqjPGn5mSw==} + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -14145,6 +14172,9 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + url-join@4.0.1: + resolution: {integrity: sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==} + use-callback-ref@1.3.3: resolution: {integrity: sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==} engines: {node: '>=10'} @@ -17244,7 +17274,7 @@ snapshots: '@img/sharp-wasm32@0.34.5': dependencies: - '@emnapi/runtime': 1.10.0 + '@emnapi/runtime': 1.11.1 optional: true '@img/sharp-win32-arm64@0.34.5': @@ -17838,7 +17868,7 @@ snapshots: '@oxc-resolver/binding-wasm32-wasi@11.15.0(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)': dependencies: - '@napi-rs/wasm-runtime': 1.1.4(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1) + '@napi-rs/wasm-runtime': 1.1.5(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1) transitivePeerDependencies: - '@emnapi/core' - '@emnapi/runtime' @@ -22957,7 +22987,7 @@ snapshots: dotenv@8.6.0: {} - dts-resolver@2.1.3(oxc-resolver@11.15.0): + dts-resolver@2.1.3(oxc-resolver@11.15.0(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)): optionalDependencies: oxc-resolver: 11.15.0(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1) @@ -23803,6 +23833,8 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 + form-data-encoder@4.1.0: {} + form-data@4.0.5: dependencies: asynckit: 0.4.0 @@ -23815,6 +23847,8 @@ snapshots: dependencies: fd-package-json: 2.0.0 + formdata-node@6.0.3: {} + formdata-polyfill@4.0.10: dependencies: fetch-blob: 3.2.0 @@ -24829,7 +24863,7 @@ snapshots: klona@2.0.6: {} - knip@5.73.4(@types/node@24.10.3)(typescript@5.9.3): + knip@5.73.4(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)(@types/node@24.10.3)(typescript@5.9.3): dependencies: '@nodelib/fs.walk': 1.2.8 '@types/node': 24.10.3 @@ -27469,14 +27503,14 @@ snapshots: robot3@0.4.1: {} - rolldown-plugin-dts@0.18.3(oxc-resolver@11.15.0)(rolldown@1.0.0-beta.53)(typescript@5.9.3): + rolldown-plugin-dts@0.18.3(oxc-resolver@11.15.0(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1))(rolldown@1.0.0-beta.53)(typescript@5.9.3): dependencies: '@babel/generator': 7.28.5 '@babel/parser': 7.29.0 '@babel/types': 7.29.0 ast-kit: 2.2.0 birpc: 3.0.0 - dts-resolver: 2.1.3(oxc-resolver@11.15.0) + dts-resolver: 2.1.3(oxc-resolver@11.15.0(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1)) get-tsconfig: 4.13.0 magic-string: 0.30.21 obug: 2.1.1 @@ -28433,7 +28467,7 @@ snapshots: minimist: 1.2.8 strip-bom: 3.0.0 - tsdown@0.17.3(oxc-resolver@11.15.0)(publint@0.3.16)(typescript@5.9.3): + tsdown@0.17.3(oxc-resolver@11.15.0(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1))(publint@0.3.16)(typescript@5.9.3): dependencies: ansis: 4.2.0 cac: 6.7.14 @@ -28443,7 +28477,7 @@ snapshots: import-without-cache: 0.2.3 obug: 2.1.1 rolldown: 1.0.0-beta.53 - rolldown-plugin-dts: 0.18.3(oxc-resolver@11.15.0)(rolldown@1.0.0-beta.53)(typescript@5.9.3) + rolldown-plugin-dts: 0.18.3(oxc-resolver@11.15.0(@emnapi/core@1.11.1)(@emnapi/runtime@1.11.1))(rolldown@1.0.0-beta.53)(typescript@5.9.3) semver: 7.7.3 tinyexec: 1.0.2 tinyglobby: 0.2.15 @@ -28509,6 +28543,18 @@ snapshots: tw-animate-css@1.4.0: {} + twelvelabs-js@1.2.8: + dependencies: + form-data: 4.0.5 + form-data-encoder: 4.1.0 + formdata-node: 6.0.3 + node-fetch: 2.7.0 + qs: 6.14.0 + readable-stream: 4.7.0 + url-join: 4.0.1 + transitivePeerDependencies: + - encoding + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 @@ -28853,6 +28899,8 @@ snapshots: dependencies: punycode: 2.3.1 + url-join@4.0.1: {} + use-callback-ref@1.3.3(@types/react@19.2.7)(react@19.2.3): dependencies: react: 19.2.3