diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dff0524..bb4b5a0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,10 @@ name: Test logtail-lambda-extension -on: [push, workflow_dispatch] +on: + push: + branches: [main] + pull_request: + workflow_dispatch: jobs: test: runs-on: ubuntu-22.04 @@ -17,7 +21,7 @@ jobs: - name: Setup node uses: actions/setup-node@v4 with: - node-version: 20 + node-version: 22 - name: Cache node_modules uses: actions/cache@v4 diff --git a/README.md b/README.md index 1e356ad..7da867f 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,33 @@ # logtail-lambda-extension -[![Test logtail-lambda-extension](https://github.com/stockstory/logtail-lambda-extension/actions/workflows/test.yml/badge.svg)](https://github.com/stockstory/logtail-lambda-extension/actions/workflows/test.yml) +[![Test logtail-lambda-extension](https://github.com/logtail/logtail-lambda-extension/actions/workflows/test.yml/badge.svg)](https://github.com/logtail/logtail-lambda-extension/actions/workflows/test.yml) -[AWS Lambda Extension](https://docs.aws.amazon.com/lambda/latest/dg/lambda-extensions.html) that uses the [Lambda Telemetry API](https://docs.aws.amazon.com/lambda/latest/dg/telemetry-api-reference.html) to forward logs to a [Logtail HTTP API source](https://docs.logtail.com/integrations/rest-api). +[AWS Lambda Extension](https://docs.aws.amazon.com/lambda/latest/dg/lambda-extensions.html) that uses the [Lambda Telemetry API](https://docs.aws.amazon.com/lambda/latest/dg/telemetry-api-reference.html) to forward logs to a [Better Stack HTTP source](https://betterstack.com/docs/logs/http-rest-api/). # Usage In order to use the extension you must first build and deploy it into your AWS infrastructure: ```shell -$ git clone git@github.com:stockstory/logtail-lambda-extension.git +$ git clone git@github.com:logtail/logtail-lambda-extension.git $ cd logtail-lambda-extension $ pnpm install $ pnpm run build $ aws lambda publish-layer-version \ --layer-name "logtail-lambda-extension" \ --zip-file "fileb://./dist/extension.zip" \ - --compatible-runtimes nodejs20.x \ + --compatible-runtimes nodejs22.x nodejs24.x \ --compatible-architectures x86_64 arm64 ``` -Once deployed set the `LOGTAIL_TOKEN` environment variable on your Lambda to your HTTP API source from Logtail, and [add the layer](https://docs.aws.amazon.com/lambda/latest/dg/invocation-layers.html) to your Lambda function. -On next invocation your Lambda logs should start appearing in your Logtail console. +Once deployed set the `SOURCE_TOKEN` and `INGESTING_HOST` environment variables on your Lambda to the token and ingesting host from your Better Stack source settings (the ingesting host looks like `s123456.lu-trq-1.betterstackdata.com`), and [add the layer](https://docs.aws.amazon.com/lambda/latest/dg/invocation-layers.html) to your Lambda function. +On next invocation your Lambda logs should start appearing in **Telemetry** -> [Live tail](https://telemetry.betterstack.com/team/0/tail). ## Environment Variables | Variable | Description | Default | | ---------------------------------------- | ------------------------------------------------------- | ------------------------------------ | -| LOGTAIL_TOKEN | Logtail source bearer token | **required** | -| LOGTAIL_HTTP_API_URL | Logtail HTTP source ingestion API URL | `https://in.logtail.com/` | +| SOURCE_TOKEN | Better Stack source token | **required** | +| INGESTING_HOST | Better Stack ingesting host | `in.logs.betterstack.com` | | EXTENSION_NAME | Name of the lambda extension | `logtail-lambda-extension` | | RECEIVER_ADDRESS | Address of the logs http receiver | `sandbox` | | RECEIVER_PORT | Port of the logs http receiver | `4243` | @@ -38,6 +38,8 @@ On next invocation your Lambda logs should start appearing in your Logtail conso | AWS_LAMBDA_RUNTIME_EXTENSION_API_VERSION | Lambda Extension API version date | `2020-01-01` | | AWS_LAMBDA_RUNTIME_TELEMETRY_API_VERSION | Telemetry API version date | `2022-07-01` | +`SOURCE_TOKEN` and `INGESTING_HOST` fall back to the legacy `LOGTAIL_TOKEN` and `LOGTAIL_HTTP_API_URL` (a full URL) variables when they are not set. + ### License Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. See [LICENSE](./LICENSE) for more information. diff --git a/package.json b/package.json index 738d624..96fcb6f 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "logtail-lambda-extension", "version": "1.0.0", - "description": "", + "description": "AWS Lambda extension that forwards function logs to Better Stack.", "main": "index.js", "engines": { "node": ">=16.0.0", diff --git a/scripts/build.ts b/scripts/build.ts index 2fcdfcc..dc69044 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -44,7 +44,7 @@ const run = () => bundle: true, minify: false, platform: 'node', - target: 'node20', + target: 'node22', outfile: outputIndex, }), (error) => new Error(`Error during build: ${error instanceof Error ? error.message : error}`), diff --git a/src/env.test.ts b/src/env.test.ts index 5639e7a..c5164e3 100644 --- a/src/env.test.ts +++ b/src/env.test.ts @@ -2,11 +2,12 @@ import { either as E } from 'fp-ts'; import { parseEnvironmentVariables } from '~/env'; import { ZodError } from 'zod'; -describe('test parseEnvironment`', () => { - test('should succeed and populate defaults', () => { +describe('parseEnvironmentVariables', () => { + test('resolves new names and populates defaults', () => { const env = { AWS_LAMBDA_RUNTIME_API: 'http://127.0.0.1:9001/', - LOGTAIL_TOKEN: 'test_token', + SOURCE_TOKEN: 'source_token', + INGESTING_HOST: 's123456.lu-trq-1.betterstackdata.com', }; const result = parseEnvironmentVariables(env); @@ -14,26 +15,76 @@ describe('test parseEnvironment`', () => { expect(result).toStrictEqual({ _tag: 'Right', right: { + NODE_ENV: 'production', AWS_LAMBDA_RUNTIME_API: 'http://127.0.0.1:9001/', AWS_LAMBDA_RUNTIME_EXTENSION_API_VERSION: '2020-01-01', AWS_LAMBDA_RUNTIME_TELEMETRY_API_VERSION: '2022-07-01', EXTENSION_NAME: 'logtail-lambda-extension', - LOGTAIL_HTTP_API_URL: 'https://in.logtail.com/', - LOGTAIL_TOKEN: 'test_token', - MAX_BYTES: 262144, - MAX_ITEMS: 10000, - NODE_ENV: 'production', RECEIVER_ADDRESS: 'sandbox', - RECEIVER_PORT: 4243, + MAX_ITEMS: 10000, + MAX_BYTES: 262144, TIMEOUT_MS: 1000, + RECEIVER_PORT: 4243, + sourceToken: 'source_token', + ingestionUrl: 'https://s123456.lu-trq-1.betterstackdata.com/', }, }); }); - test('should fail when required fields are missing', () => { - const env = {}; - const result = parseEnvironmentVariables(env); + test('falls back to legacy token and full URL used as-is', () => { + const result = parseEnvironmentVariables({ + AWS_LAMBDA_RUNTIME_API: 'http://127.0.0.1:9001/', + LOGTAIL_TOKEN: 'legacy_token', + LOGTAIL_HTTP_API_URL: 'https://in.logtail.com/', + }); + + if (!E.isRight(result)) throw new Error('expected Right'); + expect(result.right.sourceToken).toBe('legacy_token'); + expect(result.right.ingestionUrl).toBe('https://in.logtail.com/'); + }); + + test('defaults the ingestion host when no host or legacy URL is set', () => { + const result = parseEnvironmentVariables({ + AWS_LAMBDA_RUNTIME_API: 'http://127.0.0.1:9001/', + SOURCE_TOKEN: 'source_token', + }); + + if (!E.isRight(result)) throw new Error('expected Right'); + expect(result.right.ingestionUrl).toBe('https://in.logs.betterstack.com/'); + }); + + test('SOURCE_TOKEN wins over LOGTAIL_TOKEN', () => { + const result = parseEnvironmentVariables({ + AWS_LAMBDA_RUNTIME_API: 'http://127.0.0.1:9001/', + SOURCE_TOKEN: 'preferred', + LOGTAIL_TOKEN: 'legacy', + }); + + if (!E.isRight(result)) throw new Error('expected Right'); + expect(result.right.sourceToken).toBe('preferred'); + }); + + test('INGESTING_HOST wins over LOGTAIL_HTTP_API_URL', () => { + const result = parseEnvironmentVariables({ + AWS_LAMBDA_RUNTIME_API: 'http://127.0.0.1:9001/', + SOURCE_TOKEN: 'source_token', + INGESTING_HOST: 's123456.lu-trq-1.betterstackdata.com', + LOGTAIL_HTTP_API_URL: 'https://in.logtail.com/', + }); + + if (!E.isRight(result)) throw new Error('expected Right'); + expect(result.right.ingestionUrl).toBe('https://s123456.lu-trq-1.betterstackdata.com/'); + }); + + test('fails with ZodError when a runtime API is set but no token', () => { + const result = parseEnvironmentVariables({ AWS_LAMBDA_RUNTIME_API: 'http://127.0.0.1:9001/' }); + + expect(E.isLeft(result) && result.left instanceof ZodError).toBe(true); + }); + + test('fails with ZodError on completely empty env', () => { + const result = parseEnvironmentVariables({}); - expect(E.isLeft(result) && result.left instanceof ZodError).toStrictEqual(true); + expect(E.isLeft(result) && result.left instanceof ZodError).toBe(true); }); }); diff --git a/src/env.ts b/src/env.ts index f8faca7..f619904 100644 --- a/src/env.ts +++ b/src/env.ts @@ -6,7 +6,10 @@ const envNumber = z .transform((val) => Number(val)) .refine((val) => !isNaN(val)); -const envSchema = z.object({ +// Better Stack's general-purpose ingesting host; a source's own host looks like s123456.lu-trq-1.betterstackdata.com. +const DEFAULT_INGESTING_HOST = 'in.logs.betterstack.com'; + +const baseSchema = z.object({ NODE_ENV: z.union([z.literal('production'), z.literal('development'), z.literal('test')]).default('production'), AWS_LAMBDA_RUNTIME_API: z.string(), // Provided by the Lambda Env, usually http://127.0.0.1:9001/ AWS_LAMBDA_RUNTIME_EXTENSION_API_VERSION: z.string().default('2020-01-01'), // Extension API version @@ -17,17 +20,40 @@ const envSchema = z.object({ MAX_BYTES: envNumber.default('262144'), // Maximum size in bytes that the logs are buffered in memory. TIMEOUT_MS: envNumber.default('1000'), // Maximum time (in milliseconds) that a batch is buffered. RECEIVER_PORT: envNumber.default('4243'), // HTTP server receiving port - LOGTAIL_HTTP_API_URL: z.string().url().default('https://in.logtail.com/'), // Logtail HTTP API ingestion URL - LOGTAIL_TOKEN: z.string(), // Logtail token, obtain yours via the sources UI + // Better Stack source token; SOURCE_TOKEN is preferred, LOGTAIL_TOKEN is kept as a fallback. + SOURCE_TOKEN: z.string().optional(), + LOGTAIL_TOKEN: z.string().optional(), + // Better Stack ingesting host; INGESTING_HOST is preferred, LOGTAIL_HTTP_API_URL (a full URL) is kept as a fallback. + INGESTING_HOST: z.string().optional(), + LOGTAIL_HTTP_API_URL: z.string().url().optional(), }); +const envSchema = baseSchema.transform( + ({ SOURCE_TOKEN, LOGTAIL_TOKEN, INGESTING_HOST, LOGTAIL_HTTP_API_URL, ...rest }, ctx) => { + const sourceToken = SOURCE_TOKEN || LOGTAIL_TOKEN; + if (!sourceToken) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: 'Missing source token: set SOURCE_TOKEN (or the legacy LOGTAIL_TOKEN).', + }); + return z.NEVER; + } + + const ingestionUrl = INGESTING_HOST + ? `https://${INGESTING_HOST}/` + : LOGTAIL_HTTP_API_URL || `https://${DEFAULT_INGESTING_HOST}/`; + + return { ...rest, sourceToken, ingestionUrl }; + }, +); + export type EnvironmentVars = z.infer; export const parseEnvironmentVariables = ( env: Record = process.env, ): E.Either => F.pipe( - envSchema.shape, + baseSchema.shape, R.mapWithIndex((key) => env[key]), (vars) => envSchema.safeParse(vars), (parsed) => (parsed.success ? E.right(parsed.data) : E.left(parsed.error)), diff --git a/src/extension.ts b/src/extension.ts index ab022fe..4a95232 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -3,7 +3,7 @@ import { EnvironmentVars } from '~/env'; import { pollForNextEvent, registerExtension } from '~/aws/api'; import { startTelemetryHttpListener } from '~/httpListener'; import { subscribeTelemetry, SubscriptionBody } from '~/aws/subscribe'; -import { logtailLogForwarder } from '~/forwarders/logtail'; +import { betterStackLogForwarder } from '~/forwarders/betterStack'; import { AbortHandler } from '~/abortHandler'; // noinspection HttpUrlsUsage @@ -52,8 +52,8 @@ export const createExtension = ( (body) => subscribeTelemetry(telemetryBaseUrl, extensionId, body), ), ), - TE.bind('forwardLogs', ({ LOGTAIL_TOKEN, LOGTAIL_HTTP_API_URL, listener }) => - TE.of(logtailLogForwarder(LOGTAIL_TOKEN, LOGTAIL_HTTP_API_URL, listener)), + TE.bind('forwardLogs', ({ sourceToken, ingestionUrl, listener }) => + TE.of(betterStackLogForwarder(sourceToken, ingestionUrl, listener)), ), TE.map(({ EXTENSION_NAME, extensionId, extensionBaseUrl, forwardLogs, listener }) => F.pipe( diff --git a/src/forwarders/logtail.test.ts b/src/forwarders/betterStack.test.ts similarity index 89% rename from src/forwarders/logtail.test.ts rename to src/forwarders/betterStack.test.ts index c9a0952..66057a9 100644 --- a/src/forwarders/logtail.test.ts +++ b/src/forwarders/betterStack.test.ts @@ -1,15 +1,15 @@ import fetchMock from 'jest-fetch-mock'; import { Request } from 'node-fetch'; import { either as E } from 'fp-ts'; -import { logtailLogForwarder, parseMessageWithPowertoolsLogFormat } from '~/forwarders/logtail'; +import { betterStackLogForwarder, parseMessageWithPowertoolsLogFormat } from '~/forwarders/betterStack'; import { FunctionLogEvent } from '~/aws/events'; -describe('test logtail log forwarding', () => { +describe('test Better Stack log forwarding', () => { beforeEach(() => { fetchMock.resetMocks(); }); - const ingestionUrl = 'https://in.logtail.com/'; + const ingestionUrl = 'https://in.logs.betterstack.com/'; const token = 'd4ed7843'; const log: FunctionLogEvent = { @@ -27,7 +27,7 @@ describe('test logtail log forwarding', () => { logsQueue: [log, log, log], }; - const result = await logtailLogForwarder(token, ingestionUrl, listener)(); + const result = await betterStackLogForwarder(token, ingestionUrl, listener)(); expect(E.left(result)).toBeTruthy(); @@ -53,7 +53,7 @@ describe('test logtail log forwarding', () => { logsQueue: [log, log, log], }; - const result = await logtailLogForwarder(token, ingestionUrl, listener)(); + const result = await betterStackLogForwarder(token, ingestionUrl, listener)(); expect(E.left(result)).toBeTruthy(); diff --git a/src/forwarders/logtail.ts b/src/forwarders/betterStack.ts similarity index 98% rename from src/forwarders/logtail.ts rename to src/forwarders/betterStack.ts index c8583bd..5770c2a 100644 --- a/src/forwarders/logtail.ts +++ b/src/forwarders/betterStack.ts @@ -35,7 +35,7 @@ export const parseMessageWithPowertoolsLogFormat = (message: string): E.Either (): Promise => { const logs = listener.logsQueue.splice(0);