From 954f2d21c56d6b5afd6f835c155f65e1487543de Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 07:58:24 +0000 Subject: [PATCH 01/21] feat: ship redacted node logs to Better Stack --- .github/scripts/package-lock.json | 2452 +++++++++++++++++ .github/scripts/package.json | 9 + .github/scripts/phala-sync-allowed-envs.mjs | 107 + .github/workflows/docker.yml | 11 + .github/workflows/rust.yml | 5 + CHANGELOG.md | 1 + docker-compose.dstack-postgres.yaml | 269 +- .../tc-959-betterstack-log-shipping.md | 33 + log-shipper/fields.json | 92 + log-shipper/vector.tests.yaml | 341 +++ log-shipper/vector.yaml | 197 ++ scripts/node-vector-schema.mjs | 221 ++ scripts/node-vector-schema.test.mjs | 56 + tinycloud-node-server/src/main.rs | 48 +- .../src/tunnel/connection.rs | 2 +- 15 files changed, 3810 insertions(+), 34 deletions(-) create mode 100644 .github/scripts/package-lock.json create mode 100644 .github/scripts/package.json create mode 100644 .github/scripts/phala-sync-allowed-envs.mjs create mode 100644 docs/runbooks/tc-959-betterstack-log-shipping.md create mode 100644 log-shipper/fields.json create mode 100644 log-shipper/vector.tests.yaml create mode 100644 log-shipper/vector.yaml create mode 100644 scripts/node-vector-schema.mjs create mode 100644 scripts/node-vector-schema.test.mjs diff --git a/.github/scripts/package-lock.json b/.github/scripts/package-lock.json new file mode 100644 index 00000000..170c60b9 --- /dev/null +++ b/.github/scripts/package-lock.json @@ -0,0 +1,2452 @@ +{ + "name": "tinycloud-node-deploy-tools", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tinycloud-node-deploy-tools", + "dependencies": { + "@phala/cloud": "0.3.0", + "phala": "1.1.19" + } + }, + "node_modules/@adraffy/ens-normalize": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@adraffy/ens-normalize/-/ens-normalize-1.11.1.tgz", + "integrity": "sha512-nhCBV3quEgesuf7c7KYfperqSS14T8bYuvJ8PcLJp6znkZpFc0AuW4qBtr8eKVyPPe/8RSr7sglCWPU5eaxwKQ==", + "license": "MIT" + }, + "node_modules/@iarna/toml": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/@iarna/toml/-/toml-2.2.5.tgz", + "integrity": "sha512-trnsAYxU3xnS1gPHPyU961coFyLkh4gAD/0zQ5mymY4yOZ+CYvsPqUbOFSw0aDM4y0tV7tiFxL/1XfXPNC6IPg==", + "license": "ISC" + }, + "node_modules/@inquirer/ansi": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@inquirer/ansi/-/ansi-1.0.2.tgz", + "integrity": "sha512-S8qNSZiYzFd0wAcyG5AXCvUHC5Sr7xpZ9wZ2py9XR88jUz8wooStVx5M6dRzczbBWjic9NP7+rY0Xi7qqK/aMQ==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/checkbox": { + "version": "4.3.2", + "resolved": "https://registry.npmjs.org/@inquirer/checkbox/-/checkbox-4.3.2.tgz", + "integrity": "sha512-VXukHf0RR1doGe6Sm4F0Em7SWYLTHSsbGfJdS9Ja2bX5/D5uwVOEjr07cncLROdBvmnvCATYEWlHqYmXv2IlQA==", + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^1.0.2", + "@inquirer/core": "^10.3.2", + "@inquirer/figures": "^1.0.15", + "@inquirer/type": "^3.0.10", + "yoctocolors-cjs": "^2.1.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/confirm": { + "version": "5.1.21", + "resolved": "https://registry.npmjs.org/@inquirer/confirm/-/confirm-5.1.21.tgz", + "integrity": "sha512-KR8edRkIsUayMXV+o3Gv+q4jlhENF9nMYUZs9PA2HzrXeHI8M5uDag70U7RJn9yyiMZSbtF5/UexBtAVtZGSbQ==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/core": { + "version": "10.3.2", + "resolved": "https://registry.npmjs.org/@inquirer/core/-/core-10.3.2.tgz", + "integrity": "sha512-43RTuEbfP8MbKzedNqBrlhhNKVwoK//vUFNW3Q3vZ88BLcrs4kYpGg+B2mm5p2K/HfygoCxuKwJJiv8PbGmE0A==", + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^1.0.2", + "@inquirer/figures": "^1.0.15", + "@inquirer/type": "^3.0.10", + "cli-width": "^4.1.0", + "mute-stream": "^2.0.0", + "signal-exit": "^4.1.0", + "wrap-ansi": "^6.2.0", + "yoctocolors-cjs": "^2.1.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/core/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "license": "ISC", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@inquirer/editor": { + "version": "4.2.23", + "resolved": "https://registry.npmjs.org/@inquirer/editor/-/editor-4.2.23.tgz", + "integrity": "sha512-aLSROkEwirotxZ1pBaP8tugXRFCxW94gwrQLxXfrZsKkfjOYC1aRvAZuhpJOb5cu4IBTJdsCigUlf2iCOu4ZDQ==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.3.2", + "@inquirer/external-editor": "^1.0.3", + "@inquirer/type": "^3.0.10" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/expand": { + "version": "4.0.23", + "resolved": "https://registry.npmjs.org/@inquirer/expand/-/expand-4.0.23.tgz", + "integrity": "sha512-nRzdOyFYnpeYTTR2qFwEVmIWypzdAx/sIkCMeTNTcflFOovfqUk+HcFhQQVBftAh9gmGrpFj6QcGEqrDMDOiew==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10", + "yoctocolors-cjs": "^2.1.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/external-editor": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@inquirer/external-editor/-/external-editor-1.0.3.tgz", + "integrity": "sha512-RWbSrDiYmO4LbejWY7ttpxczuwQyZLBUyygsA9Nsv95hpzUWwnNTVQmAq3xuh7vNwCp07UTmE5i11XAEExx4RA==", + "license": "MIT", + "dependencies": { + "chardet": "^2.1.1", + "iconv-lite": "^0.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/figures": { + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/@inquirer/figures/-/figures-1.0.15.tgz", + "integrity": "sha512-t2IEY+unGHOzAaVM5Xx6DEWKeXlDDcNPeDyUpsRc6CUhBfU3VQOEl+Vssh7VNp1dR8MdUJBWhuObjXCsVpjN5g==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@inquirer/input": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@inquirer/input/-/input-4.3.1.tgz", + "integrity": "sha512-kN0pAM4yPrLjJ1XJBjDxyfDduXOuQHrBB8aLDMueuwUGn+vNpF7Gq7TvyVxx8u4SHlFFj4trmj+a2cbpG4Jn1g==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/number": { + "version": "3.0.23", + "resolved": "https://registry.npmjs.org/@inquirer/number/-/number-3.0.23.tgz", + "integrity": "sha512-5Smv0OK7K0KUzUfYUXDXQc9jrf8OHo4ktlEayFlelCjwMXz0299Y8OrI+lj7i4gCBY15UObk76q0QtxjzFcFcg==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/password": { + "version": "4.0.23", + "resolved": "https://registry.npmjs.org/@inquirer/password/-/password-4.0.23.tgz", + "integrity": "sha512-zREJHjhT5vJBMZX/IUbyI9zVtVfOLiTO66MrF/3GFZYZ7T4YILW5MSkEYHceSii/KtRk+4i3RE7E1CUXA2jHcA==", + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^1.0.2", + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/prompts": { + "version": "7.10.1", + "resolved": "https://registry.npmjs.org/@inquirer/prompts/-/prompts-7.10.1.tgz", + "integrity": "sha512-Dx/y9bCQcXLI5ooQ5KyvA4FTgeo2jYj/7plWfV5Ak5wDPKQZgudKez2ixyfz7tKXzcJciTxqLeK7R9HItwiByg==", + "license": "MIT", + "dependencies": { + "@inquirer/checkbox": "^4.3.2", + "@inquirer/confirm": "^5.1.21", + "@inquirer/editor": "^4.2.23", + "@inquirer/expand": "^4.0.23", + "@inquirer/input": "^4.3.1", + "@inquirer/number": "^3.0.23", + "@inquirer/password": "^4.0.23", + "@inquirer/rawlist": "^4.1.11", + "@inquirer/search": "^3.2.2", + "@inquirer/select": "^4.4.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/rawlist": { + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@inquirer/rawlist/-/rawlist-4.1.11.tgz", + "integrity": "sha512-+LLQB8XGr3I5LZN/GuAHo+GpDJegQwuPARLChlMICNdwW7OwV2izlCSCxN6cqpL0sMXmbKbFcItJgdQq5EBXTw==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.3.2", + "@inquirer/type": "^3.0.10", + "yoctocolors-cjs": "^2.1.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/search": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@inquirer/search/-/search-3.2.2.tgz", + "integrity": "sha512-p2bvRfENXCZdWF/U2BXvnSI9h+tuA8iNqtUKb9UWbmLYCRQxd8WkvwWvYn+3NgYaNwdUkHytJMGG4MMLucI1kA==", + "license": "MIT", + "dependencies": { + "@inquirer/core": "^10.3.2", + "@inquirer/figures": "^1.0.15", + "@inquirer/type": "^3.0.10", + "yoctocolors-cjs": "^2.1.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/select": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@inquirer/select/-/select-4.4.2.tgz", + "integrity": "sha512-l4xMuJo55MAe+N7Qr4rX90vypFwCajSakx59qe/tMaC1aEHWLyw68wF4o0A4SLAY4E0nd+Vt+EyskeDIqu1M6w==", + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^1.0.2", + "@inquirer/core": "^10.3.2", + "@inquirer/figures": "^1.0.15", + "@inquirer/type": "^3.0.10", + "yoctocolors-cjs": "^2.1.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@inquirer/type": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@inquirer/type/-/type-3.0.10.tgz", + "integrity": "sha512-BvziSRxfz5Ov8ch0z/n3oijRSEcEsHnhggm4xFZe93DHcUCTlutlq9Ox4SVENAfcRD22UQq7T/atg9Wr3k09eA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/@michaelhomer/jqjs": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@michaelhomer/jqjs/-/jqjs-1.6.0.tgz", + "integrity": "sha512-5dz2TQ3sudW8kPPVmp9hcjPZZVc72UtNAAFGAM4Nvx1Dusv+TeKnqt+JX0HbS7YTOlREKPkG2I0yn+s299Pyyw==", + "license": "MIT" + }, + "node_modules/@noble/ciphers": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-1.3.0.tgz", + "integrity": "sha512-2I0gnIVPtfnMw9ee9h1dJG7tp81+8Ob3OJb3Mv37rx5L40/b0i7djjCVvGOVqc9AEIQyvyu1i6ypKdFw8R8gQw==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/curves": { + "version": "1.9.7", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.9.7.tgz", + "integrity": "sha512-gbKGcRUYIjA3/zCCNaWDciTMFI0dCkvou3TL8Zmy5Nc7sJ47a0jtOeZoTaMxkuqRo9cRhjOdZJXegxYE5FN/xw==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "1.8.0" + }, + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@noble/hashes": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/@noble/hashes/-/hashes-1.8.0.tgz", + "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==", + "license": "MIT", + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@phala/cloud": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@phala/cloud/-/cloud-0.3.0.tgz", + "integrity": "sha512-mZgcu/ImtXpU8czPOugE9SAxBttCIGtdZcqZleolZAV3+amUQfmzdeslIRzFP0E2C+N8+3iTGp4aQvct7E7nRQ==", + "license": "Apache-2.0", + "dependencies": { + "@phala/dstack-sdk": "0.5.8", + "debug": "^4.4.1", + "mitt": "^3.0.1", + "ofetch": "^1.3.3", + "viem": "^2.7.0", + "zod": "^3.24.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "typescript": ">=4.5.0" + } + }, + "node_modules/@phala/dstack-sdk": { + "version": "0.5.8", + "resolved": "https://registry.npmjs.org/@phala/dstack-sdk/-/dstack-sdk-0.5.8.tgz", + "integrity": "sha512-VANlyo0Jj6pxmJl3zksHJ4/5y3VflApVmgsfcfmokk3KYKRBkW5oMkexBTR6lZaGVPOqJ48QGOLP/7CF3IEvFg==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + }, + "peerDependencies": { + "@noble/curves": "^1.8.1", + "@noble/hashes": "^1.6.1", + "@solana/web3.js": "^1.98.4", + "viem": "^2.43.3" + }, + "peerDependenciesMeta": { + "@noble/curves": { + "optional": true + }, + "@solana/web3.js": { + "optional": true + }, + "viem": { + "optional": true + } + } + }, + "node_modules/@scure/base": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/@scure/base/-/base-1.2.6.tgz", + "integrity": "sha512-g/nm5FgUa//MCj1gV09zTJTaM6KBAHqLN907YVQqf7zC49+DcO4B1so4ZX07Ef10Twr6nuqYEH9GEggFXA4Fmg==", + "license": "MIT", + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip32": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@scure/bip32/-/bip32-1.7.0.tgz", + "integrity": "sha512-E4FFX/N3f4B80AKWp5dP6ow+flD1LQZo/w8UnLGYZO674jS6YnYeepycOOksv+vLPSpgN35wgKgy+ybfTb2SMw==", + "license": "MIT", + "dependencies": { + "@noble/curves": "~1.9.0", + "@noble/hashes": "~1.8.0", + "@scure/base": "~1.2.5" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@scure/bip39": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@scure/bip39/-/bip39-1.6.0.tgz", + "integrity": "sha512-+lF0BbLiJNwVlev4eKelw1WWLaiKXw7sSl8T6FvBlWkdX+94aGJ4o8XjUdlyhTCjd8c+B3KT3JfS8P0bLRNU6A==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "~1.8.0", + "@scure/base": "~1.2.5" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/@typescript/typescript-aix-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", + "integrity": "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "aix" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.0.2.tgz", + "integrity": "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-darwin-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.0.2.tgz", + "integrity": "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.0.2.tgz", + "integrity": "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-freebsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.0.2.tgz", + "integrity": "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.0.2.tgz", + "integrity": "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==", + "cpu": [ + "arm" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.0.2.tgz", + "integrity": "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-loong64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.0.2.tgz", + "integrity": "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==", + "cpu": [ + "loong64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-mips64el": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.0.2.tgz", + "integrity": "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==", + "cpu": [ + "mips64el" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-ppc64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.0.2.tgz", + "integrity": "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==", + "cpu": [ + "ppc64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-riscv64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.0.2.tgz", + "integrity": "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==", + "cpu": [ + "riscv64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-s390x": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.0.2.tgz", + "integrity": "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==", + "cpu": [ + "s390x" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-linux-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.0.2.tgz", + "integrity": "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.0.2.tgz", + "integrity": "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-netbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.0.2.tgz", + "integrity": "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "netbsd" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.0.2.tgz", + "integrity": "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-openbsd-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.0.2.tgz", + "integrity": "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "openbsd" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-sunos-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.0.2.tgz", + "integrity": "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "sunos" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-arm64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.0.2.tgz", + "integrity": "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/@typescript/typescript-win32-x64": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.0.2.tgz", + "integrity": "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "peer": true, + "engines": { + "node": ">=16.20.0" + } + }, + "node_modules/abitype": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/abitype/-/abitype-1.2.3.tgz", + "integrity": "sha512-Ofer5QUnuUdTFsBRwARMoWKOH1ND5ehwYhJ3OJ/BQO+StkwQjHw0XyVh4vDttzHB7QOFhPHa/o413PJ82gU/Tg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/wevm" + }, + "peerDependencies": { + "typescript": ">=5.0.4", + "zod": "^3.22.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/ansi-regex": { + "version": "6.4.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.4.0.tgz", + "integrity": "sha512-KzTVk2tCWAHtYrvvvaP8bJKJq2pVinhLcGEQdtLIYPbmNGNyYe8QwNaTUYQp2J7/vIsUKt5QCqAfUkYyG9DkOw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/arg": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", + "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==", + "license": "MIT" + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/bl": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-5.1.0.tgz", + "integrity": "sha512-tv1ZJHLfTDnXE6tMHv73YgSJaWR2AFuPwMntBe7XL/GBFHnT0CLnsHMogfk5+GzCDC5ZWarSCYaIGATZt9dNsQ==", + "license": "MIT", + "dependencies": { + "buffer": "^6.0.3", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/bundle-name": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.1.tgz", + "integrity": "sha512-DdH81/zPLVS11EUgWq3tEu/xn+EzljlMYooDNdzWEnFha3R3NBMpMV1UqYIpjYHV/SgpFKMIX1Oh7o06SjM/oA==", + "license": "MIT", + "dependencies": { + "run-applescript": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/chalk": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.2.0.tgz", + "integrity": "sha512-ree3Gqw/nazQAPuJJEy+avdl7QfZMcUvmHIKgEZkGL+xOBzRvup5Hxo6LHuMceSxOabuJLJm5Yp/92R9eMmMvA==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chardet": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/chardet/-/chardet-2.2.0.tgz", + "integrity": "sha512-rddelWYNPRrXq6PtNEN2S3f6t9ILzvqaN5pVgi4kqt9jHQaXIial9PznB5iSPVlQSLNaaH22ItWz3EJtQ10+OA==", + "license": "MIT" + }, + "node_modules/cli-cursor": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-4.0.0.tgz", + "integrity": "sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg==", + "license": "MIT", + "dependencies": { + "restore-cursor": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-spinners": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", + "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", + "license": "MIT", + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-width": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.1.0.tgz", + "integrity": "sha512-ouuZd4/dm2Sw5Gmqy6bGyNNNe1qt9RpmxveLSO7KcgsTnU7RXfsw+/bukWGo1abgBiMAic068rclZsO4IWmmxQ==", + "license": "ISC", + "engines": { + "node": ">= 12" + } + }, + "node_modules/clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", + "license": "MIT", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/cross-spawn": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", + "license": "MIT", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/dedent": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.2.tgz", + "integrity": "sha512-WzMx3mW98SN+zn3hgemf4OzdmyNhhhKz5Ay0pUfQiMQ3e1g+xmTJWp/pKdwKVXhdSkAEGIIzqeuWrL3mV/AXbA==", + "license": "MIT", + "peerDependencies": { + "babel-plugin-macros": "^3.1.0" + }, + "peerDependenciesMeta": { + "babel-plugin-macros": { + "optional": true + } + } + }, + "node_modules/default-browser": { + "version": "5.5.1", + "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.5.1.tgz", + "integrity": "sha512-m1pAzaJgZ/gssEqlOhJkPJp8Xly7QyW6xcrkUa2KKcDeDSEMP7X8xipU3snUcfisTQx0w1AGae+9UtJSfVnXGw==", + "license": "MIT", + "dependencies": { + "bundle-name": "^4.1.0", + "default-browser-id": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/default-browser-id": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.1.tgz", + "integrity": "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/defaults": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz", + "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==", + "license": "MIT", + "dependencies": { + "clone": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/define-lazy-prop": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", + "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/destr": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/destr/-/destr-2.0.5.tgz", + "integrity": "sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA==", + "license": "MIT" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", + "license": "MIT" + }, + "node_modules/eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==", + "license": "MIT" + }, + "node_modules/execa": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-7.2.0.tgz", + "integrity": "sha512-UduyVP7TLB5IcAQl+OzLyLcS/l32W/GLg+AhHJ+ow40FOk2U3SAllPwR44v4vmdFwIWqpdwxxpQbF1n5ta9seA==", + "license": "MIT", + "dependencies": { + "cross-spawn": "^7.0.3", + "get-stream": "^6.0.1", + "human-signals": "^4.3.0", + "is-stream": "^3.0.0", + "merge-stream": "^2.0.0", + "npm-run-path": "^5.1.0", + "onetime": "^6.0.0", + "signal-exit": "^3.0.7", + "strip-final-newline": "^3.0.0" + }, + "engines": { + "node": "^14.18.0 || ^16.14.0 || >=18.0.0" + }, + "funding": { + "url": "https://github.com/sindresorhus/execa?sponsor=1" + } + }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fastq": { + "version": "1.20.3", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.3.tgz", + "integrity": "sha512-XKv5nnLs6nLF71NgiKJLIZFLkPyIEuOselLG7ujZnGrRfQK8HpvY+WqKhAJUAdLomwVHErVS4LfxFlPq0/FTAw==", + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/fs-extra": { + "version": "11.4.1", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.4.1.tgz", + "integrity": "sha512-KYAb4c9BJQI6QqGKthV68OHe0badztdXJWKo0WtBA9IuCFPTKvE5ZdUBglP833aMjhaSPNO4A5j/EkzZtGlKjA==", + "license": "MIT", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + }, + "engines": { + "node": ">=14.14" + } + }, + "node_modules/get-stream": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", + "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", + "license": "ISC" + }, + "node_modules/handlebars": { + "version": "4.7.10", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.10.tgz", + "integrity": "sha512-P5VJMVM7qgBn6vjXMw8WG9uVI+ncf2pi72j4de4yz5ZULLj2RGqLYaKOYGsgyrViQ0tePOVlN1tDCCXXtFqXKg==", + "license": "MIT", + "dependencies": { + "minimist": "^1.2.8", + "neo-async": "^2.6.2", + "source-map": "^0.6.1", + "wordwrap": "^1.0.0" + }, + "bin": { + "handlebars": "bin/handlebars" + }, + "engines": { + "node": ">=0.4.7" + }, + "optionalDependencies": { + "uglify-js": "^3.1.4" + } + }, + "node_modules/human-signals": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-4.3.1.tgz", + "integrity": "sha512-nZXjEF2nbo7lIw3mgYjItAfgQXog3OjJogSbKa2CQIIvSGWcKgeJnQlNXip6NglNzYH45nSRiEVimMvYL8DDqQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=14.18.0" + } + }, + "node_modules/iconv-lite": { + "version": "0.7.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz", + "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/inquirer": { + "version": "12.11.1", + "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-12.11.1.tgz", + "integrity": "sha512-9VF7mrY+3OmsAfjH3yKz/pLbJ5z22E23hENKw3/LNSaA/sAt3v49bDRY+Ygct1xwuKT+U+cBfTzjCPySna69Qw==", + "license": "MIT", + "dependencies": { + "@inquirer/ansi": "^1.0.2", + "@inquirer/core": "^10.3.2", + "@inquirer/prompts": "^7.10.1", + "@inquirer/type": "^3.0.10", + "mute-stream": "^2.0.0", + "run-async": "^4.0.6", + "rxjs": "^7.8.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/node": ">=18" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + } + } + }, + "node_modules/is-docker": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", + "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-inside-container": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", + "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", + "license": "MIT", + "dependencies": { + "is-docker": "^3.0.0" + }, + "bin": { + "is-inside-container": "cli.js" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-interactive": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-2.0.0.tgz", + "integrity": "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-3.0.0.tgz", + "integrity": "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-unicode-supported": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-1.3.0.tgz", + "integrity": "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-wsl": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.1.tgz", + "integrity": "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==", + "license": "MIT", + "dependencies": { + "is-inside-container": "^1.0.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", + "license": "ISC" + }, + "node_modules/isows": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/isows/-/isows-1.0.7.tgz", + "integrity": "sha512-I1fSfDCZL5P0v33sVqeTDSpcstAg/N+wF5HS033mogOVIp4B+oHC7oOCsA3axAbBSGTJ8QubbNmnIRN/h8U7hg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "license": "MIT", + "peerDependencies": { + "ws": "*" + } + }, + "node_modules/jsonfile": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz", + "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==", + "license": "MIT", + "dependencies": { + "universalify": "^2.0.0" + }, + "optionalDependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/kleur": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", + "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/log-symbols": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-5.1.0.tgz", + "integrity": "sha512-l0x2DvrW294C9uDCoQe1VSU4gf529FkSZ6leBl4TiqZH/e+0R7hSfHQBNut2mNygDgHwvYHfFLn6Oxb3VWj2rA==", + "license": "MIT", + "dependencies": { + "chalk": "^5.0.0", + "is-unicode-supported": "^1.1.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/merge-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", + "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", + "license": "MIT" + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mimic-fn": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", + "integrity": "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/mute-stream": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-2.0.0.tgz", + "integrity": "sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==", + "license": "ISC", + "engines": { + "node": "^18.17.0 || >=20.5.0" + } + }, + "node_modules/neo-async": { + "version": "2.6.2", + "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", + "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", + "license": "MIT" + }, + "node_modules/node-fetch-native": { + "version": "1.6.7", + "resolved": "https://registry.npmjs.org/node-fetch-native/-/node-fetch-native-1.6.7.tgz", + "integrity": "sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==", + "license": "MIT" + }, + "node_modules/npm-run-path": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-5.3.0.tgz", + "integrity": "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ==", + "license": "MIT", + "dependencies": { + "path-key": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/npm-run-path/node_modules/path-key": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-4.0.0.tgz", + "integrity": "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ofetch": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/ofetch/-/ofetch-1.5.1.tgz", + "integrity": "sha512-2W4oUZlVaqAPAil6FUg/difl6YhqhUR7x2eZY4bQCko22UXg3hptq9KLQdqFClV+Wu85UX7hNtdGTngi/1BxcA==", + "license": "MIT", + "dependencies": { + "destr": "^2.0.5", + "node-fetch-native": "^1.6.7", + "ufo": "^1.6.1" + } + }, + "node_modules/onetime": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-6.0.0.tgz", + "integrity": "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/open": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/open/-/open-10.2.0.tgz", + "integrity": "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==", + "license": "MIT", + "dependencies": { + "default-browser": "^5.2.1", + "define-lazy-prop": "^3.0.0", + "is-inside-container": "^1.0.0", + "wsl-utils": "^0.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/ora/-/ora-6.3.1.tgz", + "integrity": "sha512-ERAyNnZOfqM+Ao3RAvIXkYh5joP220yf59gVe2X/cI6SiCxIdi4c9HZKZD8R6q/RDXEje1THBju6iExiSsgJaQ==", + "license": "MIT", + "dependencies": { + "chalk": "^5.0.0", + "cli-cursor": "^4.0.0", + "cli-spinners": "^2.6.1", + "is-interactive": "^2.0.0", + "is-unicode-supported": "^1.1.0", + "log-symbols": "^5.1.0", + "stdin-discarder": "^0.1.0", + "strip-ansi": "^7.0.1", + "wcwidth": "^1.0.1" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ox": { + "version": "0.14.56", + "resolved": "https://registry.npmjs.org/ox/-/ox-0.14.56.tgz", + "integrity": "sha512-Feinlj+deWEam6RIwurRE+eHZ2N3hoUb3AAjQ10BmxRCYIymfqNkv0eZ+4ln4GkgDmSGLD2rGtTMqsFYSVDC6w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "license": "MIT", + "dependencies": { + "@adraffy/ens-normalize": "^1.11.0", + "@noble/ciphers": "^1.3.0", + "@noble/curves": "1.9.1", + "@noble/hashes": "^1.8.0", + "@scure/bip32": "^1.7.0", + "@scure/bip39": "^1.6.0", + "abitype": "^1.2.3", + "eventemitter3": "5.0.1" + }, + "peerDependencies": { + "typescript": ">=5.4.0" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/ox/node_modules/@noble/curves": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.9.1.tgz", + "integrity": "sha512-k11yZxZg+t+gWvBbIswW0yoJlu8cHOC7dhunwOzoWH/mXGBiYyR4YY6hAEK/3EUs4UpB8la1RfdRpeGsFHkWsA==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "1.8.0" + }, + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/phala": { + "version": "1.1.19", + "resolved": "https://registry.npmjs.org/phala/-/phala-1.1.19.tgz", + "integrity": "sha512-vZbRZ7Yl+rXCpUZQJz6HrtxEsVntaM2T69tikN5IbtKnC3ZChgAe5Uj7M9Mg7HPqDSBeW5cKqQ1EnxUP+HgaPQ==", + "license": "Apache-2.0", + "dependencies": { + "@iarna/toml": "^2.2.5", + "@michaelhomer/jqjs": "^1.5.0", + "@noble/curves": "^1.8.1", + "@phala/cloud": "^0.2.9", + "arg": "^5.0.2", + "chalk": "5.2.0", + "dedent": "^1.7.0", + "execa": "^7.2.0", + "fast-glob": "^3.3.3", + "fs-extra": "^11.3.0", + "handlebars": "^4.7.8", + "inquirer": "^12.4.2", + "ofetch": "^1.3.3", + "open": "^10.0.0", + "ora": "^6.3.1", + "prompts": "^2.4.2", + "semver": "^7.6.3", + "viem": "^2.7.0", + "zod": "^3.24.1" + }, + "bin": { + "pha": "dist/index.js", + "phala": "dist/index.js" + } + }, + "node_modules/phala/node_modules/@phala/cloud": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/@phala/cloud/-/cloud-0.2.10.tgz", + "integrity": "sha512-eQXJxbBlJ8xA4e+MmB3AZd9jgdbO3tFh+qu7KL6CS5Ta64LNKlrV3vdke3oUvB22xbc/qqKQ6dIkJx5pTdY7gA==", + "license": "Apache-2.0", + "dependencies": { + "@phala/dstack-sdk": "0.5.8", + "debug": "^4.4.1", + "mitt": "^3.0.1", + "ofetch": "^1.3.3", + "viem": "^2.7.0", + "zod": "^3.24.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "typescript": ">=4.5.0" + } + }, + "node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/prompts": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", + "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", + "license": "MIT", + "dependencies": { + "kleur": "^3.0.3", + "sisteransi": "^1.0.5" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/restore-cursor": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-4.0.0.tgz", + "integrity": "sha512-I9fPXU9geO9bHOt9pHHOhOkYerIMsmVaWB0rA2AI9ERh/+x/i7MV5HKBNrg+ljO5eoPVgCcnFuRjJ9uH6I/3eg==", + "license": "MIT", + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/restore-cursor/node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/restore-cursor/node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/run-applescript": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.1.0.tgz", + "integrity": "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/run-async": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/run-async/-/run-async-4.0.6.tgz", + "integrity": "sha512-IoDlSLTs3Yq593mb3ZoKWKXMNu3UpObxhgA/Xuid5p4bbfi2jdY1Hj0m1K+0/tEuQTxIGMhQDqGjKb7RuxGpAQ==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/rxjs": { + "version": "7.8.2", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", + "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.1.0" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "license": "MIT", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/sisteransi": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", + "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", + "license": "MIT" + }, + "node_modules/source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stdin-discarder": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/stdin-discarder/-/stdin-discarder-0.1.0.tgz", + "integrity": "sha512-xhV7w8S+bUwlPTb4bAOUQhv8/cSS5offJuX8GQGq32ONF0ZtDWKfkdomM3HMRA+LhX6um/FZ0COqlwsjD53LeQ==", + "license": "MIT", + "dependencies": { + "bl": "^5.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-final-newline": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-3.0.0.tgz", + "integrity": "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", + "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", + "license": "Apache-2.0", + "peer": true, + "bin": { + "tsc": "bin/tsc" + }, + "engines": { + "node": ">=16.20.0" + }, + "optionalDependencies": { + "@typescript/typescript-aix-ppc64": "7.0.2", + "@typescript/typescript-darwin-arm64": "7.0.2", + "@typescript/typescript-darwin-x64": "7.0.2", + "@typescript/typescript-freebsd-arm64": "7.0.2", + "@typescript/typescript-freebsd-x64": "7.0.2", + "@typescript/typescript-linux-arm": "7.0.2", + "@typescript/typescript-linux-arm64": "7.0.2", + "@typescript/typescript-linux-loong64": "7.0.2", + "@typescript/typescript-linux-mips64el": "7.0.2", + "@typescript/typescript-linux-ppc64": "7.0.2", + "@typescript/typescript-linux-riscv64": "7.0.2", + "@typescript/typescript-linux-s390x": "7.0.2", + "@typescript/typescript-linux-x64": "7.0.2", + "@typescript/typescript-netbsd-arm64": "7.0.2", + "@typescript/typescript-netbsd-x64": "7.0.2", + "@typescript/typescript-openbsd-arm64": "7.0.2", + "@typescript/typescript-openbsd-x64": "7.0.2", + "@typescript/typescript-sunos-x64": "7.0.2", + "@typescript/typescript-win32-arm64": "7.0.2", + "@typescript/typescript-win32-x64": "7.0.2" + } + }, + "node_modules/ufo": { + "version": "1.6.4", + "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.4.tgz", + "integrity": "sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==", + "license": "MIT" + }, + "node_modules/uglify-js": { + "version": "3.19.3", + "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", + "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", + "license": "BSD-2-Clause", + "optional": true, + "bin": { + "uglifyjs": "bin/uglifyjs" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/universalify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz", + "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==", + "license": "MIT", + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/viem": { + "version": "2.57.5", + "resolved": "https://registry.npmjs.org/viem/-/viem-2.57.5.tgz", + "integrity": "sha512-BaUdS5lJJTxr6hJX17K0AjHbgUhsolsqH8CCbTq1WMunW/I/Siwc9VrTwccCcNPppgqa/gY4TqK8Zl8LLmlTlw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/wevm" + } + ], + "license": "MIT", + "dependencies": { + "@noble/curves": "1.9.1", + "@noble/hashes": "1.8.0", + "@scure/bip32": "1.7.0", + "@scure/bip39": "1.6.0", + "abitype": "1.2.3", + "isows": "1.0.7", + "ox": "0.14.56", + "ws": "8.21.0" + }, + "peerDependencies": { + "typescript": ">=5.0.4" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, + "node_modules/viem/node_modules/@noble/curves": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@noble/curves/-/curves-1.9.1.tgz", + "integrity": "sha512-k11yZxZg+t+gWvBbIswW0yoJlu8cHOC7dhunwOzoWH/mXGBiYyR4YY6hAEK/3EUs4UpB8la1RfdRpeGsFHkWsA==", + "license": "MIT", + "dependencies": { + "@noble/hashes": "1.8.0" + }, + "engines": { + "node": "^14.21.3 || >=16" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/wcwidth": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", + "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", + "license": "MIT", + "dependencies": { + "defaults": "^1.0.3" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "license": "ISC", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/wordwrap": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", + "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", + "license": "MIT" + }, + "node_modules/wrap-ansi": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-6.2.0.tgz", + "integrity": "sha512-r6lPcBGxZXlIcymEu7InxDMhdW0KDxpLgoFLcguasxCaJ/SOIZwINatK9KY/tf+ZrlywOKU0UDj3ATXUBfxJXA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/wsl-utils": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-0.1.0.tgz", + "integrity": "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==", + "license": "MIT", + "dependencies": { + "is-wsl": "^3.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/yoctocolors-cjs": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/yoctocolors-cjs/-/yoctocolors-cjs-2.1.3.tgz", + "integrity": "sha512-U/PBtDf35ff0D8X8D0jfdzHYEPFxAI7jJlxZXwCSez5M3190m+QobIfh+sWDWSHMCWWJN2AWamkegn6vr6YBTw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/.github/scripts/package.json b/.github/scripts/package.json new file mode 100644 index 00000000..c826936f --- /dev/null +++ b/.github/scripts/package.json @@ -0,0 +1,9 @@ +{ + "name": "tinycloud-node-deploy-tools", + "private": true, + "description": "Deploy tooling installed from a lockfile before production secrets are in scope.", + "dependencies": { + "@phala/cloud": "0.3.0", + "phala": "1.1.19" + } +} diff --git a/.github/scripts/phala-sync-allowed-envs.mjs b/.github/scripts/phala-sync-allowed-envs.mjs new file mode 100644 index 00000000..2ec8f668 --- /dev/null +++ b/.github/scripts/phala-sync-allowed-envs.mjs @@ -0,0 +1,107 @@ +import { readFileSync } from "node:fs"; +import { + createClient, + getCvmInfo, + getCvmComposeFile, + updateCvmEnvs, + encryptEnvVars, +} from "@phala/cloud"; + +const apiKey = process.env.PHALA_CLOUD_API_KEY; +const cvmId = process.env.PHALA_CVM_ID; +const envFile = process.env.ENV_FILE; +if (!apiKey || !cvmId || !envFile) { + console.error("PHALA_CLOUD_API_KEY, PHALA_CVM_ID and ENV_FILE are required"); + process.exit(1); +} + +const envs = []; +for (const line of readFileSync(envFile, "utf8").split("\n")) { + if (!line.trim() || line.startsWith("#")) continue; + const i = line.indexOf("="); + if (i <= 0) continue; + envs.push({ key: line.slice(0, i), value: line.slice(i + 1) }); +} +const envKeys = envs.map((e) => e.key); +if (envKeys.length === 0) { + console.error(`No env vars parsed from ${envFile}`); + process.exit(1); +} + +const client = createClient({ apiKey }); +const cvmRef = { id: cvmId }; +const compose = await getCvmComposeFile(client, cvmRef, { schema: false }); +const allowed = new Set(compose?.allowed_envs ?? []); +const nextAllowed = [...new Set([...allowed, ...envKeys])]; +const missing = envKeys.filter((k) => !allowed.has(k)); +if (missing.length === 0) { + console.log(`allowed_envs already contains all ${envKeys.length} deploy env keys — nothing to do.`); + process.exit(0); +} +console.log(`allowed_envs is missing ${missing.length} deploy env key(s): ${missing.join(", ")}`); + +const info = await getCvmInfo(client, cvmRef, { schema: false }); +const pubkey = info?.encrypted_env_pubkey ?? info?.kms_info?.encrypted_env_pubkey; +if (!pubkey) { + console.error("Could not resolve encrypted_env_pubkey from CVM info"); + process.exit(1); +} + +const encryptedEnv = await encryptEnvVars(envs, pubkey); +let result = await updateCvmEnvs( + client, + { ...cvmRef, encrypted_env: encryptedEnv, env_keys: nextAllowed }, + { schema: false }, +); +console.log("phase 1 response:", JSON.stringify(result)); + +if (result?.status === "precondition_required") { + result = await updateCvmEnvs( + client, + { + ...cvmRef, + encrypted_env: encryptedEnv, + env_keys: nextAllowed, + compose_hash: result.compose_hash, + }, + { schema: false }, + ); + console.log("phase 2 response:", JSON.stringify(result)); +} + +if (result?.status !== "in_progress") { + console.error(`Unexpected env update result: ${JSON.stringify(result)}`); + process.exit(1); +} +console.log( + `Env update started (correlation ${result.correlation_id}, allowed_envs_changed=${result.allowed_envs_changed}). Waiting for CVM to settle...`, +); + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +let started = false; +for (let i = 0; i < 120; i++) { + await sleep(5000); + const cur = await getCvmInfo(client, cvmRef, { schema: false }); + const status = cur?.status ?? "unknown"; + if (!started) { + if (status !== "running") started = true; + else if (i >= 12) started = true; + console.log(`[${(i + 1) * 5}s] status=${status} (waiting for update to ${started ? "finish" : "start"})`); + continue; + } + console.log(`[${(i + 1) * 5}s] status=${status}`); + if (status === "running") break; + if (status === "failed" || status === "error") { + console.error(`CVM entered status '${status}' after env update`); + process.exit(1); + } +} + +const after = await getCvmComposeFile(client, cvmRef, { schema: false }); +const afterAllowed = new Set(after?.allowed_envs ?? []); +const stillMissing = nextAllowed.filter((k) => !afterAllowed.has(k)); +if (stillMissing.length > 0) { + console.error(`allowed_envs still missing after update: ${stillMissing.join(", ")}`); + process.exit(1); +} +console.log(`allowed_envs now contains all ${envKeys.length} deploy env keys.`); diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9ccb6d19..fa0f2e13 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -428,6 +428,8 @@ jobs: # required for centralized KMS", which broke the 2026-07-28 deploy. The # last green deploy (run 29953848846, 2026-07-22) ran on 1.1.19. Only # unpin once the prod CVM is migrated or 1.1.20+ is verified against it. + - name: Install Phala SDK helpers + run: npm ci --prefix .github/scripts --ignore-scripts - name: Install Phala CLI run: npm install -g phala@1.1.19 @@ -598,6 +600,7 @@ jobs: PHALA_ENV_CLOUDFLARE_API_TOKEN: ${{ secrets.PROD_TINYCLOUD_CLOUDFLARE_API_TOKEN }} PHALA_ENV_DSTACK_GATEWAY_DOMAIN: ${{ secrets.PROD_TINYCLOUD_DSTACK_GATEWAY_DOMAIN }} PHALA_ENV_CERTBOT_EMAIL: ${{ secrets.PROD_TINYCLOUD_CERTBOT_EMAIL }} + PHALA_ENV_BETTERSTACK_SOURCE_TOKEN: ${{ secrets.PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN }} PHALA_ENV_SQL_IDENTITY_FENCE: ${{ inputs.sql_identity_fence }} # Update the existing prod CVM (name: tinycloud-node) in the # Tiny Cloud workspace. The PHALA_CLOUD_API_KEY secret is @@ -619,6 +622,7 @@ jobs: PHALA_ENV_CLOUDFLARE_API_TOKEN PHALA_ENV_DSTACK_GATEWAY_DOMAIN PHALA_ENV_CERTBOT_EMAIL + PHALA_ENV_BETTERSTACK_SOURCE_TOKEN PHALA_ENV_SQL_IDENTITY_FENCE ) @@ -642,6 +646,7 @@ jobs: printf 'CLOUDFLARE_API_TOKEN=%s\n' "$PHALA_ENV_CLOUDFLARE_API_TOKEN" printf 'DSTACK_GATEWAY_DOMAIN=%s\n' "$PHALA_ENV_DSTACK_GATEWAY_DOMAIN" printf 'CERTBOT_EMAIL=%s\n' "$PHALA_ENV_CERTBOT_EMAIL" + printf 'BETTERSTACK_SOURCE_TOKEN=%s\n' "$PHALA_ENV_BETTERSTACK_SOURCE_TOKEN" printf 'SQL_IDENTITY_FENCE=%s\n' "$PHALA_ENV_SQL_IDENTITY_FENCE" # SSH public keys authorized on the CVM (checked into the repo, # see dstack-authorized-keys). The Phala Cloud pre-launch script @@ -657,6 +662,12 @@ jobs: -f docker-compose.dstack-postgres.yaml config --quiet phala deploy --cvm-id tinycloud-node -c docker-compose.dstack-postgres.yaml -e .phala-prod.env --no-public-logs --no-public-sysinfo --wait + - name: Sync production allowed_envs + env: + PHALA_CLOUD_API_KEY: ${{ secrets.PHALA_CLOUD_API_KEY }} + PHALA_CVM_ID: tinycloud-node + ENV_FILE: .phala-prod.env + run: node .github/scripts/phala-sync-allowed-envs.mjs # TC-312: prove the deploy actually converged. The image tag matching the # crate version is enforced at build time by version-guard, but nothing diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 91069892..88699c6a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -187,6 +187,11 @@ jobs: uses: actions/setup-node@v4 with: node-version: 20 + - name: Check generated log allowlist and compose parity + run: | + node scripts/node-vector-schema.mjs --check + node scripts/node-vector-schema.mjs --check-compose + node --test scripts/node-vector-schema.test.mjs - name: Verify generated capability artifacts match capabilities.json run: node scripts/gen-capabilities.mjs --check - name: Verify deployed Policy/v3 probe contract diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e0e76c1..c1561227 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## [Unreleased] +- TC-959: add a Vector sidecar that forwards bounded JSON node warnings/errors and redacted ingress failures, slow requests, and per-minute successful-request counts to Better Stack. VRL starts from typed allowlisted fields and drops unrecognized or malformed records; the node remains independent of Vector availability. The shipper is pinned and resource-capped; the node peak is pending the reviewed Docker burst check. See `docs/runbooks/tc-959-betterstack-log-shipping.md`. - Security (TC-1132, TC-1080 Parts C–D): webhook registration and every delivery attempt share an egress policy covering special-use IPv4/IPv6, embedded IPv4 (mapped, compatible, NAT64 and 6to4), Teredo and site-local IPv6. Ambiguous local-use NAT64 is refused if any valid IPv4 interpretation is forbidden. Corrected public controls such as `192.2.1.1` and `198.51.0.1` are no longer over-blocked. Mixed DNS answers retain only permitted addresses. - Webhooks deny the resolved addresses of the configured network database, quota URL, space allowlist and explicit S3 endpoint on every port, including aliases and embedded IPv4 forms. The service deny-set refreshes every ten minutes; validation waits for an in-progress refresh, and failed/stale resolution closes egress until recovery. Temporary DNS/policy failures retry on the existing bounded schedule; invalid or forbidden callbacks dead-letter. Existing subscriptions whose destinations become forbidden dead-letter on the next attempt. diff --git a/docker-compose.dstack-postgres.yaml b/docker-compose.dstack-postgres.yaml index 9afa4196..db8c02be 100644 --- a/docker-compose.dstack-postgres.yaml +++ b/docker-compose.dstack-postgres.yaml @@ -12,15 +12,18 @@ services: # ghcr.io/tinycloudlabs/tinycloud-node:1.4.0-dstack-duckdb image: ghcr.io/tinycloudlabs/tinycloud-node:dstack restart: unless-stopped - # TC-970: cap the node's Docker log (stdout: one line per request plus - # application logs) at 3 x 50 MB instead of the guest daemon default - # (current dstack OS images: json-file, 10 x 100 MB). json-file keeps the - # dstack log API (`/logs/`) working. + # TC-959: asynchronously forward redacted JSON records to the local Vector + # receiver. The queue is bounded in events; Docker's daemon remains + # independent of Vector availability. logging: - driver: json-file + driver: fluentd options: - max-size: "50m" - max-file: "3" + fluentd-address: "127.0.0.1:24224" + fluentd-async: "true" + fluentd-buffer-limit: "1000" + tag: "tinycloud.node" + cache-max-size: "20m" + cache-max-file: "5" ports: - "8000:8000" volumes: @@ -34,6 +37,7 @@ services: AWS_ACCESS_KEY_ID: "${AWS_KEY}" AWS_SECRET_ACCESS_KEY: "${AWS_SECRET}" AWS_DEFAULT_REGION: "${AWS_REGION}" + TINYCLOUD_LOGGING__FORMAT: json TINYCLOUD_LOG_LEVEL: normal # Enable Prometheus telemetry (metrics, span histograms, pool gauges). # The metrics handler is intentionally NOT gateway-exposed: port 8001 is @@ -106,15 +110,17 @@ services: - /var/run/dstack.sock:/var/run/dstack.sock - cert-data:/etc/letsencrypt restart: unless-stopped - # TC-970: the ingress logs one JSON line per request to stdout. Cap its - # Docker log at 3 x 50 MB instead of relying on the guest daemon default - # (current dstack OS images ship json-file with 10 x 100 MB). json-file - # keeps the dstack log API (`/logs/`) working. + # TC-959: only sanitized ingress JSON is forwarded; raw access-log paths + # are not forwarded because the ingress emits route templates. logging: - driver: json-file + driver: fluentd options: - max-size: "50m" - max-file: "3" + fluentd-address: "127.0.0.1:24224" + fluentd-async: "true" + fluentd-buffer-limit: "1000" + tag: "tinycloud.ingress" + cache-max-size: "20m" + cache-max-file: "5" # The dstack gateway's TLS-terminating routes (-.:443) # only forward to compose-published ports, so guest sshd on :22 is @@ -129,6 +135,241 @@ services: extra_hosts: - "host.docker.internal:host-gateway" command: TCP-LISTEN:10022,fork,reuseaddr TCP:host.docker.internal:22 + # Only the node app and ingress ship logs. Keep this helper's logs local + # and capped; Vector's own logs use the same exception below. + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" + + vector: + image: timberio/vector:0.59.0-alpine@sha256:0e36bc2a290039e9b6a5e92f04bd2f133223506f72d09682c1df16c015172b7a + restart: unless-stopped + mem_limit: 64m + memswap_limit: 64m + cpus: 0.25 + ports: + - "127.0.0.1:24224:24224" + environment: + BETTERSTACK_SOURCE_TOKEN: "${BETTERSTACK_SOURCE_TOKEN:?BETTERSTACK_SOURCE_TOKEN is required}" + BETTERSTACK_SOURCE_HOST: "${BETTERSTACK_SOURCE_HOST:-s2795437.us-west-2a.betterstackdata.com}" + VECTOR_CONFIG_YAML: | + data_dir: /var/lib/vector + secret: + bs: { type: directory, path: /run/vector-secrets, remove_trailing_whitespace: true } + sources: + docker_fluent: { type: fluent, mode: tcp, address: 0.0.0.0:24224 } + transforms: + allowlist: + type: remap + inputs: [docker_fluent] + drop_on_abort: true + drop_on_error: true + source: | + # BEGIN GENERATED FROM fields.json + if exists(.partial_message) { abort } + if !is_string(.tag) { abort } + tag = .tag + if !is_string(.log) { abort } + j, parse_err = parse_json(.log) + if parse_err != null || !is_object(j) { abort } + out = {} + if tag == "tinycloud.node" { + fields = get(j, ["fields"]) ?? null + if !is_object(fields) { abort } + level = get(j, ["level"]) ?? null + if !is_string(level) { abort } + level = downcase(level) + if !includes(["warn", "error"], level) { abort } + msg = get(fields, ["message"]) ?? null + if !is_string(msg) { abort } + if msg == "invocation replay insert failed" { event = "node_warning"; event_code = "invocation_replay_insert_failed" } + else if msg == "invocation replay cleanup failed" { event = "node_warning"; event_code = "invocation_replay_cleanup_failed" } + else if starts_with(msg, "dstack socket available but get_info failed: ") { event = "node_warning"; event_code = "dstack_get_info_failed" } + else if msg == "share-v2 local-tee feature active: deriving a deterministic, non-attested TEE identity from node key material" { event = "node_warning"; event_code = "share_local_tee_identity" } + else if msg == "retention: delivered hook delivery prune failed" { event = "node_warning"; event_code = "retention_hook_delivery_prune_failed" } + else if msg == "retention: dead-letter hook delivery prune failed" { event = "node_warning"; event_code = "retention_dead_letter_prune_failed" } + else if msg == "retention: signed KV ticket prune failed" { event = "node_warning"; event_code = "retention_signed_ticket_prune_failed" } + else if starts_with(msg, "quota service returned an invalid body: ") { event = "node_warning"; event_code = "quota_invalid_body" } + else if starts_with(msg, "quota service returned ") { event = "node_warning"; event_code = "quota_fallback" } + else if msg == "failed to load tunnel state; tunnel not started" { event = "node_warning"; event_code = "tunnel_state_load_failed" } + else if msg == "invalid loopback API address; tunnel not started" { event = "node_warning"; event_code = "tunnel_address_invalid" } + else if msg == "failed to load link state; LAN listener not started" { event = "node_warning"; event_code = "link_state_load_failed" } + else if msg == "failed to bind link LAN listener; LAN listener not started" { event = "node_warning"; event_code = "link_bind_failed" } + else if msg == "failed to build TLS config; LAN listener not started" { event = "node_warning"; event_code = "link_tls_config_failed" } + else if msg == "link LAN listener exited with error" { event = "node_error"; event_code = "link_listener_error" } + else if msg == "failed to record link listener bind state" { event = "node_warning"; event_code = "link_listener_state_write_failed" } + else if msg == "failed to record link listener bind failure" { event = "node_warning"; event_code = "link_bind_state_write_failed" } + else if msg == "failed to load config for auto-renew" { event = "node_warning"; event_code = "link_auto_renew_config_failed" } + else if msg == "link auto-renew failed" { event = "node_warning"; event_code = "link_auto_renew_failed" } + else if msg == "failed to hot-reload renewed link cert" { event = "node_warning"; event_code = "link_cert_reload_failed" } + else if msg == "failed to reload TLS material after link auto-renew" { event = "node_warning"; event_code = "link_listener_cert_reload_failed" } + else if msg == "link disable: service delete failed; local state removed anyway" { event = "node_warning"; event_code = "link_disable_failed" } + else if msg == "link connection error" { event = "node_warning"; event_code = "link_connection_error" } + else if msg == "link listener accept error" { event = "node_warning"; event_code = "link_listener_accept_failed" } + else if msg == "webhook delivery attempt failed" { event = "node_warning"; event_code = "webhook_delivery_failed" } + else if msg == "webhook dispatcher loop failed" { event = "node_warning"; event_code = "webhook_dispatcher_failed" } + else if msg == "addressed delegation authorization failed" { event = "node_error"; event_code = "share_authorization_failed" } + else if msg == "native share authorization denied" { event = "node_error"; event_code = "share_native_authorization_denied" } + else if msg == "native share KV put failed" { event = "node_error"; event_code = "share_native_kv_put_failed" } + else if msg == "share v2 readiness probe failed" { event = "node_error"; event_code = "share_v2_readiness_failed" } + else if msg == "share v2 is enabled but shareEmail.invitationPublicKey is unset" { event = "node_error"; event_code = "share_invitation_key_missing" } + else if msg == "configured share invitation key does not match the node's derived signing key" { event = "node_error"; event_code = "share_invitation_key_mismatch" } + else if msg == "share/v2 policy challenge denied" { event = "node_warning"; event_code = "share_policy_denied" } + else if starts_with(msg, "Failed to get TDX quote: ") { event = "node_error"; event_code = "attestation_quote_failed" } + else if msg == "failed to read committed hook events" { event = "node_warning"; event_code = "hook_events_read_failed" } + else if msg == "failed to load kv write hook rows" { event = "node_warning"; event_code = "hook_rows_load_failed" } + else if msg == "failed to load kv delete hook rows" { event = "node_warning"; event_code = "hook_rows_load_failed" } + else if msg == "missing committed kv hook row for invocation" { event = "node_warning"; event_code = "missing_committed_hook_row" } + else if msg == "database write committed but webhook enqueue failed" { event = "node_warning"; event_code = "webhook_enqueue_failed" } + else if msg == "tunnel: failed to prepare connection attempt" { event = "node_warning"; event_code = "tunnel_connection_prepare_failed" } + else if msg == "tunnel: no frame within idle timeout; treating connection as dead" { event = "node_warning"; event_code = "tunnel_idle_timeout" } + else if msg == "tunnel: read error" { event = "node_warning"; event_code = "tunnel_read_failed" } + else if msg == "tunnel: dropping malformed frame" { event = "node_warning"; event_code = "tunnel_malformed_frame" } + else if msg == "tunnel: relay sent an error frame" { event = "node_warning"; event_code = "tunnel_relay_error" } + else if msg == "tunnel: failed to write frame" { event = "node_warning"; event_code = "tunnel_write_failed" } + else if msg == "tunnel: failed to record runtime state" { event = "node_warning"; event_code = "tunnel_state_record_failed" } + else if msg == "tunnel: failed to clear runtime state after disable" { event = "node_warning"; event_code = "tunnel_state_clear_failed" } + else if msg == "tunnel authentication rejected" { event = "node_warning"; event_code = "tunnel_auth_rejected" } + else if msg == "node fatal panic" { event = "node_fatal_panic"; event_code = "other"; level = "error" } + else { abort } + if event == "node_warning" && level != "warn" { abort } + if (event == "node_error" || event == "node_fatal_panic") && level != "error" { abort } + out.svc = "api" + out.level = level + out.event = event + out.event_code = event_code + ts = get(j, ["timestamp"]) ?? null + if !is_string(ts) { abort } + parsed_ts, time_err = parse_timestamp(ts, format: "%+") + if time_err != null { abort } + out.ts = format_timestamp!(parsed_ts, format: "%Y-%m-%dT%H:%M:%S%.3fZ") + } else if tag == "tinycloud.ingress" { + out.svc = "ingress" + ts = get(j, ["time"]) ?? null + if !is_string(ts) { abort } + parsed_ts, time_err = parse_timestamp(ts, format: "%+") + if time_err != null { abort } + out.ts = format_timestamp!(parsed_ts, format: "%Y-%m-%dT%H:%M:%S%.3fZ") + route = get(j, ["route"]) ?? null + if !(is_string(route) && includes(["/healthz","/info","/version","/invoke","/delegate","/revoke","/attestation","/signed/kv","/signed/kv/","/admin/quota","/admin/quota/","/admin/usage","/hooks/events","/hooks/tickets","/hooks/webhooks","/hooks/webhooks/","/delegation/query","/delegation/status","/encryption/networks","/encryption/networks/","/encryption/networks//revoke","/encryption/networks//decrypt","/policy/v3/status","/policy/v3/status/","/policy/v3/policies","/policy/v3/challenges","/policy/v3/delegations","/policy/v3/enforcer-bindings","/policy/v3/deliveries/authorize","/.well-known/tinycloud/node-keys","/.well-known/encryption/network/","/public//kv","/public//kv/","/peer/generate/","/evidences/",""], route)) { abort } + out.route = route + method = get(j, ["method"]) ?? null + if !(is_string(method) && includes(["GET","HEAD","POST","PUT","PATCH","DELETE","OPTIONS",""], method)) { abort } + out.method = method + status = get(j, ["status"]) ?? null + if !(is_integer(status) && status >= 100 && status <= 599) { abort } + out.status = status + bytes = get(j, ["bytes"]) ?? null + if !(is_integer(bytes) && bytes >= 0 && bytes <= 1000000000) { abort } + out.bytes_sent = bytes + upstream_status = get(j, ["upstream_status"]) ?? "" + if upstream_status == "-" { upstream_status = "" } + if !(is_string(upstream_status) && length(upstream_status) <= 3 && match(upstream_status, r'^(|[1-5][0-9]{2})$$')) { abort } + out.upstream_status = upstream_status + request_time = get(j, ["request_time"]) ?? null + if !((is_integer(request_time) || is_float(request_time)) && request_time >= 0 && request_time <= 86400) { abort } + out.request_time_seconds = request_time + status_class = if status >= 500 { "5xx" } else if status >= 400 { "4xx" } else if status >= 300 { "3xx" } else if status >= 200 { "2xx" } else { "1xx" } + out.status_class = status_class + if status >= 400 { out.level = if status >= 500 { "error" } else { "warn" }; out.event = "ingress_request_failed" } + else if status >= 200 && status < 300 { out.level = "info"; out.event = "ingress_success_count"; out.request_count = 1 } + else if request_time >= 1 { out.level = "warn"; out.event = "ingress_request_slow" } + else { abort } + } else { abort } + . = out + # END GENERATED FROM fields.json + split_success: + type: route + inputs: [allowlist] + route: + success: .event == "ingress_success_count" + slow_success: .event == "ingress_success_count" && .request_time_seconds >= 1 + other: .event != "ingress_success_count" + slow_success: + type: remap + inputs: [split_success.slow_success] + drop_on_error: true + source: | + . = { + "level": "warn", + "ts": .ts, + "event": "ingress_request_slow", + "svc": "ingress", + "route": .route, + "method": .method, + "status": .status, + "status_class": .status_class, + "request_time_seconds": .request_time_seconds, + "bytes_sent": .bytes_sent, + "upstream_status": .upstream_status + } + success_minute: + type: remap + inputs: [split_success.success] + drop_on_error: true + source: | + .minute = slice!(.ts, 0, 16) + success_reduce: + type: reduce + inputs: [success_minute] + group_by: [route, minute] + expire_after_secs: 75 + flush_period_secs: 60 + merge_strategies: + request_count: sum + success_output: + type: remap + inputs: [success_reduce] + drop_on_error: true + drop_on_abort: true + source: | + route = .route + count = .request_count + if !is_integer(count) || count < 1 || count > 1000000 { abort } + . = { + "level": "info", + "ts": format_timestamp!(now(), format: "%Y-%m-%dT%H:%M:%S%.3fZ"), + "event": "ingress_success_count", + "svc": "ingress", + "route": route, + "request_count": count + } + throttle: + type: throttle + inputs: [success_output, slow_success, split_success.other] + window_secs: 60 + threshold: 300 + sinks: + betterstack: + type: http + inputs: [throttle] + method: post + uri: "https://betterstack-placeholder.invalid/" + encoding: { codec: json } + compression: gzip + auth: { strategy: bearer, token: "SECRET[bs.bs_token]" } + batch: { max_bytes: 4000000, timeout_secs: 5 } + buffer: { type: memory, max_events: 1000, when_full: drop_newest } + entrypoint: + - /bin/sh + - -ec + - | + umask 077 + mkdir -p /run/vector-secrets + case "$$BETTERSTACK_SOURCE_HOST" in + ''|*[!A-Za-z0-9.-]*) printf '%s\n' 'invalid Better Stack host' >&2; exit 1 ;; + esac + printf '%s' "$$BETTERSTACK_SOURCE_TOKEN" > /run/vector-secrets/bs_token + printf '%s' "$$VECTOR_CONFIG_YAML" | sed "s/betterstack-placeholder.invalid/$$BETTERSTACK_SOURCE_HOST/g" > /etc/vector/vector.yaml + unset BETTERSTACK_SOURCE_TOKEN BETTERSTACK_SOURCE_HOST VECTOR_CONFIG_YAML + exec vector --config /etc/vector/vector.yaml + logging: + driver: json-file + options: + max-size: "10m" + max-file: "3" volumes: cert-data: diff --git a/docs/runbooks/tc-959-betterstack-log-shipping.md b/docs/runbooks/tc-959-betterstack-log-shipping.md new file mode 100644 index 00000000..72ddc8f3 --- /dev/null +++ b/docs/runbooks/tc-959-betterstack-log-shipping.md @@ -0,0 +1,33 @@ +# TC-959: tinycloud-node Better Stack log shipping + +## Phase 1 data path + +The node app emits JSON tracing records (`TINYCLOUD_LOGGING__FORMAT=json`). The ingress emits its TC-970 redacted access records. Docker forwards only these two services to the local Vector Fluent source using the asynchronous Fluentd driver. `log-shipper/vector.yaml` is the source config; the generator embeds an escaped copy into Compose and CI checks parity. + +The VRL remap starts from an empty object. Node records require a known fixed message; variable error text is matched only by a fixed prefix and never copied. Unknown messages, non-JSON, non-object JSON, malformed typed fields, and Docker partial records are dropped and increment the Vector remap component's internal discarded-event counters. Those counters are not exposed through an API or metrics sink. Ingress routes, methods, status, duration, response bytes, and upstream status are validated against `log-shipper/fields.json`; only route templates are emitted. Raw paths, query strings, request headers, error text, and arbitrary nested values are not forwarded. Remap failures are dropped. + +Every ingress 4xx/5xx and request taking at least one second is sent as a bounded request event. Every 2xx contributes to a per-minute counter grouped by route template; a slow 2xx also emits its slow-request event. Vector throttles the combined output at 300 events per 60 seconds. The sink retains at most 1,000 events in memory and drops newest when full. There is no Vector API or exposed metrics port; Fluent input is published only on `127.0.0.1:24224`. + +The ingest hostname is configured once as `BETTERSTACK_SOURCE_HOST` in the Vector service environment; the entrypoint validates it and substitutes it into the generated config's placeholder before Vector starts. The source token is the GitHub `production` environment secret `PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN`; deploy writes it into the sealed environment as `BETTERSTACK_SOURCE_TOKEN`, synchronizes `allowed_envs`, and Vector writes it into its directory-secret mount. No token value belongs in this repository or logs. + +At go-live, create a Better Stack alert for `event = node_fatal_panic`. + +## Redaction boundary + +The allowlist excludes signed-KV tickets, authorization headers, tokens, transcript text, SQL values, meeting URLs, IP addresses, raw paths/IDs, owner DIDs or wallet addresses, space IDs, e-mail addresses, nested objects, arrays, and all unknown keys. The panic hook emits only a fixed JSON message and does not serialize panic text or source location. See `log-shipper/vector.tests.yaml` for forbidden-value, malformed-value, unknown-key, route-template, and fixed-panic cases. + +## Resource and volume budget + +The node CVM is tdx.medium: 2 vCPU, 4 GiB RAM, 20 GiB disk. Vector is hard-capped at 64 MiB (1.56% of RAM) and 0.25 CPU (12.5% of 2 vCPU). Measured node Vector memory and CPU peaks are **PENDING** the reviewed Docker burst check; the ptx-batch observation of 16.83 MiB under a 7,000-event burst is a reference only, not a node measurement. + +Each of the two Fluentd producers has a 1,000-event queue. Budgeting with the ptx-batch assumption of a 16 KiB raw line plus 1 KiB event/driver overhead gives `17 KiB × 1,000 × 2 = 34,816,000 bytes` (33.2 MiB). Adding Vector's 64 MiB cap gives 97.2 MiB (2.37% of 4 GiB). Docker does not enforce the assumed 16 KiB raw-line ceiling. The explicit Compose `mem_limit` sum is 64 MiB (Vector only); the node, ingress, and ssh helper currently have no Compose memory caps, so this is not a bounded whole-guest memory total. Do not treat remaining guest memory as guaranteed headroom. + +The supplied raw baseline is about 250,000 ingress lines/day. With 36 route templates, successful requests can emit at most 51,840 per-minute route counters/day (`36 × 1,440`); at the maximum serialized counter size of 172 bytes, that is about 8.9 MB/day before compression. Failed, slow, and application warning/error events add to this. The strict throttle ceiling is 432,000 shipped events/day; the largest bounded ingress event is 280 bytes, so continuous saturation could reach 120.96 MB/day (about 3.63 GB per 30 days). No production traffic sample is available here; the routine daily estimate is therefore **not measured**, and the saturation bound can exceed the free-plan monthly allowance. Check Better Stack ingestion during the Docker/release window before go-live. + +## Local log retention exceptions + +The node app and ingress use the asynchronous Fluentd driver with a 1,000-event limit and `20m × 5` cache settings each. The Vector sidecar and ssh gateway retain capped `json-file` logs (`10m × 3`) and are not sent to themselves; this prevents a feedback loop and keeps helper logs local. Postgres is managed outside this Compose file, so there is no database container log stream to ship. No service depends on Vector for startup or request handling. + +## Verification and deploy gate + +Non-Docker CI checks the canonical common + `services.tinycloud-node` schema, generated VRL, generated Compose parity, and generator tests. The single `/tmp/tc-959-node-docker-check.sh` review script runs the pinned Vector VRL suite, Compose render parity, node-serving checks with Vector stopped/restarted, and a Vector memory burst; it is intentionally not run by this change. Production deployment remains Amos-owned. Compose hash, app-id, KMS attestation, and `allowed_envs` must be checked by the release gate; the node Vector peak remains pending until that window. diff --git a/log-shipper/fields.json b/log-shipper/fields.json new file mode 100644 index 00000000..a1b849f7 --- /dev/null +++ b/log-shipper/fields.json @@ -0,0 +1,92 @@ +{ + "version": 1, + "common": { + "level": { "type": "enum", "values": ["debug", "info", "warn", "error"] }, + "ts": { "type": "string", "regex": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$", "max_len": 24 }, + "event": { "type": "event" } + }, + "reader_output": { + "bucket": { "type": "string", "regex": "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$", "max_len": 19 }, + "status_class": { "type": "enum", "values": ["1xx", "2xx", "3xx", "4xx", "5xx", "unknown"] }, + "count": { "type": "integer", "min": 0, "max": 1000000000 } + }, + "services": { + "ptx-batch": { + "events": [ + "batch_artifact_deletion_failed", + "batch_claim_recovered", + "batch_artifact_reappeared_after_verified_deletion", + "batch_transcription_create", + "batch_transcription_cancel", + "batch_transcription_delete", + "batch_retention_lag_exceeded", + "batch_request_failed", + "batch_admission_mode_changed", + "batch_admission_stopped_by_disk_high_water", + "batch_upload_refused_by_disk_high_water", + "batch_upload_accepted", + "batch_upload_rejected", + "batch_job_finished", + "batch_claim_fenced_out", + "batch_job_failed_unexpectedly", + "batch_vad_fell_back_to_the_floor_threshold", + "batch_vad_could_not_cover_the_recording", + "batch_diarization_failed", + "batch_provider_rejected_the_service_credential_or_model", + "batch_job_failed_at_dispatch", + "batch_worker_heartbeat_failed", + "batch_sweep_failed", + "batch_worker_iteration_failed", + "batch_worker_started" + ], + "fields": { + "svc": { "type": "enum", "values": ["api", "worker"] }, + "correlationId": { "type": "string", "regex": "^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$", "max_len": 36 }, + "transcriptionId": { "type": "string", "regex": "^trn_[0-9A-HJKMNP-TV-Z]{26}$", "max_len": 30 }, + "job_id": { "type": "string", "regex": "^trn_[0-9A-HJKMNP-TV-Z]{26}$", "max_len": 30 }, + "stage": { "type": "enum", "values": ["create", "cancel", "delete", "health", "api_unhandled", "admission", "upload", "deletion", "claim_recovery", "reconcile", "vad", "diarize", "pipeline", "fence", "dispatch", "heartbeat", "sweep", "loop"] }, + "code": { "type": "enum", "values": ["invalid_request", "diarization_unavailable", "unauthorized", "insufficient_scope", "idempotency_conflict", "transcription_not_found", "active_transcription_exists", "recording_too_large", "quota_exceeded", "service_busy", "service_paused", "service_unavailable", "upload_capability_limit", "upload_capability_invalid", "upload_capability_expired", "upload_length_mismatch", "unsupported_media_type", "upload_in_progress", "upload_interrupted", "upload_rejected", "transcript_expired", "not_found", "internal_error", "upload_expired", "upload_integrity_failed", "invalid_audio", "recording_too_long", "unsupported_recording", "no_speech", "provider_unavailable", "provider_outcome_unknown", "processing_timeout", "processing_failed", "transcription_failed", "cancelled"], "nullable": true }, + "alert": { "type": "boolean" }, + "outcome": { "type": "enum", "values": ["completed", "failed", "no_speech"] }, + "duration_seconds": { "type": "number", "min": 0, "max": 86400 }, + "tinfoil_calls": { "type": "integer", "min": 0, "max": 100000, "nullable": true }, + "tinfoil_audio_seconds": { "type": "number", "min": 0, "max": 7200, "nullable": true }, + "voiced_seconds": { "type": "number", "min": 0, "max": 7200, "nullable": true }, + "dropped_above_floor_pct": { "type": "number", "min": 0, "max": 100, "nullable": true }, + "vad_fallback": { "type": "boolean", "nullable": true }, + "segments": { "type": "integer", "min": 0, "max": 100000, "nullable": true }, + "content_type": { "type": "enum", "values": ["audio/mpeg", "audio/wav", "audio/ogg", "audio/mp4", "audio/webm", "audio/flac"] }, + "retentionLagSeconds": { "type": "number", "min": 0, "max": 31536000 }, + "usedPercent": { "type": "number", "min": 0, "max": 100 }, + "mode": { "type": "enum", "values": ["open", "drain", "closed"] }, + "created": { "type": "boolean" } + } + }, + "tinycloud-node": { + "events": [ + "node_fatal_panic", + "node_warning", + "node_error", + "ingress_request_failed", + "ingress_request_slow", + "ingress_success_count" + ], + "fields": { + "svc": { "type": "enum", "values": ["api", "ingress"] }, + "route": { "type": "enum", "values": ["/healthz", "/info", "/version", "/invoke", "/delegate", "/revoke", "/attestation", "/signed/kv", "/signed/kv/", "/admin/quota", "/admin/quota/", "/admin/usage", "/hooks/events", "/hooks/tickets", "/hooks/webhooks", "/hooks/webhooks/", "/delegation/query", "/delegation/status", "/encryption/networks", "/encryption/networks/", "/encryption/networks//revoke", "/encryption/networks//decrypt", "/policy/v3/status", "/policy/v3/status/", "/policy/v3/policies", "/policy/v3/challenges", "/policy/v3/delegations", "/policy/v3/enforcer-bindings", "/policy/v3/deliveries/authorize", "/.well-known/tinycloud/node-keys", "/.well-known/encryption/network/", "/public//kv", "/public//kv/", "/peer/generate/", "/evidences/", ""] }, + "method": { "type": "enum", "values": ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", ""] }, + "status": { "type": "integer", "min": 100, "max": 599 }, + "status_class": { "type": "enum", "values": ["1xx", "2xx", "3xx", "4xx", "5xx"] }, + "request_time_seconds": { "type": "number", "min": 0, "max": 86400 }, + "request_count": { "type": "integer", "min": 1, "max": 1000000 }, + "bytes_sent": { "type": "integer", "min": 0, "max": 1000000000 }, + "upstream_status": { "type": "string", "regex": "^(|[1-5][0-9]{2})$", "max_len": 3 }, + "event_code": { "type": "enum", "values": ["invocation_replay_insert_failed", "invocation_replay_cleanup_failed", "dstack_get_info_failed", "share_local_tee_identity", "retention_hook_delivery_prune_failed", "retention_dead_letter_prune_failed", "retention_signed_ticket_prune_failed", "quota_invalid_body", "quota_fallback", "tunnel_state_load_failed", "tunnel_address_invalid", "link_state_load_failed", "link_bind_invalid", "link_bind_failed", "link_bind_state_write_failed", "link_tls_config_failed", "link_listener_error", "link_listener_state_write_failed", "link_auto_renew_config_failed", "link_auto_renew_failed", "link_cert_reload_failed", "link_listener_cert_reload_failed", "link_disable_failed", "link_connection_error", "link_listener_accept_failed", "webhook_delivery_failed", "webhook_dispatcher_failed", "share_authorization_failed", "share_native_authorization_denied", "share_native_kv_put_failed", "share_v2_readiness_failed", "share_invitation_key_missing", "share_invitation_key_mismatch", "share_policy_denied", "attestation_quote_failed", "hook_events_read_failed", "hook_rows_load_failed", "missing_committed_hook_row", "webhook_enqueue_failed", "tunnel_connection_prepare_failed", "tunnel_idle_timeout", "tunnel_read_failed", "tunnel_malformed_frame", "tunnel_relay_error", "tunnel_write_failed", "tunnel_state_record_failed", "tunnel_state_clear_failed", "tunnel_auth_rejected", "other"] } + } + }, + "tinychat-backend": { "events": [], "fields": {} }, + "tinycloud-mcp": { "events": [], "fields": {} }, + "ptx-dev": { "events": [], "fields": {} }, + "openkey-api": { "events": [], "fields": {} } + } +} diff --git a/log-shipper/vector.tests.yaml b/log-shipper/vector.tests.yaml new file mode 100644 index 00000000..ba810e84 --- /dev/null +++ b/log-shipper/vector.tests.yaml @@ -0,0 +1,341 @@ +tests: + - name: "known node warning is reduced to fixed allowlisted fields" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"invocation replay insert failed","error":"private error"},"target":"private target","unknown":"private"}' + outputs: + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(.event, "node_warning") + assert_eq!(.event_code, "invocation_replay_insert_failed") + assert_eq!(.level, "warn") + assert_eq!(length(keys(.)), 5) + assert!(!contains(encode_json(.), "private")) + - name: "forbidden value probes are removed from a known event" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"invocation replay insert failed","ticket":"signed-kv-ticket-probe","authorization":"Bearer auth-probe","token":"tc_live_probe","transcript":"transcript-probe","sql":"SELECT ''sql-probe''","meeting_url":"https://meet.invalid/probe","ip":"198.51.100.25","path":"/private/owner-42","did":"did:key:did-probe","wallet":"0xwallet-probe","space":"space-probe","email":"owner@example.invalid","nested":{"text":"nested-probe"},"array":["array-probe"]}}' + outputs: + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(length(keys(.)), 5) + assert!(!contains(encode_json(.), "signed-kv-ticket-probe")) + assert!(!contains(encode_json(.), "auth-probe")) + assert!(!contains(encode_json(.), "tc_live_probe")) + assert!(!contains(encode_json(.), "transcript-probe")) + assert!(!contains(encode_json(.), "sql-probe")) + assert!(!contains(encode_json(.), "https://meet.invalid/probe")) + assert!(!contains(encode_json(.), "198.51.100.25")) + assert!(!contains(encode_json(.), "/private/owner-42")) + assert!(!contains(encode_json(.), "did-probe")) + assert!(!contains(encode_json(.), "0xwallet-probe")) + assert!(!contains(encode_json(.), "space-probe")) + assert!(!contains(encode_json(.), "owner@example.invalid")) + assert!(!contains(encode_json(.), "nested-probe")) + assert!(!contains(encode_json(.), "array-probe")) + - name: "random-looking unknown keys cannot extend output" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"invocation replay insert failed"},"x_71a9c2":"random-probe-a","x_4b28ef":{"nested":"random-probe-b"},"x_905de1":["random-probe-c"],"x_aa03f7":true}' + outputs: + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(length(keys(.)), 5) + assert!(!contains(encode_json(.), "random-probe")) + - name: "dynamic tunnel reason is replaced by a fixed event" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"tunnel authentication rejected","close_code":4001,"reason":"did:key:private-probe"}}' + outputs: + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(.event_code, "tunnel_auth_rejected") + assert!(!contains(encode_json(.), "private-probe")) + - name: "fatal panic emits a fixed alertable event without panic text" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"ERROR","fields":{"message":"node fatal panic"},"secret":"panic-probe"}' + outputs: + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(.event, "node_fatal_panic") + assert_eq!(.level, "error") + assert_eq!(.event_code, "other") + assert!(!contains(encode_json(.), "panic-probe")) + - name: "signed-KV ticket and authorization header text is dropped" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"ERROR","fields":{"message":"GET /signed/kv/ticket-probe Authorization: Bearer bearer-probe"}}' + no_outputs_from: [allowlist] + - name: "tokens and API credentials are dropped" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"token tc_live_probe api_key=key-probe"}}' + no_outputs_from: [allowlist] + - name: "transcript text is dropped" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"ERROR","fields":{"message":"Transcript: this private sentence must never ship"}}' + no_outputs_from: [allowlist] + - name: "SQL literal values are dropped" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"ERROR","fields":{"message":"SELECT * FROM users WHERE email=\u0027owner@example.invalid\u0027"}}' + no_outputs_from: [allowlist] + - name: "meeting URL is dropped" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"meeting https://meet.google.com/abc-defg-hij"}}' + no_outputs_from: [allowlist] + - name: "raw IP address in unknown message is dropped" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"client 198.51.100.25"}}' + no_outputs_from: [allowlist] + - name: "raw path with identifiers is dropped" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"ERROR","fields":{"message":"failed at /tmp/owner-42/private.db"}}' + no_outputs_from: [allowlist] + - name: "owner DID and wallet address text is dropped" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"did:pkh:eip155:1:0x1111111111111111111111111111111111111111 0x2222222222222222222222222222222222222222"}}' + no_outputs_from: [allowlist] + - name: "space identifier is dropped" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"ERROR","fields":{"message":"tinycloud:pkh:eip155:1:0x1111111111111111111111111111111111111111:default"}}' + no_outputs_from: [allowlist] + - name: "email address is dropped" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"ERROR","fields":{"message":"owner@example.invalid"}}' + no_outputs_from: [allowlist] + - name: "nested object and array message smuggling aborts" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":{"text":"secret"}},"array":[{"owner":"private"}]}' + no_outputs_from: [allowlist] + - name: "wrongly typed allowed top-level fields abort" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":{"private":"path"},"level":["ERROR"],"fields":{"message":"invocation replay insert failed"}}' + no_outputs_from: [allowlist] + - name: "wrongly typed ingress allowlist values are dropped" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":17,"status":401,"bytes":0,"request_time":0.02,"upstream_status":"401"}' + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":["GET"],"route":"/info","status":200,"bytes":0,"request_time":0.02,"upstream_status":"200"}' + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/info","status":"200","bytes":0,"request_time":0.02,"upstream_status":"200"}' + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/info","status":200,"bytes":"0","request_time":0.02,"upstream_status":"200"}' + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/info","status":200,"bytes":0,"request_time":"0.02","upstream_status":"200"}' + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/info","status":200,"bytes":0,"request_time":0.02,"upstream_status":{"code":"200"}}' + no_outputs_from: [allowlist] + - name: "unknown nested and top-level keys do not expand output" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"invocation replay insert failed","arbitrary":{"text":"private"}},"extra":["unknown"]}' + outputs: + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(length(keys(.)), 5) + assert!(!exists(.extra)) + assert!(!contains(encode_json(.), "private")) + - name: "unknown node message is dropped" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"ERROR","fields":{"message":"unknown request /signed/kv/ticket-probe"}}' + no_outputs_from: [allowlist] + - name: "non-JSON and non-object records are dropped" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: "not-json Authorization Bearer probe" + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: "[]" + no_outputs_from: [allowlist] + - name: "Docker partial records are dropped" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"invocation replay insert failed"}}' + partial_message: true + no_outputs_from: [allowlist] + - name: "ingress 4xx and route template ship without raw request values" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"POST","route":"/signed/kv/","protocol":"HTTP/1.1","status":401,"bytes":0,"request_time":0.02,"upstream_status":"401"}' + outputs: + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(.event, "ingress_request_failed") + assert_eq!(.status, 401) + assert_eq!(.route, "/signed/kv/") + assert!(!contains(encode_json(.), "ticket-probe")) + - name: "ingress raw route, IP, and identifier values are rejected" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/signed/kv/ticket-probe","status":404,"bytes":0,"request_time":0.02,"remote_addr":"198.51.100.8"}' + no_outputs_from: [allowlist] + - name: "ingress success emits bounded counter input by route only" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/info","status":200,"bytes":30,"request_time":0.01,"remote_addr":"198.51.100.8"}' + outputs: + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(.event, "ingress_success_count") + assert_eq!(.request_count, 1) + assert_eq!(.route, "/info") + assert!(!exists(.remote_addr)) + - name: "slow 2xx requests remain counted and emit the slow event" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/info","status":200,"bytes":30,"request_time":1.2,"upstream_status":"200"}' + outputs: + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(.event, "ingress_success_count") + assert_eq!(.request_count, 1) + - extract_from: slow_success + conditions: + - type: vrl + source: | + assert_eq!(.event, "ingress_request_slow") + assert_eq!(.request_time_seconds, 1.2) + assert_eq!(.route, "/info") + - name: "per-minute success output is rebuilt from the route counter" + inputs: + - insert_at: success_reduce + type: log + log_fields: + route: "/info" + request_count: 120 + minute: "2026-10-14T10:00" + outputs: + - extract_from: success_output + conditions: + - type: vrl + source: | + assert_eq!(.event, "ingress_success_count") + assert_eq!(.request_count, 120) + assert_eq!(.route, "/info") + assert_eq!(length(keys(.)), 6) diff --git a/log-shipper/vector.yaml b/log-shipper/vector.yaml new file mode 100644 index 00000000..7a43c55c --- /dev/null +++ b/log-shipper/vector.yaml @@ -0,0 +1,197 @@ +data_dir: /var/lib/vector +secret: + bs: { type: directory, path: /run/vector-secrets, remove_trailing_whitespace: true } +sources: + docker_fluent: { type: fluent, mode: tcp, address: 0.0.0.0:24224 } +transforms: + allowlist: + type: remap + inputs: [docker_fluent] + drop_on_abort: true + drop_on_error: true + source: | + # BEGIN GENERATED FROM fields.json + if exists(.partial_message) { abort } + if !is_string(.tag) { abort } + tag = .tag + if !is_string(.log) { abort } + j, parse_err = parse_json(.log) + if parse_err != null || !is_object(j) { abort } + out = {} + if tag == "tinycloud.node" { + fields = get(j, ["fields"]) ?? null + if !is_object(fields) { abort } + level = get(j, ["level"]) ?? null + if !is_string(level) { abort } + level = downcase(level) + if !includes(["warn", "error"], level) { abort } + msg = get(fields, ["message"]) ?? null + if !is_string(msg) { abort } + if msg == "invocation replay insert failed" { event = "node_warning"; event_code = "invocation_replay_insert_failed" } + else if msg == "invocation replay cleanup failed" { event = "node_warning"; event_code = "invocation_replay_cleanup_failed" } + else if starts_with(msg, "dstack socket available but get_info failed: ") { event = "node_warning"; event_code = "dstack_get_info_failed" } + else if msg == "share-v2 local-tee feature active: deriving a deterministic, non-attested TEE identity from node key material" { event = "node_warning"; event_code = "share_local_tee_identity" } + else if msg == "retention: delivered hook delivery prune failed" { event = "node_warning"; event_code = "retention_hook_delivery_prune_failed" } + else if msg == "retention: dead-letter hook delivery prune failed" { event = "node_warning"; event_code = "retention_dead_letter_prune_failed" } + else if msg == "retention: signed KV ticket prune failed" { event = "node_warning"; event_code = "retention_signed_ticket_prune_failed" } + else if starts_with(msg, "quota service returned an invalid body: ") { event = "node_warning"; event_code = "quota_invalid_body" } + else if starts_with(msg, "quota service returned ") { event = "node_warning"; event_code = "quota_fallback" } + else if msg == "failed to load tunnel state; tunnel not started" { event = "node_warning"; event_code = "tunnel_state_load_failed" } + else if msg == "invalid loopback API address; tunnel not started" { event = "node_warning"; event_code = "tunnel_address_invalid" } + else if msg == "failed to load link state; LAN listener not started" { event = "node_warning"; event_code = "link_state_load_failed" } + else if msg == "failed to bind link LAN listener; LAN listener not started" { event = "node_warning"; event_code = "link_bind_failed" } + else if msg == "failed to build TLS config; LAN listener not started" { event = "node_warning"; event_code = "link_tls_config_failed" } + else if msg == "link LAN listener exited with error" { event = "node_error"; event_code = "link_listener_error" } + else if msg == "failed to record link listener bind state" { event = "node_warning"; event_code = "link_listener_state_write_failed" } + else if msg == "failed to record link listener bind failure" { event = "node_warning"; event_code = "link_bind_state_write_failed" } + else if msg == "failed to load config for auto-renew" { event = "node_warning"; event_code = "link_auto_renew_config_failed" } + else if msg == "link auto-renew failed" { event = "node_warning"; event_code = "link_auto_renew_failed" } + else if msg == "failed to hot-reload renewed link cert" { event = "node_warning"; event_code = "link_cert_reload_failed" } + else if msg == "failed to reload TLS material after link auto-renew" { event = "node_warning"; event_code = "link_listener_cert_reload_failed" } + else if msg == "link disable: service delete failed; local state removed anyway" { event = "node_warning"; event_code = "link_disable_failed" } + else if msg == "link connection error" { event = "node_warning"; event_code = "link_connection_error" } + else if msg == "link listener accept error" { event = "node_warning"; event_code = "link_listener_accept_failed" } + else if msg == "webhook delivery attempt failed" { event = "node_warning"; event_code = "webhook_delivery_failed" } + else if msg == "webhook dispatcher loop failed" { event = "node_warning"; event_code = "webhook_dispatcher_failed" } + else if msg == "addressed delegation authorization failed" { event = "node_error"; event_code = "share_authorization_failed" } + else if msg == "native share authorization denied" { event = "node_error"; event_code = "share_native_authorization_denied" } + else if msg == "native share KV put failed" { event = "node_error"; event_code = "share_native_kv_put_failed" } + else if msg == "share v2 readiness probe failed" { event = "node_error"; event_code = "share_v2_readiness_failed" } + else if msg == "share v2 is enabled but shareEmail.invitationPublicKey is unset" { event = "node_error"; event_code = "share_invitation_key_missing" } + else if msg == "configured share invitation key does not match the node's derived signing key" { event = "node_error"; event_code = "share_invitation_key_mismatch" } + else if msg == "share/v2 policy challenge denied" { event = "node_warning"; event_code = "share_policy_denied" } + else if starts_with(msg, "Failed to get TDX quote: ") { event = "node_error"; event_code = "attestation_quote_failed" } + else if msg == "failed to read committed hook events" { event = "node_warning"; event_code = "hook_events_read_failed" } + else if msg == "failed to load kv write hook rows" { event = "node_warning"; event_code = "hook_rows_load_failed" } + else if msg == "failed to load kv delete hook rows" { event = "node_warning"; event_code = "hook_rows_load_failed" } + else if msg == "missing committed kv hook row for invocation" { event = "node_warning"; event_code = "missing_committed_hook_row" } + else if msg == "database write committed but webhook enqueue failed" { event = "node_warning"; event_code = "webhook_enqueue_failed" } + else if msg == "tunnel: failed to prepare connection attempt" { event = "node_warning"; event_code = "tunnel_connection_prepare_failed" } + else if msg == "tunnel: no frame within idle timeout; treating connection as dead" { event = "node_warning"; event_code = "tunnel_idle_timeout" } + else if msg == "tunnel: read error" { event = "node_warning"; event_code = "tunnel_read_failed" } + else if msg == "tunnel: dropping malformed frame" { event = "node_warning"; event_code = "tunnel_malformed_frame" } + else if msg == "tunnel: relay sent an error frame" { event = "node_warning"; event_code = "tunnel_relay_error" } + else if msg == "tunnel: failed to write frame" { event = "node_warning"; event_code = "tunnel_write_failed" } + else if msg == "tunnel: failed to record runtime state" { event = "node_warning"; event_code = "tunnel_state_record_failed" } + else if msg == "tunnel: failed to clear runtime state after disable" { event = "node_warning"; event_code = "tunnel_state_clear_failed" } + else if msg == "tunnel authentication rejected" { event = "node_warning"; event_code = "tunnel_auth_rejected" } + else if msg == "node fatal panic" { event = "node_fatal_panic"; event_code = "other"; level = "error" } + else { abort } + if event == "node_warning" && level != "warn" { abort } + if (event == "node_error" || event == "node_fatal_panic") && level != "error" { abort } + out.svc = "api" + out.level = level + out.event = event + out.event_code = event_code + ts = get(j, ["timestamp"]) ?? null + if !is_string(ts) { abort } + parsed_ts, time_err = parse_timestamp(ts, format: "%+") + if time_err != null { abort } + out.ts = format_timestamp!(parsed_ts, format: "%Y-%m-%dT%H:%M:%S%.3fZ") + } else if tag == "tinycloud.ingress" { + out.svc = "ingress" + ts = get(j, ["time"]) ?? null + if !is_string(ts) { abort } + parsed_ts, time_err = parse_timestamp(ts, format: "%+") + if time_err != null { abort } + out.ts = format_timestamp!(parsed_ts, format: "%Y-%m-%dT%H:%M:%S%.3fZ") + route = get(j, ["route"]) ?? null + if !(is_string(route) && includes(["/healthz","/info","/version","/invoke","/delegate","/revoke","/attestation","/signed/kv","/signed/kv/","/admin/quota","/admin/quota/","/admin/usage","/hooks/events","/hooks/tickets","/hooks/webhooks","/hooks/webhooks/","/delegation/query","/delegation/status","/encryption/networks","/encryption/networks/","/encryption/networks//revoke","/encryption/networks//decrypt","/policy/v3/status","/policy/v3/status/","/policy/v3/policies","/policy/v3/challenges","/policy/v3/delegations","/policy/v3/enforcer-bindings","/policy/v3/deliveries/authorize","/.well-known/tinycloud/node-keys","/.well-known/encryption/network/","/public//kv","/public//kv/","/peer/generate/","/evidences/",""], route)) { abort } + out.route = route + method = get(j, ["method"]) ?? null + if !(is_string(method) && includes(["GET","HEAD","POST","PUT","PATCH","DELETE","OPTIONS",""], method)) { abort } + out.method = method + status = get(j, ["status"]) ?? null + if !(is_integer(status) && status >= 100 && status <= 599) { abort } + out.status = status + bytes = get(j, ["bytes"]) ?? null + if !(is_integer(bytes) && bytes >= 0 && bytes <= 1000000000) { abort } + out.bytes_sent = bytes + upstream_status = get(j, ["upstream_status"]) ?? "" + if upstream_status == "-" { upstream_status = "" } + if !(is_string(upstream_status) && length(upstream_status) <= 3 && match(upstream_status, r'^(|[1-5][0-9]{2})$')) { abort } + out.upstream_status = upstream_status + request_time = get(j, ["request_time"]) ?? null + if !((is_integer(request_time) || is_float(request_time)) && request_time >= 0 && request_time <= 86400) { abort } + out.request_time_seconds = request_time + status_class = if status >= 500 { "5xx" } else if status >= 400 { "4xx" } else if status >= 300 { "3xx" } else if status >= 200 { "2xx" } else { "1xx" } + out.status_class = status_class + if status >= 400 { out.level = if status >= 500 { "error" } else { "warn" }; out.event = "ingress_request_failed" } + else if status >= 200 && status < 300 { out.level = "info"; out.event = "ingress_success_count"; out.request_count = 1 } + else if request_time >= 1 { out.level = "warn"; out.event = "ingress_request_slow" } + else { abort } + } else { abort } + . = out + # END GENERATED FROM fields.json + split_success: + type: route + inputs: [allowlist] + route: + success: .event == "ingress_success_count" + slow_success: .event == "ingress_success_count" && .request_time_seconds >= 1 + other: .event != "ingress_success_count" + slow_success: + type: remap + inputs: [split_success.slow_success] + drop_on_error: true + source: | + . = { + "level": "warn", + "ts": .ts, + "event": "ingress_request_slow", + "svc": "ingress", + "route": .route, + "method": .method, + "status": .status, + "status_class": .status_class, + "request_time_seconds": .request_time_seconds, + "bytes_sent": .bytes_sent, + "upstream_status": .upstream_status + } + success_minute: + type: remap + inputs: [split_success.success] + drop_on_error: true + source: | + .minute = slice!(.ts, 0, 16) + success_reduce: + type: reduce + inputs: [success_minute] + group_by: [route, minute] + expire_after_secs: 75 + flush_period_secs: 60 + merge_strategies: + request_count: sum + success_output: + type: remap + inputs: [success_reduce] + drop_on_error: true + drop_on_abort: true + source: | + route = .route + count = .request_count + if !is_integer(count) || count < 1 || count > 1000000 { abort } + . = { + "level": "info", + "ts": format_timestamp!(now(), format: "%Y-%m-%dT%H:%M:%S%.3fZ"), + "event": "ingress_success_count", + "svc": "ingress", + "route": route, + "request_count": count + } + throttle: + type: throttle + inputs: [success_output, slow_success, split_success.other] + window_secs: 60 + threshold: 300 +sinks: + betterstack: + type: http + inputs: [throttle] + method: post + uri: "https://betterstack-placeholder.invalid/" + encoding: { codec: json } + compression: gzip + auth: { strategy: bearer, token: "SECRET[bs.bs_token]" } + batch: { max_bytes: 4000000, timeout_secs: 5 } + buffer: { type: memory, max_events: 1000, when_full: drop_newest } diff --git a/scripts/node-vector-schema.mjs b/scripts/node-vector-schema.mjs new file mode 100644 index 00000000..4edd5950 --- /dev/null +++ b/scripts/node-vector-schema.mjs @@ -0,0 +1,221 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +const root = fileURLToPath(new URL("../", import.meta.url)); +const schemaPath = `${root}log-shipper/fields.json`; +const vectorPath = `${root}log-shipper/vector.yaml`; +const beginMarker = "# BEGIN GENERATED FROM fields.json"; +const endMarker = "# END GENERATED FROM fields.json"; + +const sourceMessages = { + "invocation replay insert failed": ["node_warning", "invocation_replay_insert_failed"], + "invocation replay cleanup failed": ["node_warning", "invocation_replay_cleanup_failed"], + "dstack socket available but get_info failed: ": ["node_warning", "dstack_get_info_failed"], + "share-v2 local-tee feature active: deriving a deterministic, non-attested TEE identity from node key material": ["node_warning", "share_local_tee_identity"], + "retention: delivered hook delivery prune failed": ["node_warning", "retention_hook_delivery_prune_failed"], + "retention: dead-letter hook delivery prune failed": ["node_warning", "retention_dead_letter_prune_failed"], + "retention: signed KV ticket prune failed": ["node_warning", "retention_signed_ticket_prune_failed"], + "quota service returned an invalid body: ": ["node_warning", "quota_invalid_body"], + "quota service returned ": ["node_warning", "quota_fallback"], + "failed to load tunnel state; tunnel not started": ["node_warning", "tunnel_state_load_failed"], + "invalid loopback API address; tunnel not started": ["node_warning", "tunnel_address_invalid"], + "failed to load link state; LAN listener not started": ["node_warning", "link_state_load_failed"], + "failed to bind link LAN listener; LAN listener not started": ["node_warning", "link_bind_failed"], + "failed to build TLS config; LAN listener not started": ["node_warning", "link_tls_config_failed"], + "link LAN listener exited with error": ["node_error", "link_listener_error"], + "failed to record link listener bind state": ["node_warning", "link_listener_state_write_failed"], + "failed to record link listener bind failure": ["node_warning", "link_bind_state_write_failed"], + "failed to load config for auto-renew": ["node_warning", "link_auto_renew_config_failed"], + "link auto-renew failed": ["node_warning", "link_auto_renew_failed"], + "failed to hot-reload renewed link cert": ["node_warning", "link_cert_reload_failed"], + "failed to reload TLS material after link auto-renew": ["node_warning", "link_listener_cert_reload_failed"], + "link disable: service delete failed; local state removed anyway": ["node_warning", "link_disable_failed"], + "link connection error": ["node_warning", "link_connection_error"], + "link listener accept error": ["node_warning", "link_listener_accept_failed"], + "webhook delivery attempt failed": ["node_warning", "webhook_delivery_failed"], + "webhook dispatcher loop failed": ["node_warning", "webhook_dispatcher_failed"], + "addressed delegation authorization failed": ["node_error", "share_authorization_failed"], + "native share authorization denied": ["node_error", "share_native_authorization_denied"], + "native share KV put failed": ["node_error", "share_native_kv_put_failed"], + "share v2 readiness probe failed": ["node_error", "share_v2_readiness_failed"], + "share v2 is enabled but shareEmail.invitationPublicKey is unset": ["node_error", "share_invitation_key_missing"], + "configured share invitation key does not match the node's derived signing key": ["node_error", "share_invitation_key_mismatch"], + "share/v2 policy challenge denied": ["node_warning", "share_policy_denied"], + "Failed to get TDX quote: ": ["node_error", "attestation_quote_failed"], + "failed to read committed hook events": ["node_warning", "hook_events_read_failed"], + "failed to load kv write hook rows": ["node_warning", "hook_rows_load_failed"], + "failed to load kv delete hook rows": ["node_warning", "hook_rows_load_failed"], + "missing committed kv hook row for invocation": ["node_warning", "missing_committed_hook_row"], + "database write committed but webhook enqueue failed": ["node_warning", "webhook_enqueue_failed"], + "tunnel: failed to prepare connection attempt": ["node_warning", "tunnel_connection_prepare_failed"], + "tunnel: no frame within idle timeout; treating connection as dead": ["node_warning", "tunnel_idle_timeout"], + "tunnel: read error": ["node_warning", "tunnel_read_failed"], + "tunnel: dropping malformed frame": ["node_warning", "tunnel_malformed_frame"], + "tunnel: relay sent an error frame": ["node_warning", "tunnel_relay_error"], + "tunnel: failed to write frame": ["node_warning", "tunnel_write_failed"], + "tunnel: failed to record runtime state": ["node_warning", "tunnel_state_record_failed"], + "tunnel: failed to clear runtime state after disable": ["node_warning", "tunnel_state_clear_failed"], + "tunnel authentication rejected": ["node_warning", "tunnel_auth_rejected"], +}; + +const quote = (value) => JSON.stringify(value); + +function vrlPredicate(name, spec, accessor) { + if (spec.type === "enum") return `is_string(${accessor}) && includes(${JSON.stringify(spec.values)}, ${accessor})`; + if (spec.type === "integer") return `is_integer(${accessor}) && ${accessor} >= ${spec.min} && ${accessor} <= ${spec.max}`; + if (spec.type === "number") return `(is_integer(${accessor}) || is_float(${accessor})) && ${accessor} >= ${spec.min} && ${accessor} <= ${spec.max}`; + if (spec.type === "boolean") return `is_boolean(${accessor})`; + if (spec.type === "string") return `is_string(${accessor}) && length(${accessor}) <= ${spec.max_len} && match(${accessor}, r'${spec.regex}')`; + throw new Error(`unsupported type for ${name}`); +} +export function renderAllowlist(schema) { + + checkNodeSchema(schema); + const service = schema.services["tinycloud-node"]; + + const lines = [ + beginMarker, + "if exists(.partial_message) { abort }", + "if !is_string(.tag) { abort }", + "tag = .tag", + "if !is_string(.log) { abort }", + "j, parse_err = parse_json(.log)", + "if parse_err != null || !is_object(j) { abort }", + "out = {}", + "if tag == \"tinycloud.node\" {", + " fields = get(j, [\"fields\"]) ?? null", + " if !is_object(fields) { abort }", + " level = get(j, [\"level\"]) ?? null", + " if !is_string(level) { abort }", + " level = downcase(level)", + " if !includes([\"warn\", \"error\"], level) { abort }", + " msg = get(fields, [\"message\"]) ?? null", + " if !is_string(msg) { abort }", + ]; + const messagePairs = Object.entries(sourceMessages); + for (const [index, [message, [event, code]]] of messagePairs.entries()) { + const condition = message.endsWith(": ") || message === "quota service returned " ? `starts_with(msg, ${quote(message)})` : `msg == ${quote(message)}`; + lines.push(` ${index === 0 ? "if" : "else if"} ${condition} { event = ${quote(event)}; event_code = ${quote(code)} }`); + } + lines.push(" else if msg == \"node fatal panic\" { event = \"node_fatal_panic\"; event_code = \"other\"; level = \"error\" }"); + lines.push(" else { abort }"); + lines.push(" if event == \"node_warning\" && level != \"warn\" { abort }"); + lines.push(" if (event == \"node_error\" || event == \"node_fatal_panic\") && level != \"error\" { abort }"); + lines.push(" out.svc = \"api\""); + lines.push(" out.level = level"); + lines.push(" out.event = event"); + lines.push(" out.event_code = event_code"); + lines.push(" ts = get(j, [\"timestamp\"]) ?? null"); + lines.push(" if !is_string(ts) { abort }"); + lines.push(" parsed_ts, time_err = parse_timestamp(ts, format: \"%+\")"); + lines.push(" if time_err != null { abort }"); + lines.push(" out.ts = format_timestamp!(parsed_ts, format: \"%Y-%m-%dT%H:%M:%S%.3fZ\")"); + lines.push("} else if tag == \"tinycloud.ingress\" {"); + lines.push(" out.svc = \"ingress\""); + lines.push(" ts = get(j, [\"time\"]) ?? null"); + lines.push(" if !is_string(ts) { abort }"); + lines.push(" parsed_ts, time_err = parse_timestamp(ts, format: \"%+\")"); + lines.push(" if time_err != null { abort }"); + lines.push(" out.ts = format_timestamp!(parsed_ts, format: \"%Y-%m-%dT%H:%M:%S%.3fZ\")"); + lines.push(" route = get(j, [\"route\"]) ?? null"); + lines.push(` if !(${vrlPredicate("route", service.fields.route, "route")}) { abort }`); + lines.push(" out.route = route"); + lines.push(" method = get(j, [\"method\"]) ?? null"); + lines.push(` if !(${vrlPredicate("method", service.fields.method, "method")}) { abort }`); + lines.push(" out.method = method"); + lines.push(" status = get(j, [\"status\"]) ?? null"); + lines.push(` if !(${vrlPredicate("status", service.fields.status, "status")}) { abort }`); + lines.push(" out.status = status"); + lines.push(" bytes = get(j, [\"bytes\"]) ?? null"); + lines.push(` if !(${vrlPredicate("bytes_sent", service.fields.bytes_sent, "bytes")}) { abort }`); + lines.push(" out.bytes_sent = bytes"); + lines.push(" upstream_status = get(j, [\"upstream_status\"]) ?? \"\""); + lines.push(" if upstream_status == \"-\" { upstream_status = \"\" }"); + lines.push(` if !(${vrlPredicate("upstream_status", service.fields.upstream_status, "upstream_status")}) { abort }`); + lines.push(" out.upstream_status = upstream_status"); + lines.push(" request_time = get(j, [\"request_time\"]) ?? null"); + lines.push(` if !(${vrlPredicate("request_time_seconds", service.fields.request_time_seconds, "request_time")}) { abort }`); + lines.push(" out.request_time_seconds = request_time"); + lines.push(" status_class = if status >= 500 { \"5xx\" } else if status >= 400 { \"4xx\" } else if status >= 300 { \"3xx\" } else if status >= 200 { \"2xx\" } else { \"1xx\" }"); + lines.push(" out.status_class = status_class"); + lines.push(" if status >= 400 { out.level = if status >= 500 { \"error\" } else { \"warn\" }; out.event = \"ingress_request_failed\" }"); + lines.push(" else if status >= 200 && status < 300 { out.level = \"info\"; out.event = \"ingress_success_count\"; out.request_count = 1 }"); + lines.push(" else if request_time >= 1 { out.level = \"warn\"; out.event = \"ingress_request_slow\" }"); + lines.push(" else { abort }"); + lines.push("} else { abort }"); + lines.push(". = out"); + lines.push(endMarker); + return lines.join("\n"); +} + +export function checkVectorFile(schema, yaml = readFileSync(vectorPath, "utf8")) { + const match = yaml.match(/# BEGIN GENERATED FROM fields\.json[\s\S]*?# END GENERATED FROM fields\.json/); + if (!match) throw new Error("vector.yaml is missing generated allowlist markers"); + const actual = match[0].split("\n").map((line) => line.replace(/^ /, "")).join("\n"); + if (actual !== renderAllowlist(schema)) throw new Error("vector.yaml allowlist differs from fields.json-generated VRL"); + for (const name of ["allowlist", "slow_success", "success_minute", "success_output"]) { + const block = yaml.match(new RegExp(`^ ${name}:\\n([\\s\\S]*?)(?=\\n [a-z][a-z0-9_]*:\\n|\\nsinks:)`, "m"))?.[1]; + if (!block?.includes("type: remap") || !block.includes("drop_on_error: true")) throw new Error(`${name} remap must set drop_on_error: true`); + } + const successOutput = yaml.match(/ success_output:\n([\s\S]*?)(?=\n [a-z][a-z0-9_]*:\n|$)/)?.[1]; + if (!successOutput?.includes("drop_on_abort: true") || !successOutput.includes("drop_on_error: true")) throw new Error("success_output must drop aborts and remap errors"); + const count = schema.services["tinycloud-node"].fields.request_count; + const countGuard = `if !is_integer(count) || count < ${count.min} || count > ${count.max} { abort }`; + if (!successOutput.includes(countGuard)) throw new Error("success_output count bounds differ from fields.json"); +} +export function renderComposeVectorConfig(vectorYaml) { + return vectorYaml.trimEnd().split("\n").map((line) => ` ${line.replaceAll("$", () => "$$")}`).join("\n"); +} + +export function checkComposeVectorConfig(compose = readFileSync(`${root}docker-compose.dstack-postgres.yaml`, "utf8"), vectorYaml = readFileSync(vectorPath, "utf8")) { + const start = compose.indexOf(" VECTOR_CONFIG_YAML: |\n"); + const end = compose.indexOf(" entrypoint:", start); + if (start < 0 || end < 0) throw new Error("compose Vector config block not found"); + const actual = compose.slice(start + " VECTOR_CONFIG_YAML: |\n".length, end) + .split("\n") + .filter((line) => line.startsWith(" ")) + .map((line) => line.slice(8).replaceAll("$$", "$")) + .join("\n") + .replace(/\n+$/, ""); + if (actual !== vectorYaml.trimEnd()) throw new Error("compose Vector config differs from log-shipper/vector.yaml"); +} + +function writeComposeVectorConfig() { + const composePath = `${root}docker-compose.dstack-postgres.yaml`; + const compose = readFileSync(composePath, "utf8"); + const start = compose.indexOf(" VECTOR_CONFIG_YAML: |\n"); + const end = compose.indexOf(" entrypoint:", start); + if (start < 0 || end < 0) throw new Error("compose Vector config block not found"); + const block = ` VECTOR_CONFIG_YAML: |\n${renderComposeVectorConfig(readFileSync(vectorPath, "utf8"))}\n`; + writeFileSync(composePath, `${compose.slice(0, start)}${block}${compose.slice(end)}`); +} + +export function checkNodeSchema(schema) { + const service = schema.services?.["tinycloud-node"]; + if (!service) throw new Error("fields.json has no services.tinycloud-node section"); + if (Object.keys(schema.common ?? {}).sort().join(",") !== "event,level,ts") throw new Error("common must contain exactly level, ts and event"); + if (JSON.stringify(schema.common.level) !== JSON.stringify({ type: "enum", values: ["debug", "info", "warn", "error"] })) throw new Error("common.level differs from the emitted level allowlist"); + if (JSON.stringify(schema.common.ts) !== JSON.stringify({ type: "string", regex: "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$", max_len: 24 })) throw new Error("common.ts differs from the emitted timestamp format"); + if (schema.common.event?.type !== "event") throw new Error("common.event must use event type"); + if (service.events.join(",") !== "node_fatal_panic,node_warning,node_error,ingress_request_failed,ingress_request_slow,ingress_success_count") throw new Error("tinycloud-node event set differs from the reviewed set"); + if (Object.values(sourceMessages).some(([, code]) => !new Set(service.fields.event_code?.values ?? []).has(code))) throw new Error("source message map contains an event_code missing from fields.json"); + if (Object.keys(service.fields).sort().join(",") !== [ + "bytes_sent", "event_code", "method", "request_count", "request_time_seconds", "route", "status", "status_class", "svc", "upstream_status", + ].sort().join(",")) throw new Error("tinycloud-node fields differ from the reviewed field set"); + if (service.fields.request_count?.type !== "integer" || service.fields.request_count.min !== 1 || service.fields.request_count.max !== 1000000) throw new Error("request_count must remain bounded to 1..1000000"); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const schema = JSON.parse(readFileSync(schemaPath, "utf8")); + const rendered = renderAllowlist(schema); + if (process.argv.includes("--check")) checkVectorFile(schema); + else if (process.argv.includes("--write")) { + const yaml = readFileSync(vectorPath, "utf8"); + const block = rendered.split("\n").map((line) => ` ${line}`).join("\n"); + const updated = yaml.replace(/ # BEGIN GENERATED FROM fields\.json[\s\S]*? # END GENERATED FROM fields\.json/, () => block); + if (!yaml.includes("# BEGIN GENERATED FROM fields.json") || !yaml.includes("# END GENERATED FROM fields.json")) throw new Error("vector.yaml is missing generated allowlist markers"); + if (updated !== yaml) writeFileSync(vectorPath, updated); + } else if (process.argv.includes("--write-compose")) writeComposeVectorConfig(); + else if (process.argv.includes("--check-compose")) checkComposeVectorConfig(); + else process.stdout.write(`${rendered}\n`); +} diff --git a/scripts/node-vector-schema.test.mjs b/scripts/node-vector-schema.test.mjs new file mode 100644 index 00000000..55ac346b --- /dev/null +++ b/scripts/node-vector-schema.test.mjs @@ -0,0 +1,56 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { checkVectorFile, renderAllowlist, renderComposeVectorConfig } from "./node-vector-schema.mjs"; + +const schema = JSON.parse(readFileSync(new URL("../log-shipper/fields.json", import.meta.url), "utf8")); + +test("generated allowlist emits known fixed messages and builds output from validated fields", () => { + const result = renderAllowlist(schema); + assert.match(result, /msg == "invocation replay insert failed"/); + assert.match(result, /event_code = "invocation_replay_insert_failed"/); + assert.match(result, /out = \{\}/); + assert.doesNotMatch(result, /out\.message|out\.msg|\. = j/); + assert.match(result, /parse_err != null \|\| !is_object\(j\) \{ abort \}/); + assert.match(result, /if exists\(\.partial_message\) \{ abort \}/); +}); + +test("unknown event message and malformed allowed values abort instead of forwarding", () => { + const result = renderAllowlist(schema); + assert.match(result, /else \{ abort \}/); + assert.match(result, /is_integer\(status\) && status >= 100 && status <= 599/); +}); + +test("generator rejects schema drift outside common and tinycloud-node", () => { + assert.throws(() => renderAllowlist({ ...schema, common: { ...schema.common, reader_output: {} } }), /common/); + assert.throws(() => renderAllowlist({ ...schema, services: {} }), /tinycloud-node/); + assert.throws( + () => renderAllowlist({ ...schema, common: { ...schema.common, level: { type: "enum", values: ["warn", "error"] } } }), + /common\.level/, + ); + assert.throws( + () => renderAllowlist({ + ...schema, + services: { + ...schema.services, + "tinycloud-node": { + ...schema.services["tinycloud-node"], + fields: { + ...schema.services["tinycloud-node"].fields, + request_count: { type: "integer", min: 1, max: 1000001 }, + }, + }, + }, + }), + /request_count/, + ); +}); +test("success counter validates its bounded output and abort policy", () => { + assert.doesNotThrow(() => checkVectorFile(schema)); +}); +test("compose embeds source Vector config without allowing Compose to interpolate its regexes", () => { + assert.equal( + renderComposeVectorConfig("sources:\n test: true\nregex: ^[a-z]+$"), + " sources:\n test: true\n regex: ^[a-z]+$$", + ); +}); diff --git a/tinycloud-node-server/src/main.rs b/tinycloud-node-server/src/main.rs index 19471198..c1155ee7 100644 --- a/tinycloud-node-server/src/main.rs +++ b/tinycloud-node-server/src/main.rs @@ -26,26 +26,23 @@ fn build_config_figment() -> rocket::figment::Figment { .merge(Env::prefixed("ROCKET_").global()) } -/// Report panics off the main thread (request handlers and background tasks) -/// by location only. Their messages can format request data, and production -/// logs are public (TC-961). Startup panics on the main thread keep the -/// default report, which operators need to fix a bad configuration. +/// Emit only a fixed panic record: panic messages and source locations can +/// contain request data, while production logs are public (TC-961). +fn panic_log_line() -> String { + let timestamp = time::OffsetDateTime::now_utc() + .format(&time::format_description::well_known::Rfc3339) + .expect("current UTC timestamp is RFC3339-formattable"); + serde_json::json!({ + "timestamp": timestamp, + "level": "ERROR", + "fields": { "message": "node fatal panic" }, + }) + .to_string() +} + fn install_panic_hook() { - let default_hook = std::panic::take_hook(); - std::panic::set_hook(Box::new(move |info| { - let thread = std::thread::current(); - if thread.name() == Some("main") { - default_hook(info); - return; - } - let location = info - .location() - .map(|location| format!("{}:{}", location.file(), location.line())); - eprintln!( - "thread '{}' panicked at {}: ", - thread.name().unwrap_or(""), - location.as_deref().unwrap_or(""), - ); + std::panic::set_hook(Box::new(|_info| { + eprintln!("{}", panic_log_line()); })); } @@ -262,4 +259,17 @@ mod tests { assert_eq!(cfg.share_email.readiness_max_age_seconds, 42); } + #[test] + fn panic_log_record_is_json_and_contains_only_a_fixed_message() { + let line = panic_log_line(); + let record: serde_json::Value = serde_json::from_str(&line).expect("panic record JSON"); + assert!(record["timestamp"].as_str().is_some()); + assert_eq!(record["level"], "ERROR"); + assert_eq!(record["fields"]["message"], "node fatal panic"); + assert_eq!( + record["fields"].as_object().expect("fields object").len(), + 1 + ); + assert_eq!(record.as_object().expect("record object").len(), 3); + } } diff --git a/tinycloud-node-server/src/tunnel/connection.rs b/tinycloud-node-server/src/tunnel/connection.rs index 6a8c4def..697fb2d4 100644 --- a/tinycloud-node-server/src/tunnel/connection.rs +++ b/tinycloud-node-server/src/tunnel/connection.rs @@ -351,7 +351,7 @@ async fn connect_and_serve( ); return AttemptResult::Superseded; } - tracing::warn!(%message); + tracing::warn!(close_code = code, "tunnel authentication rejected"); record_state( &ctx.data_root, TunnelRuntimeState { From 564865c98e256a4b5a89b11bf869220745bb8459 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 08:59:10 +0000 Subject: [PATCH 02/21] feat: harden TC-959 node log shipping --- .github/workflows/docker.yml | 2 +- .github/workflows/rust.yml | 20 ++ README.md | 7 + docker-compose.dstack-postgres.yaml | 156 ++++++++---- .../tc-959-betterstack-log-shipping.md | 15 +- log-shipper/fields.json | 4 +- log-shipper/vector.tests.yaml | 226 +++++++++++++++++- log-shipper/vector.yaml | 144 ++++++++--- scripts/node-vector-schema.mjs | 111 +++++++-- scripts/node-vector-schema.test.mjs | 65 ++++- tinycloud-node-server/src/runtime.rs | 24 ++ 11 files changed, 653 insertions(+), 121 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fa0f2e13..4862b734 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -622,7 +622,6 @@ jobs: PHALA_ENV_CLOUDFLARE_API_TOKEN PHALA_ENV_DSTACK_GATEWAY_DOMAIN PHALA_ENV_CERTBOT_EMAIL - PHALA_ENV_BETTERSTACK_SOURCE_TOKEN PHALA_ENV_SQL_IDENTITY_FENCE ) @@ -662,6 +661,7 @@ jobs: -f docker-compose.dstack-postgres.yaml config --quiet phala deploy --cvm-id tinycloud-node -c docker-compose.dstack-postgres.yaml -e .phala-prod.env --no-public-logs --no-public-sysinfo --wait + # Run only after the compose deploy so a Phala allowed_envs sync failure cannot gate node startup. - name: Sync production allowed_envs env: PHALA_CLOUD_API_KEY: ${{ secrets.PHALA_CLOUD_API_KEY }} diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 88699c6a..132b34a0 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -192,6 +192,26 @@ jobs: node scripts/node-vector-schema.mjs --check node scripts/node-vector-schema.mjs --check-compose node --test scripts/node-vector-schema.test.mjs + - name: Test Vector allowlist (0.59.0, checksum pinned) + run: | + set -eu + version=0.59.0 + archive="vector-${version}-x86_64-unknown-linux-musl.tar.gz" + expected_sha256=a8dbc43c18ae25d0b23a712c9262f3aad904e352b1db5e7db1d9c4aecab59496 + release_url="https://github.com/vectordotdev/vector/releases/download/v${version}" + curl -fsSL "${release_url}/${archive}" -o "${RUNNER_TEMP}/${archive}" + curl -fsSL "${release_url}/vector-${version}-SHA256SUMS" -o "${RUNNER_TEMP}/vector-${version}-SHA256SUMS" + printf '%s %s\n' "${expected_sha256}" "${archive}" | (cd "${RUNNER_TEMP}" && sha256sum --check) + (cd "${RUNNER_TEMP}" && sha256sum --check --ignore-missing "vector-${version}-SHA256SUMS") + mkdir -p "${RUNNER_TEMP}/vector" + tar -xzf "${RUNNER_TEMP}/${archive}" -C "${RUNNER_TEMP}/vector" + vector="${RUNNER_TEMP}/vector/vector-x86_64-unknown-linux-musl/bin/vector" + mkdir -m 700 -p /tmp/vector-secrets + trap 'rm -rf /tmp/vector-secrets' EXIT + printf '%s' 'dummy-vector-test-token' > /tmp/vector-secrets/bs_token + chmod 600 /tmp/vector-secrets/bs_token + "${vector}" validate --no-environment log-shipper/vector.yaml + "${vector}" test log-shipper/vector.yaml log-shipper/vector.tests.yaml - name: Verify generated capability artifacts match capabilities.json run: node scripts/gen-capabilities.mjs --check - name: Verify deployed Policy/v3 probe contract diff --git a/README.md b/README.md index e65dee08..abf3ddad 100644 --- a/README.md +++ b/README.md @@ -294,6 +294,13 @@ When `keys.type` is `Static` the instance will use an array of bytes as a static The secret MUST contain at least 32 bytes of entropy (either randomly generated or derived in a cryptographically secure way). It is STRONGLY RECOMMENDED that the secret be given via environment variables and NOT in the `tinycloud.toml` config file. Additionally it is STRONGLY RECOMMENDED that the secret be backed up in a secure place if used in production. Loss of the secret will result in total loss of function for the TinyCloud Protocol instance. +## Production log shipping + +The production Compose file sends redacted JSON records through Vector to Better Stack. A missing source token leaves only Vector idle; the node and ingress start independently and use asynchronous Fluentd logging. + +Node output is bounded to 280 JSON bytes plus one line separator. The caps are 2 panic/min, 5 app ERROR/min, 5 WARN/min, and 27/min through the shared request lane. WARN records first pass their 5/min cap, then share the request lane with aggregates and slow requests; ERROR and panic remain isolated. The conservative monthly ceiling charges all lane caps separately: `(2 + 5 + 5 + 27) × 60 × 24 × 30 × 281 = 473,428,800 bytes` (about 0.473 GB per 30 days), below the node's 0.48 GB allocation of the shared free-plan quota. Ingress 4xx/5xx counts aggregate by route template and status class; successful requests aggregate by route. There are 36 route values, including `` for unknown paths. +The Docker review check pulls the registry `dstack` image; its Linux/amd64 manifest measured 17,115,015 compressed bytes across seven layers. + ## Running TinyCloud Protocol instances can be started via command line, e.g.: diff --git a/docker-compose.dstack-postgres.yaml b/docker-compose.dstack-postgres.yaml index db8c02be..f9b67030 100644 --- a/docker-compose.dstack-postgres.yaml +++ b/docker-compose.dstack-postgres.yaml @@ -37,7 +37,7 @@ services: AWS_ACCESS_KEY_ID: "${AWS_KEY}" AWS_SECRET_ACCESS_KEY: "${AWS_SECRET}" AWS_DEFAULT_REGION: "${AWS_REGION}" - TINYCLOUD_LOGGING__FORMAT: json + TINYCLOUD_LOG__FORMAT: Json TINYCLOUD_LOG_LEVEL: normal # Enable Prometheus telemetry (metrics, span histograms, pool gauges). # The metrics handler is intentionally NOT gateway-exposed: port 8001 is @@ -152,12 +152,12 @@ services: ports: - "127.0.0.1:24224:24224" environment: - BETTERSTACK_SOURCE_TOKEN: "${BETTERSTACK_SOURCE_TOKEN:?BETTERSTACK_SOURCE_TOKEN is required}" + BETTERSTACK_SOURCE_TOKEN: "${BETTERSTACK_SOURCE_TOKEN:-}" BETTERSTACK_SOURCE_HOST: "${BETTERSTACK_SOURCE_HOST:-s2795437.us-west-2a.betterstackdata.com}" VECTOR_CONFIG_YAML: | data_dir: /var/lib/vector secret: - bs: { type: directory, path: /run/vector-secrets, remove_trailing_whitespace: true } + bs: { type: directory, path: /tmp/vector-secrets, remove_trailing_whitespace: true } sources: docker_fluent: { type: fluent, mode: tcp, address: 0.0.0.0:24224 } transforms: @@ -180,19 +180,21 @@ services: if !is_object(fields) { abort } level = get(j, ["level"]) ?? null if !is_string(level) { abort } - level = downcase(level) + level = downcase!(level) if !includes(["warn", "error"], level) { abort } msg = get(fields, ["message"]) ?? null if !is_string(msg) { abort } + event = null + event_code = null if msg == "invocation replay insert failed" { event = "node_warning"; event_code = "invocation_replay_insert_failed" } else if msg == "invocation replay cleanup failed" { event = "node_warning"; event_code = "invocation_replay_cleanup_failed" } - else if starts_with(msg, "dstack socket available but get_info failed: ") { event = "node_warning"; event_code = "dstack_get_info_failed" } + else if starts_with!(msg, "dstack socket available but get_info failed: ") { event = "node_warning"; event_code = "dstack_get_info_failed" } else if msg == "share-v2 local-tee feature active: deriving a deterministic, non-attested TEE identity from node key material" { event = "node_warning"; event_code = "share_local_tee_identity" } else if msg == "retention: delivered hook delivery prune failed" { event = "node_warning"; event_code = "retention_hook_delivery_prune_failed" } else if msg == "retention: dead-letter hook delivery prune failed" { event = "node_warning"; event_code = "retention_dead_letter_prune_failed" } else if msg == "retention: signed KV ticket prune failed" { event = "node_warning"; event_code = "retention_signed_ticket_prune_failed" } - else if starts_with(msg, "quota service returned an invalid body: ") { event = "node_warning"; event_code = "quota_invalid_body" } - else if starts_with(msg, "quota service returned ") { event = "node_warning"; event_code = "quota_fallback" } + else if starts_with!(msg, "quota service returned an invalid body: ") { event = "node_warning"; event_code = "quota_invalid_body" } + else if starts_with!(msg, "quota service returned ") { event = "node_warning"; event_code = "quota_fallback" } else if msg == "failed to load tunnel state; tunnel not started" { event = "node_warning"; event_code = "tunnel_state_load_failed" } else if msg == "invalid loopback API address; tunnel not started" { event = "node_warning"; event_code = "tunnel_address_invalid" } else if msg == "failed to load link state; LAN listener not started" { event = "node_warning"; event_code = "link_state_load_failed" } @@ -217,7 +219,7 @@ services: else if msg == "share v2 is enabled but shareEmail.invitationPublicKey is unset" { event = "node_error"; event_code = "share_invitation_key_missing" } else if msg == "configured share invitation key does not match the node's derived signing key" { event = "node_error"; event_code = "share_invitation_key_mismatch" } else if msg == "share/v2 policy challenge denied" { event = "node_warning"; event_code = "share_policy_denied" } - else if starts_with(msg, "Failed to get TDX quote: ") { event = "node_error"; event_code = "attestation_quote_failed" } + else if starts_with!(msg, "Failed to get TDX quote: ") { event = "node_error"; event_code = "attestation_quote_failed" } else if msg == "failed to read committed hook events" { event = "node_warning"; event_code = "hook_events_read_failed" } else if msg == "failed to load kv write hook rows" { event = "node_warning"; event_code = "hook_rows_load_failed" } else if msg == "failed to load kv delete hook rows" { event = "node_warning"; event_code = "hook_rows_load_failed" } @@ -232,10 +234,11 @@ services: else if msg == "tunnel: failed to record runtime state" { event = "node_warning"; event_code = "tunnel_state_record_failed" } else if msg == "tunnel: failed to clear runtime state after disable" { event = "node_warning"; event_code = "tunnel_state_clear_failed" } else if msg == "tunnel authentication rejected" { event = "node_warning"; event_code = "tunnel_auth_rejected" } - else if msg == "node fatal panic" { event = "node_fatal_panic"; event_code = "other"; level = "error" } + else if msg == "node fatal panic" { event = "node_fatal_panic"; event_code = "other" } else { abort } if event == "node_warning" && level != "warn" { abort } if (event == "node_error" || event == "node_fatal_panic") && level != "error" { abort } + level = if event == "node_warning" { "warn" } else { "error" } out.svc = "api" out.level = level out.event = event @@ -253,43 +256,77 @@ services: if time_err != null { abort } out.ts = format_timestamp!(parsed_ts, format: "%Y-%m-%dT%H:%M:%S%.3fZ") route = get(j, ["route"]) ?? null - if !(is_string(route) && includes(["/healthz","/info","/version","/invoke","/delegate","/revoke","/attestation","/signed/kv","/signed/kv/","/admin/quota","/admin/quota/","/admin/usage","/hooks/events","/hooks/tickets","/hooks/webhooks","/hooks/webhooks/","/delegation/query","/delegation/status","/encryption/networks","/encryption/networks/","/encryption/networks//revoke","/encryption/networks//decrypt","/policy/v3/status","/policy/v3/status/","/policy/v3/policies","/policy/v3/challenges","/policy/v3/delegations","/policy/v3/enforcer-bindings","/policy/v3/deliveries/authorize","/.well-known/tinycloud/node-keys","/.well-known/encryption/network/","/public//kv","/public//kv/","/peer/generate/","/evidences/",""], route)) { abort } + if !is_string(route) { abort } + route = to_string!(route) + if length(route) > 256 { abort } + if !includes(["/healthz","/info","/version","/invoke","/delegate","/revoke","/attestation","/signed/kv","/signed/kv/","/admin/quota","/admin/quota/","/admin/usage","/hooks/events","/hooks/tickets","/hooks/webhooks","/hooks/webhooks/","/delegation/query","/delegation/status","/encryption/networks","/encryption/networks/","/encryption/networks//revoke","/encryption/networks//decrypt","/policy/v3/status","/policy/v3/status/","/policy/v3/policies","/policy/v3/challenges","/policy/v3/delegations","/policy/v3/enforcer-bindings","/policy/v3/deliveries/authorize","/.well-known/tinycloud/node-keys","/.well-known/encryption/network/","/public//kv","/public//kv/","/peer/generate/","/evidences/"], route) { route = "" } + if !is_string(route) { abort } + route = to_string!(route) + if !includes(["/healthz","/info","/version","/invoke","/delegate","/revoke","/attestation","/signed/kv","/signed/kv/","/admin/quota","/admin/quota/","/admin/usage","/hooks/events","/hooks/tickets","/hooks/webhooks","/hooks/webhooks/","/delegation/query","/delegation/status","/encryption/networks","/encryption/networks/","/encryption/networks//revoke","/encryption/networks//decrypt","/policy/v3/status","/policy/v3/status/","/policy/v3/policies","/policy/v3/challenges","/policy/v3/delegations","/policy/v3/enforcer-bindings","/policy/v3/deliveries/authorize","/.well-known/tinycloud/node-keys","/.well-known/encryption/network/","/public//kv","/public//kv/","/peer/generate/","/evidences/",""], route) { abort } out.route = route method = get(j, ["method"]) ?? null - if !(is_string(method) && includes(["GET","HEAD","POST","PUT","PATCH","DELETE","OPTIONS",""], method)) { abort } + if !is_string(method) { abort } + method = to_string!(method) + if !includes(["GET","HEAD","POST","PUT","PATCH","DELETE","OPTIONS",""], method) { abort } out.method = method status = get(j, ["status"]) ?? null - if !(is_integer(status) && status >= 100 && status <= 599) { abort } + if !is_integer(status) { abort } + status = to_int!(status) + if status < 100 || status > 599 { abort } out.status = status bytes = get(j, ["bytes"]) ?? null - if !(is_integer(bytes) && bytes >= 0 && bytes <= 1000000000) { abort } + if !is_integer(bytes) { abort } + bytes = to_int!(bytes) + if bytes < 0 || bytes > 1000000000 { abort } out.bytes_sent = bytes - upstream_status = get(j, ["upstream_status"]) ?? "" - if upstream_status == "-" { upstream_status = "" } - if !(is_string(upstream_status) && length(upstream_status) <= 3 && match(upstream_status, r'^(|[1-5][0-9]{2})$$')) { abort } + upstream_status_raw = "-" + if exists(j.upstream_status) { upstream_status_raw = j.upstream_status } + if !is_string(upstream_status_raw) { abort } + upstream_status_raw = to_string!(upstream_status_raw) + if length(upstream_status_raw) > 63 { abort } + upstream_status = "" + if upstream_status_raw != "-" && upstream_status_raw != "" { + if !match!(upstream_status_raw, r'^[1-5][0-9]{2}(, ?[1-5][0-9]{2})*$$') { abort } + upstream_status = slice!(upstream_status_raw, 0, 3) + } + if !is_string(upstream_status) { abort } + upstream_status = to_string!(upstream_status) + if length(upstream_status) > 3 { abort } + if !match!(upstream_status, r'^(|[1-5][0-9]{2})$$') { abort } out.upstream_status = upstream_status request_time = get(j, ["request_time"]) ?? null - if !((is_integer(request_time) || is_float(request_time)) && request_time >= 0 && request_time <= 86400) { abort } + if !(is_integer(request_time) || is_float(request_time)) { abort } + request_time = to_float!(request_time) + if request_time < 0 || request_time > 86400 { abort } out.request_time_seconds = request_time status_class = if status >= 500 { "5xx" } else if status >= 400 { "4xx" } else if status >= 300 { "3xx" } else if status >= 200 { "2xx" } else { "1xx" } out.status_class = status_class - if status >= 400 { out.level = if status >= 500 { "error" } else { "warn" }; out.event = "ingress_request_failed" } + if status >= 400 { out.level = if status >= 500 { "error" } else { "warn" }; out.event = "ingress_failure_count"; out.request_count = 1 } else if status >= 200 && status < 300 { out.level = "info"; out.event = "ingress_success_count"; out.request_count = 1 } else if request_time >= 1 { out.level = "warn"; out.event = "ingress_request_slow" } else { abort } } else { abort } . = out + if length(encode_json(.)) > 280 { abort } # END GENERATED FROM fields.json - split_success: + classify_by_budget: type: route inputs: [allowlist] route: - success: .event == "ingress_success_count" - slow_success: .event == "ingress_success_count" && .request_time_seconds >= 1 - other: .event != "ingress_success_count" + panic: .event == "node_fatal_panic" + app_error: .event == "node_error" + app_warning: .event == "node_warning" + ingress_summary: .event == "ingress_failure_count" || .event == "ingress_success_count" + ingress_slow: .event == "ingress_request_slow" + split_requests: + type: route + inputs: [classify_by_budget.ingress_summary] + route: + aggregate: .event == "ingress_failure_count" || .event == "ingress_success_count" + slow_success: .event == "ingress_success_count" && (to_float(.request_time_seconds) ?? 0.0) >= 1.0 slow_success: type: remap - inputs: [split_success.slow_success] + inputs: [split_requests.slow_success] drop_on_error: true source: | . = { @@ -305,46 +342,79 @@ services: "bytes_sent": .bytes_sent, "upstream_status": .upstream_status } - success_minute: + request_minute: type: remap - inputs: [split_success.success] + inputs: [split_requests.aggregate] drop_on_error: true source: | .minute = slice!(.ts, 0, 16) - success_reduce: + request_reduce: type: reduce - inputs: [success_minute] - group_by: [route, minute] - expire_after_secs: 75 - flush_period_secs: 60 + inputs: [request_minute] + group_by: [route, status_class, event, minute] + expire_after_ms: 75000 + end_every_period_ms: 60000 + flush_period_ms: 1000 merge_strategies: request_count: sum - success_output: + request_output: type: remap - inputs: [success_reduce] + inputs: [request_reduce] drop_on_error: true drop_on_abort: true source: | route = .route + status_class = .status_class + event = .event count = .request_count - if !is_integer(count) || count < 1 || count > 1000000 { abort } + if !is_integer(count) { abort } + count = to_int!(count) + if count < 1 || count > 1000000 { abort } + level = null + if event == "ingress_success_count" && status_class == "2xx" { level = "info" } + else if event == "ingress_failure_count" && includes(["4xx", "5xx"], status_class) { + level = if status_class == "5xx" { "error" } else { "warn" } + } else { abort } . = { - "level": "info", + "level": level, "ts": format_timestamp!(now(), format: "%Y-%m-%dT%H:%M:%S%.3fZ"), - "event": "ingress_success_count", + "event": event, "svc": "ingress", "route": route, + "status_class": status_class, "request_count": count } - throttle: + throttle_panic: + type: throttle + inputs: [classify_by_budget.panic] + window_secs: 60 + threshold: 2 + throttle_app_error: + type: throttle + inputs: [classify_by_budget.app_error] + window_secs: 60 + threshold: 5 + throttle_app_warning: type: throttle - inputs: [success_output, slow_success, split_success.other] + inputs: [classify_by_budget.app_warning] window_secs: 60 - threshold: 300 + threshold: 5 + throttle_ingress: + type: throttle + inputs: [request_output, classify_by_budget.ingress_slow, slow_success, throttle_app_warning] + window_secs: 60 + threshold: 27 + budget_size_guard: + type: remap + inputs: [throttle_panic, throttle_app_error, throttle_ingress] + drop_on_error: true + drop_on_abort: true + source: | + if length(encode_json(.)) > 280 { abort } sinks: betterstack: type: http - inputs: [throttle] + inputs: [budget_size_guard] method: post uri: "https://betterstack-placeholder.invalid/" encoding: { codec: json } @@ -357,11 +427,15 @@ services: - -ec - | umask 077 - mkdir -p /run/vector-secrets + if [ -z "$$BETTERSTACK_SOURCE_TOKEN" ]; then + printf '%s\n' 'Better Stack token is absent; Vector is idle' + while :; do sleep 3600; done + fi + mkdir -p /tmp/vector-secrets case "$$BETTERSTACK_SOURCE_HOST" in ''|*[!A-Za-z0-9.-]*) printf '%s\n' 'invalid Better Stack host' >&2; exit 1 ;; esac - printf '%s' "$$BETTERSTACK_SOURCE_TOKEN" > /run/vector-secrets/bs_token + printf '%s' "$$BETTERSTACK_SOURCE_TOKEN" > /tmp/vector-secrets/bs_token printf '%s' "$$VECTOR_CONFIG_YAML" | sed "s/betterstack-placeholder.invalid/$$BETTERSTACK_SOURCE_HOST/g" > /etc/vector/vector.yaml unset BETTERSTACK_SOURCE_TOKEN BETTERSTACK_SOURCE_HOST VECTOR_CONFIG_YAML exec vector --config /etc/vector/vector.yaml diff --git a/docs/runbooks/tc-959-betterstack-log-shipping.md b/docs/runbooks/tc-959-betterstack-log-shipping.md index 72ddc8f3..3ae5412b 100644 --- a/docs/runbooks/tc-959-betterstack-log-shipping.md +++ b/docs/runbooks/tc-959-betterstack-log-shipping.md @@ -2,27 +2,30 @@ ## Phase 1 data path -The node app emits JSON tracing records (`TINYCLOUD_LOGGING__FORMAT=json`). The ingress emits its TC-970 redacted access records. Docker forwards only these two services to the local Vector Fluent source using the asynchronous Fluentd driver. `log-shipper/vector.yaml` is the source config; the generator embeds an escaped copy into Compose and CI checks parity. +The node app emits JSON tracing records (`TINYCLOUD_LOG__FORMAT=Json`). The ingress emits its TC-970 redacted access records. Docker forwards only these two services to the local Vector Fluent source using the asynchronous Fluentd driver. `log-shipper/vector.yaml` is the source config; the generator embeds an escaped copy into Compose and CI checks parity. The VRL remap starts from an empty object. Node records require a known fixed message; variable error text is matched only by a fixed prefix and never copied. Unknown messages, non-JSON, non-object JSON, malformed typed fields, and Docker partial records are dropped and increment the Vector remap component's internal discarded-event counters. Those counters are not exposed through an API or metrics sink. Ingress routes, methods, status, duration, response bytes, and upstream status are validated against `log-shipper/fields.json`; only route templates are emitted. Raw paths, query strings, request headers, error text, and arbitrary nested values are not forwarded. Remap failures are dropped. -Every ingress 4xx/5xx and request taking at least one second is sent as a bounded request event. Every 2xx contributes to a per-minute counter grouped by route template; a slow 2xx also emits its slow-request event. Vector throttles the combined output at 300 events per 60 seconds. The sink retains at most 1,000 events in memory and drops newest when full. There is no Vector API or exposed metrics port; Fluent input is published only on `127.0.0.1:24224`. +Ingress failures are counted per minute by `(route, status_class)`, successful requests by route, and slow requests remain bounded warning events; a slow 2xx is both counted and marked slow. Panic has a dedicated 2/min lane and node ERROR has a dedicated 5/min lane. Node WARN is capped at 5/min and then shares the request lane with ingress aggregates and slow requests, capped at 27/min. Request traffic and WARN cannot consume the panic or ERROR lanes. The sink retains at most 1,000 events in memory and drops newest when full. There is no Vector API or exposed metrics port; Fluent input is published only on `127.0.0.1:24224`. -The ingest hostname is configured once as `BETTERSTACK_SOURCE_HOST` in the Vector service environment; the entrypoint validates it and substitutes it into the generated config's placeholder before Vector starts. The source token is the GitHub `production` environment secret `PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN`; deploy writes it into the sealed environment as `BETTERSTACK_SOURCE_TOKEN`, synchronizes `allowed_envs`, and Vector writes it into its directory-secret mount. No token value belongs in this repository or logs. +The ingest hostname is configured once as `BETTERSTACK_SOURCE_HOST` in the Vector service environment; the entrypoint validates it and substitutes it into the generated config's placeholder before Vector starts. The source token is the GitHub `production` environment secret `PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN`; deploy writes it into the sealed environment as `BETTERSTACK_SOURCE_TOKEN` and synchronizes `allowed_envs` after compose startup. An absent token leaves only Vector idle; the node and ingress have no dependency on it. Docker container environment metadata is visible to principals with `docker inspect` access, so restrict CVM Docker access accordingly. No token value belongs in this repository or logs. At go-live, create a Better Stack alert for `event = node_fatal_panic`. ## Redaction boundary -The allowlist excludes signed-KV tickets, authorization headers, tokens, transcript text, SQL values, meeting URLs, IP addresses, raw paths/IDs, owner DIDs or wallet addresses, space IDs, e-mail addresses, nested objects, arrays, and all unknown keys. The panic hook emits only a fixed JSON message and does not serialize panic text or source location. See `log-shipper/vector.tests.yaml` for forbidden-value, malformed-value, unknown-key, route-template, and fixed-panic cases. +The allowlist excludes signed-KV tickets, authorization headers, tokens, transcript text, SQL values, meeting URLs, X-Forwarded-For and CF-Connecting-IP values, IPv4/IPv6 addresses, raw paths/IDs, UCAN/SIWE/delegation strings, owner DIDs or wallet addresses, space IDs, e-mail addresses, nested objects, arrays, and all unknown keys. Unknown ingress paths collapse to `` only after the raw route string passes its 256-character bound. The panic hook emits only a fixed JSON message and does not serialize panic text or source location; a WARN cannot forge that alert. See `log-shipper/vector.tests.yaml` for forbidden-value, malformed-value, priority-lane flood, route-template, multi-upstream, and fixed-panic cases. ## Resource and volume budget The node CVM is tdx.medium: 2 vCPU, 4 GiB RAM, 20 GiB disk. Vector is hard-capped at 64 MiB (1.56% of RAM) and 0.25 CPU (12.5% of 2 vCPU). Measured node Vector memory and CPU peaks are **PENDING** the reviewed Docker burst check; the ptx-batch observation of 16.83 MiB under a 7,000-event burst is a reference only, not a node measurement. +The Linux/amd64 `ghcr.io/tinycloudlabs/tinycloud-node:dstack` registry manifest measured 17,115,015 compressed layer bytes across seven layers. The Docker check uses that registry image; no local build is needed for the small pull. -Each of the two Fluentd producers has a 1,000-event queue. Budgeting with the ptx-batch assumption of a 16 KiB raw line plus 1 KiB event/driver overhead gives `17 KiB × 1,000 × 2 = 34,816,000 bytes` (33.2 MiB). Adding Vector's 64 MiB cap gives 97.2 MiB (2.37% of 4 GiB). Docker does not enforce the assumed 16 KiB raw-line ceiling. The explicit Compose `mem_limit` sum is 64 MiB (Vector only); the node, ingress, and ssh helper currently have no Compose memory caps, so this is not a bounded whole-guest memory total. Do not treat remaining guest memory as guaranteed headroom. +The conservative monthly calculation charges each cap independently: `2 + 5 + 5 + 27 = 39 events/minute`. Over 30 days: `39 × 60 × 24 × 30 × 281 = 473,428,800 bytes` (about 0.473 GB, 452 MiB), below the node's 0.48 GB allocation of the shared 3 GB free quota. Per-cap arithmetic: panic `2 × 60 × 24 × 30 × 281 = 24,278,400 bytes`; app ERROR and WARN each `5 × 60 × 24 × 30 × 281 = 60,696,000 bytes`; shared request lane `27 × 60 × 24 × 30 × 281 = 327,758,400 bytes`. WARN is counted in both its own 5/min cap and the 27/min request cap in this conservative estimate. -The supplied raw baseline is about 250,000 ingress lines/day. With 36 route templates, successful requests can emit at most 51,840 per-minute route counters/day (`36 × 1,440`); at the maximum serialized counter size of 172 bytes, that is about 8.9 MB/day before compression. Failed, slow, and application warning/error events add to this. The strict throttle ceiling is 432,000 shipped events/day; the largest bounded ingress event is 280 bytes, so continuous saturation could reach 120.96 MB/day (about 3.63 GB per 30 days). No production traffic sample is available here; the routine daily estimate is therefore **not measured**, and the saturation bound can exceed the free-plan monthly allowance. Check Better Stack ingestion during the Docker/release window before go-live. +The successful-counter route enum has 36 members (35 known templates plus ``). A route string longer than 256 characters is dropped before normalization; other unknown paths map to ``. Failure counts aggregate by route and status class. The transformed event-size guard and per-lane throttles cap emitted bytes; excess events are dropped in their own lane rather than borrowing another lane's quota. These limits bound node output, not other services' shared quota usage. + +Each of the two Fluentd producers has a 1,000-event queue. The planning assumption of a 16 KiB raw line plus 1 KiB event/driver overhead gives `17 KiB × 1,000 × 2 = 34,816,000 bytes` (33.2 MiB). Adding Vector's 64 MiB cap gives 97.2 MiB (2.37% of 4 GiB). Docker does not enforce the assumed 16 KiB raw-line ceiling, so this is not a guaranteed producer-queue byte bound. The explicit Compose `mem_limit` sum is 64 MiB (Vector only); the node, ingress, and ssh helper have no Compose memory caps, so this is not a bounded whole-guest memory total. Do not treat remaining guest memory as guaranteed headroom. ## Local log retention exceptions diff --git a/log-shipper/fields.json b/log-shipper/fields.json index a1b849f7..94759583 100644 --- a/log-shipper/fields.json +++ b/log-shipper/fields.json @@ -67,13 +67,13 @@ "node_fatal_panic", "node_warning", "node_error", - "ingress_request_failed", + "ingress_failure_count", "ingress_request_slow", "ingress_success_count" ], "fields": { "svc": { "type": "enum", "values": ["api", "ingress"] }, - "route": { "type": "enum", "values": ["/healthz", "/info", "/version", "/invoke", "/delegate", "/revoke", "/attestation", "/signed/kv", "/signed/kv/", "/admin/quota", "/admin/quota/", "/admin/usage", "/hooks/events", "/hooks/tickets", "/hooks/webhooks", "/hooks/webhooks/", "/delegation/query", "/delegation/status", "/encryption/networks", "/encryption/networks/", "/encryption/networks//revoke", "/encryption/networks//decrypt", "/policy/v3/status", "/policy/v3/status/", "/policy/v3/policies", "/policy/v3/challenges", "/policy/v3/delegations", "/policy/v3/enforcer-bindings", "/policy/v3/deliveries/authorize", "/.well-known/tinycloud/node-keys", "/.well-known/encryption/network/", "/public//kv", "/public//kv/", "/peer/generate/", "/evidences/", ""] }, + "route": { "type": "enum", "values": ["/healthz", "/info", "/version", "/invoke", "/delegate", "/revoke", "/attestation", "/signed/kv", "/signed/kv/", "/admin/quota", "/admin/quota/", "/admin/usage", "/hooks/events", "/hooks/tickets", "/hooks/webhooks", "/hooks/webhooks/", "/delegation/query", "/delegation/status", "/encryption/networks", "/encryption/networks/", "/encryption/networks//revoke", "/encryption/networks//decrypt", "/policy/v3/status", "/policy/v3/status/", "/policy/v3/policies", "/policy/v3/challenges", "/policy/v3/delegations", "/policy/v3/enforcer-bindings", "/policy/v3/deliveries/authorize", "/.well-known/tinycloud/node-keys", "/.well-known/encryption/network/", "/public//kv", "/public//kv/", "/peer/generate/", "/evidences/", ""] }, "method": { "type": "enum", "values": ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", ""] }, "status": { "type": "integer", "min": 100, "max": 599 }, "status_class": { "type": "enum", "values": ["1xx", "2xx", "3xx", "4xx", "5xx"] }, diff --git a/log-shipper/vector.tests.yaml b/log-shipper/vector.tests.yaml index ba810e84..29f7548c 100644 --- a/log-shipper/vector.tests.yaml +++ b/log-shipper/vector.tests.yaml @@ -273,17 +273,32 @@ tests: conditions: - type: vrl source: | - assert_eq!(.event, "ingress_request_failed") + assert_eq!(.event, "ingress_failure_count") assert_eq!(.status, 401) assert_eq!(.route, "/signed/kv/") assert!(!contains(encode_json(.), "ticket-probe")) - - name: "ingress raw route, IP, and identifier values are rejected" + - name: "unknown ingress path is collapsed to bounded unmatched route" inputs: - insert_at: allowlist type: log log_fields: tag: "tinycloud.ingress" log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/signed/kv/ticket-probe","status":404,"bytes":0,"request_time":0.02,"remote_addr":"198.51.100.8"}' + outputs: + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(.event, "ingress_failure_count") + assert_eq!(.route, "") + assert!(!contains(encode_json(.), "ticket-probe")) + - name: "oversized raw ingress route is rejected before normalization" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx","status":404,"bytes":0,"request_time":0.02}' no_outputs_from: [allowlist] - name: "ingress success emits bounded counter input by route only" inputs: @@ -307,7 +322,7 @@ tests: type: log log_fields: tag: "tinycloud.ingress" - log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/info","status":200,"bytes":30,"request_time":1.2,"upstream_status":"200"}' + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/info","status":200,"bytes":30,"request_time":1.2,"upstream_status":"200","remote_addr":"2001:db8::1"}' outputs: - extract_from: allowlist conditions: @@ -322,20 +337,215 @@ tests: assert_eq!(.event, "ingress_request_slow") assert_eq!(.request_time_seconds, 1.2) assert_eq!(.route, "/info") - - name: "per-minute success output is rebuilt from the route counter" + - name: "per-minute request output retains route status class and count" inputs: - - insert_at: success_reduce + - insert_at: request_reduce type: log log_fields: route: "/info" + status_class: "5xx" + event: "ingress_failure_count" request_count: 120 minute: "2026-10-14T10:00" outputs: - - extract_from: success_output + - extract_from: request_output conditions: - type: vrl source: | - assert_eq!(.event, "ingress_success_count") + assert_eq!(.event, "ingress_failure_count") + assert_eq!(.status_class, "5xx") assert_eq!(.request_count, 120) assert_eq!(.route, "/info") - assert_eq!(length(keys(.)), 6) + assert_eq!(.level, "error") + assert_eq!(length(keys(.)), 7) + - name: "404 and slow 2xx floods cannot consume panic or app error lanes" + inputs: + - &slow2xx + insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/info","status":200,"bytes":0,"request_time":2.0,"upstream_status":"200"}' + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - *slow2xx + - &failure404 + insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/info","status":404,"bytes":0,"request_time":0.02,"upstream_status":"-"}' + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - *failure404 + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"ERROR","fields":{"message":"addressed delegation authorization failed"}}' + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"ERROR","fields":{"message":"node fatal panic"}}' + outputs: + - extract_from: throttle_panic + conditions: + - type: vrl + source: | + assert_eq!(.event, "node_fatal_panic") + assert_eq!(.level, "error") + - extract_from: throttle_app_error + conditions: + - type: vrl + source: | + assert_eq!(.event, "node_error") + assert_eq!(.level, "error") + - name: "WARN cannot forge the fatal panic alert" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"node fatal panic"}}' + no_outputs_from: [allowlist] + - name: "node WARN events pass through both their cap and the shared request lane" + inputs: + - insert_at: throttle_app_warning + type: log + log_fields: + level: "warn" + ts: "2026-10-14T10:00:00.000Z" + event: "node_warning" + svc: "api" + event_code: "quota_fallback" + outputs: + - extract_from: throttle_ingress + conditions: + - type: vrl + source: | + assert_eq!(.event, "node_warning") + assert_eq!(.level, "warn") + - name: "multiple upstream statuses normalize to a bounded 5xx code" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"POST","route":"/invoke","status":502,"bytes":0,"request_time":0.2,"upstream_status":"502, 502"}' + outputs: + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(.event, "ingress_failure_count") + assert_eq!(.level, "error") + assert_eq!(.status_class, "5xx") + assert_eq!(.upstream_status, "502") + - name: "UCAN SIWE and delegation text is dropped" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"ERROR","fields":{"message":"UCAN ucan-probe"}}' + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"ERROR","fields":{"message":"SIWE signed-message-probe"}}' + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"ERROR","fields":{"message":"delegation private-delegation-probe"}}' + no_outputs_from: [allowlist] + - name: "ingress forwarding headers nested envelope and IPv6 are excluded" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/info","status":404,"bytes":0,"request_time":0.02,"remote_addr":"2001:db8::1","headers":{"X-Forwarded-For":"198.51.100.9","CF-Connecting-IP":"2001:db8::2","Authorization":"UCAN private-ucan-probe"},"envelope":{"owner":"did:key:nested-did-probe","email":"nested@example.invalid"}}' + outputs: + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(.event, "ingress_failure_count") + assert_eq!(.route, "/info") + assert!(!contains(encode_json(.), "2001:db8::1")) + assert!(!contains(encode_json(.), "198.51.100.9")) + assert!(!contains(encode_json(.), "2001:db8::2")) + assert!(!contains(encode_json(.), "private-ucan-probe")) + assert!(!contains(encode_json(.), "nested-did-probe")) + assert!(!contains(encode_json(.), "nested@example.invalid")) + assert!(!exists(.headers)) + assert!(!exists(.envelope)) + - name: "raw request path on 2xx is replaced by unmatched template" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/signed/kv/ticket-probe","status":200,"bytes":0,"request_time":0.02}' + outputs: + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(.event, "ingress_success_count") + assert_eq!(.route, "") + assert!(!contains(encode_json(.), "ticket-probe")) diff --git a/log-shipper/vector.yaml b/log-shipper/vector.yaml index 7a43c55c..f972c293 100644 --- a/log-shipper/vector.yaml +++ b/log-shipper/vector.yaml @@ -1,6 +1,6 @@ data_dir: /var/lib/vector secret: - bs: { type: directory, path: /run/vector-secrets, remove_trailing_whitespace: true } + bs: { type: directory, path: /tmp/vector-secrets, remove_trailing_whitespace: true } sources: docker_fluent: { type: fluent, mode: tcp, address: 0.0.0.0:24224 } transforms: @@ -23,19 +23,21 @@ transforms: if !is_object(fields) { abort } level = get(j, ["level"]) ?? null if !is_string(level) { abort } - level = downcase(level) + level = downcase!(level) if !includes(["warn", "error"], level) { abort } msg = get(fields, ["message"]) ?? null if !is_string(msg) { abort } + event = null + event_code = null if msg == "invocation replay insert failed" { event = "node_warning"; event_code = "invocation_replay_insert_failed" } else if msg == "invocation replay cleanup failed" { event = "node_warning"; event_code = "invocation_replay_cleanup_failed" } - else if starts_with(msg, "dstack socket available but get_info failed: ") { event = "node_warning"; event_code = "dstack_get_info_failed" } + else if starts_with!(msg, "dstack socket available but get_info failed: ") { event = "node_warning"; event_code = "dstack_get_info_failed" } else if msg == "share-v2 local-tee feature active: deriving a deterministic, non-attested TEE identity from node key material" { event = "node_warning"; event_code = "share_local_tee_identity" } else if msg == "retention: delivered hook delivery prune failed" { event = "node_warning"; event_code = "retention_hook_delivery_prune_failed" } else if msg == "retention: dead-letter hook delivery prune failed" { event = "node_warning"; event_code = "retention_dead_letter_prune_failed" } else if msg == "retention: signed KV ticket prune failed" { event = "node_warning"; event_code = "retention_signed_ticket_prune_failed" } - else if starts_with(msg, "quota service returned an invalid body: ") { event = "node_warning"; event_code = "quota_invalid_body" } - else if starts_with(msg, "quota service returned ") { event = "node_warning"; event_code = "quota_fallback" } + else if starts_with!(msg, "quota service returned an invalid body: ") { event = "node_warning"; event_code = "quota_invalid_body" } + else if starts_with!(msg, "quota service returned ") { event = "node_warning"; event_code = "quota_fallback" } else if msg == "failed to load tunnel state; tunnel not started" { event = "node_warning"; event_code = "tunnel_state_load_failed" } else if msg == "invalid loopback API address; tunnel not started" { event = "node_warning"; event_code = "tunnel_address_invalid" } else if msg == "failed to load link state; LAN listener not started" { event = "node_warning"; event_code = "link_state_load_failed" } @@ -60,7 +62,7 @@ transforms: else if msg == "share v2 is enabled but shareEmail.invitationPublicKey is unset" { event = "node_error"; event_code = "share_invitation_key_missing" } else if msg == "configured share invitation key does not match the node's derived signing key" { event = "node_error"; event_code = "share_invitation_key_mismatch" } else if msg == "share/v2 policy challenge denied" { event = "node_warning"; event_code = "share_policy_denied" } - else if starts_with(msg, "Failed to get TDX quote: ") { event = "node_error"; event_code = "attestation_quote_failed" } + else if starts_with!(msg, "Failed to get TDX quote: ") { event = "node_error"; event_code = "attestation_quote_failed" } else if msg == "failed to read committed hook events" { event = "node_warning"; event_code = "hook_events_read_failed" } else if msg == "failed to load kv write hook rows" { event = "node_warning"; event_code = "hook_rows_load_failed" } else if msg == "failed to load kv delete hook rows" { event = "node_warning"; event_code = "hook_rows_load_failed" } @@ -75,10 +77,11 @@ transforms: else if msg == "tunnel: failed to record runtime state" { event = "node_warning"; event_code = "tunnel_state_record_failed" } else if msg == "tunnel: failed to clear runtime state after disable" { event = "node_warning"; event_code = "tunnel_state_clear_failed" } else if msg == "tunnel authentication rejected" { event = "node_warning"; event_code = "tunnel_auth_rejected" } - else if msg == "node fatal panic" { event = "node_fatal_panic"; event_code = "other"; level = "error" } + else if msg == "node fatal panic" { event = "node_fatal_panic"; event_code = "other" } else { abort } if event == "node_warning" && level != "warn" { abort } if (event == "node_error" || event == "node_fatal_panic") && level != "error" { abort } + level = if event == "node_warning" { "warn" } else { "error" } out.svc = "api" out.level = level out.event = event @@ -96,43 +99,77 @@ transforms: if time_err != null { abort } out.ts = format_timestamp!(parsed_ts, format: "%Y-%m-%dT%H:%M:%S%.3fZ") route = get(j, ["route"]) ?? null - if !(is_string(route) && includes(["/healthz","/info","/version","/invoke","/delegate","/revoke","/attestation","/signed/kv","/signed/kv/","/admin/quota","/admin/quota/","/admin/usage","/hooks/events","/hooks/tickets","/hooks/webhooks","/hooks/webhooks/","/delegation/query","/delegation/status","/encryption/networks","/encryption/networks/","/encryption/networks//revoke","/encryption/networks//decrypt","/policy/v3/status","/policy/v3/status/","/policy/v3/policies","/policy/v3/challenges","/policy/v3/delegations","/policy/v3/enforcer-bindings","/policy/v3/deliveries/authorize","/.well-known/tinycloud/node-keys","/.well-known/encryption/network/","/public//kv","/public//kv/","/peer/generate/","/evidences/",""], route)) { abort } + if !is_string(route) { abort } + route = to_string!(route) + if length(route) > 256 { abort } + if !includes(["/healthz","/info","/version","/invoke","/delegate","/revoke","/attestation","/signed/kv","/signed/kv/","/admin/quota","/admin/quota/","/admin/usage","/hooks/events","/hooks/tickets","/hooks/webhooks","/hooks/webhooks/","/delegation/query","/delegation/status","/encryption/networks","/encryption/networks/","/encryption/networks//revoke","/encryption/networks//decrypt","/policy/v3/status","/policy/v3/status/","/policy/v3/policies","/policy/v3/challenges","/policy/v3/delegations","/policy/v3/enforcer-bindings","/policy/v3/deliveries/authorize","/.well-known/tinycloud/node-keys","/.well-known/encryption/network/","/public//kv","/public//kv/","/peer/generate/","/evidences/"], route) { route = "" } + if !is_string(route) { abort } + route = to_string!(route) + if !includes(["/healthz","/info","/version","/invoke","/delegate","/revoke","/attestation","/signed/kv","/signed/kv/","/admin/quota","/admin/quota/","/admin/usage","/hooks/events","/hooks/tickets","/hooks/webhooks","/hooks/webhooks/","/delegation/query","/delegation/status","/encryption/networks","/encryption/networks/","/encryption/networks//revoke","/encryption/networks//decrypt","/policy/v3/status","/policy/v3/status/","/policy/v3/policies","/policy/v3/challenges","/policy/v3/delegations","/policy/v3/enforcer-bindings","/policy/v3/deliveries/authorize","/.well-known/tinycloud/node-keys","/.well-known/encryption/network/","/public//kv","/public//kv/","/peer/generate/","/evidences/",""], route) { abort } out.route = route method = get(j, ["method"]) ?? null - if !(is_string(method) && includes(["GET","HEAD","POST","PUT","PATCH","DELETE","OPTIONS",""], method)) { abort } + if !is_string(method) { abort } + method = to_string!(method) + if !includes(["GET","HEAD","POST","PUT","PATCH","DELETE","OPTIONS",""], method) { abort } out.method = method status = get(j, ["status"]) ?? null - if !(is_integer(status) && status >= 100 && status <= 599) { abort } + if !is_integer(status) { abort } + status = to_int!(status) + if status < 100 || status > 599 { abort } out.status = status bytes = get(j, ["bytes"]) ?? null - if !(is_integer(bytes) && bytes >= 0 && bytes <= 1000000000) { abort } + if !is_integer(bytes) { abort } + bytes = to_int!(bytes) + if bytes < 0 || bytes > 1000000000 { abort } out.bytes_sent = bytes - upstream_status = get(j, ["upstream_status"]) ?? "" - if upstream_status == "-" { upstream_status = "" } - if !(is_string(upstream_status) && length(upstream_status) <= 3 && match(upstream_status, r'^(|[1-5][0-9]{2})$')) { abort } + upstream_status_raw = "-" + if exists(j.upstream_status) { upstream_status_raw = j.upstream_status } + if !is_string(upstream_status_raw) { abort } + upstream_status_raw = to_string!(upstream_status_raw) + if length(upstream_status_raw) > 63 { abort } + upstream_status = "" + if upstream_status_raw != "-" && upstream_status_raw != "" { + if !match!(upstream_status_raw, r'^[1-5][0-9]{2}(, ?[1-5][0-9]{2})*$') { abort } + upstream_status = slice!(upstream_status_raw, 0, 3) + } + if !is_string(upstream_status) { abort } + upstream_status = to_string!(upstream_status) + if length(upstream_status) > 3 { abort } + if !match!(upstream_status, r'^(|[1-5][0-9]{2})$') { abort } out.upstream_status = upstream_status request_time = get(j, ["request_time"]) ?? null - if !((is_integer(request_time) || is_float(request_time)) && request_time >= 0 && request_time <= 86400) { abort } + if !(is_integer(request_time) || is_float(request_time)) { abort } + request_time = to_float!(request_time) + if request_time < 0 || request_time > 86400 { abort } out.request_time_seconds = request_time status_class = if status >= 500 { "5xx" } else if status >= 400 { "4xx" } else if status >= 300 { "3xx" } else if status >= 200 { "2xx" } else { "1xx" } out.status_class = status_class - if status >= 400 { out.level = if status >= 500 { "error" } else { "warn" }; out.event = "ingress_request_failed" } + if status >= 400 { out.level = if status >= 500 { "error" } else { "warn" }; out.event = "ingress_failure_count"; out.request_count = 1 } else if status >= 200 && status < 300 { out.level = "info"; out.event = "ingress_success_count"; out.request_count = 1 } else if request_time >= 1 { out.level = "warn"; out.event = "ingress_request_slow" } else { abort } } else { abort } . = out + if length(encode_json(.)) > 280 { abort } # END GENERATED FROM fields.json - split_success: + classify_by_budget: type: route inputs: [allowlist] route: - success: .event == "ingress_success_count" - slow_success: .event == "ingress_success_count" && .request_time_seconds >= 1 - other: .event != "ingress_success_count" + panic: .event == "node_fatal_panic" + app_error: .event == "node_error" + app_warning: .event == "node_warning" + ingress_summary: .event == "ingress_failure_count" || .event == "ingress_success_count" + ingress_slow: .event == "ingress_request_slow" + split_requests: + type: route + inputs: [classify_by_budget.ingress_summary] + route: + aggregate: .event == "ingress_failure_count" || .event == "ingress_success_count" + slow_success: .event == "ingress_success_count" && (to_float(.request_time_seconds) ?? 0.0) >= 1.0 slow_success: type: remap - inputs: [split_success.slow_success] + inputs: [split_requests.slow_success] drop_on_error: true source: | . = { @@ -148,46 +185,79 @@ transforms: "bytes_sent": .bytes_sent, "upstream_status": .upstream_status } - success_minute: + request_minute: type: remap - inputs: [split_success.success] + inputs: [split_requests.aggregate] drop_on_error: true source: | .minute = slice!(.ts, 0, 16) - success_reduce: + request_reduce: type: reduce - inputs: [success_minute] - group_by: [route, minute] - expire_after_secs: 75 - flush_period_secs: 60 + inputs: [request_minute] + group_by: [route, status_class, event, minute] + expire_after_ms: 75000 + end_every_period_ms: 60000 + flush_period_ms: 1000 merge_strategies: request_count: sum - success_output: + request_output: type: remap - inputs: [success_reduce] + inputs: [request_reduce] drop_on_error: true drop_on_abort: true source: | route = .route + status_class = .status_class + event = .event count = .request_count - if !is_integer(count) || count < 1 || count > 1000000 { abort } + if !is_integer(count) { abort } + count = to_int!(count) + if count < 1 || count > 1000000 { abort } + level = null + if event == "ingress_success_count" && status_class == "2xx" { level = "info" } + else if event == "ingress_failure_count" && includes(["4xx", "5xx"], status_class) { + level = if status_class == "5xx" { "error" } else { "warn" } + } else { abort } . = { - "level": "info", + "level": level, "ts": format_timestamp!(now(), format: "%Y-%m-%dT%H:%M:%S%.3fZ"), - "event": "ingress_success_count", + "event": event, "svc": "ingress", "route": route, + "status_class": status_class, "request_count": count } - throttle: + throttle_panic: + type: throttle + inputs: [classify_by_budget.panic] + window_secs: 60 + threshold: 2 + throttle_app_error: + type: throttle + inputs: [classify_by_budget.app_error] + window_secs: 60 + threshold: 5 + throttle_app_warning: type: throttle - inputs: [success_output, slow_success, split_success.other] + inputs: [classify_by_budget.app_warning] window_secs: 60 - threshold: 300 + threshold: 5 + throttle_ingress: + type: throttle + inputs: [request_output, classify_by_budget.ingress_slow, slow_success, throttle_app_warning] + window_secs: 60 + threshold: 27 + budget_size_guard: + type: remap + inputs: [throttle_panic, throttle_app_error, throttle_ingress] + drop_on_error: true + drop_on_abort: true + source: | + if length(encode_json(.)) > 280 { abort } sinks: betterstack: type: http - inputs: [throttle] + inputs: [budget_size_guard] method: post uri: "https://betterstack-placeholder.invalid/" encoding: { codec: json } diff --git a/scripts/node-vector-schema.mjs b/scripts/node-vector-schema.mjs index 4edd5950..edda7bf8 100644 --- a/scripts/node-vector-schema.mjs +++ b/scripts/node-vector-schema.mjs @@ -60,13 +60,30 @@ const sourceMessages = { const quote = (value) => JSON.stringify(value); -function vrlPredicate(name, spec, accessor) { - if (spec.type === "enum") return `is_string(${accessor}) && includes(${JSON.stringify(spec.values)}, ${accessor})`; - if (spec.type === "integer") return `is_integer(${accessor}) && ${accessor} >= ${spec.min} && ${accessor} <= ${spec.max}`; - if (spec.type === "number") return `(is_integer(${accessor}) || is_float(${accessor})) && ${accessor} >= ${spec.min} && ${accessor} <= ${spec.max}`; - if (spec.type === "boolean") return `is_boolean(${accessor})`; - if (spec.type === "string") return `is_string(${accessor}) && length(${accessor}) <= ${spec.max_len} && match(${accessor}, r'${spec.regex}')`; - throw new Error(`unsupported type for ${name}`); +function vrlValidationLines(spec, accessor) { + if (spec.type === "enum") return [ + `if !is_string(${accessor}) { abort }`, + `${accessor} = to_string!(${accessor})`, + `if !includes(${JSON.stringify(spec.values)}, ${accessor}) { abort }`, + ]; + if (spec.type === "integer") return [ + `if !is_integer(${accessor}) { abort }`, + `${accessor} = to_int!(${accessor})`, + `if ${accessor} < ${spec.min} || ${accessor} > ${spec.max} { abort }`, + ]; + if (spec.type === "number") return [ + `if !(is_integer(${accessor}) || is_float(${accessor})) { abort }`, + `${accessor} = to_float!(${accessor})`, + `if ${accessor} < ${spec.min} || ${accessor} > ${spec.max} { abort }`, + ]; + if (spec.type === "boolean") return [`if !is_boolean(${accessor}) { abort }`]; + if (spec.type === "string") return [ + `if !is_string(${accessor}) { abort }`, + `${accessor} = to_string!(${accessor})`, + `if length(${accessor}) > ${spec.max_len} { abort }`, + `if !match!(${accessor}, r'${spec.regex}') { abort }`, + ]; + throw new Error(`unsupported type for ${spec.type}`); } export function renderAllowlist(schema) { @@ -87,20 +104,23 @@ export function renderAllowlist(schema) { " if !is_object(fields) { abort }", " level = get(j, [\"level\"]) ?? null", " if !is_string(level) { abort }", - " level = downcase(level)", + " level = downcase!(level)", " if !includes([\"warn\", \"error\"], level) { abort }", " msg = get(fields, [\"message\"]) ?? null", " if !is_string(msg) { abort }", + " event = null", + " event_code = null", ]; const messagePairs = Object.entries(sourceMessages); for (const [index, [message, [event, code]]] of messagePairs.entries()) { - const condition = message.endsWith(": ") || message === "quota service returned " ? `starts_with(msg, ${quote(message)})` : `msg == ${quote(message)}`; + const condition = message.endsWith(": ") || message === "quota service returned " ? `starts_with!(msg, ${quote(message)})` : `msg == ${quote(message)}`; lines.push(` ${index === 0 ? "if" : "else if"} ${condition} { event = ${quote(event)}; event_code = ${quote(code)} }`); } - lines.push(" else if msg == \"node fatal panic\" { event = \"node_fatal_panic\"; event_code = \"other\"; level = \"error\" }"); + lines.push(" else if msg == \"node fatal panic\" { event = \"node_fatal_panic\"; event_code = \"other\" }"); lines.push(" else { abort }"); lines.push(" if event == \"node_warning\" && level != \"warn\" { abort }"); lines.push(" if (event == \"node_error\" || event == \"node_fatal_panic\") && level != \"error\" { abort }"); + lines.push(" level = if event == \"node_warning\" { \"warn\" } else { \"error\" }"); lines.push(" out.svc = \"api\""); lines.push(" out.level = level"); lines.push(" out.event = event"); @@ -118,32 +138,46 @@ export function renderAllowlist(schema) { lines.push(" if time_err != null { abort }"); lines.push(" out.ts = format_timestamp!(parsed_ts, format: \"%Y-%m-%dT%H:%M:%S%.3fZ\")"); lines.push(" route = get(j, [\"route\"]) ?? null"); - lines.push(` if !(${vrlPredicate("route", service.fields.route, "route")}) { abort }`); + lines.push(" if !is_string(route) { abort }"); + lines.push(" route = to_string!(route)"); + lines.push(" if length(route) > 256 { abort }"); + const knownRoutes = service.fields.route.values.filter((value) => value !== ""); + lines.push(` if !includes(${JSON.stringify(knownRoutes)}, route) { route = "" }`); + for (const line of vrlValidationLines(service.fields.route, "route")) lines.push(` ${line}`); lines.push(" out.route = route"); lines.push(" method = get(j, [\"method\"]) ?? null"); - lines.push(` if !(${vrlPredicate("method", service.fields.method, "method")}) { abort }`); + for (const line of vrlValidationLines(service.fields.method, "method")) lines.push(` ${line}`); lines.push(" out.method = method"); lines.push(" status = get(j, [\"status\"]) ?? null"); - lines.push(` if !(${vrlPredicate("status", service.fields.status, "status")}) { abort }`); + for (const line of vrlValidationLines(service.fields.status, "status")) lines.push(` ${line}`); lines.push(" out.status = status"); lines.push(" bytes = get(j, [\"bytes\"]) ?? null"); - lines.push(` if !(${vrlPredicate("bytes_sent", service.fields.bytes_sent, "bytes")}) { abort }`); + for (const line of vrlValidationLines(service.fields.bytes_sent, "bytes")) lines.push(` ${line}`); lines.push(" out.bytes_sent = bytes"); - lines.push(" upstream_status = get(j, [\"upstream_status\"]) ?? \"\""); - lines.push(" if upstream_status == \"-\" { upstream_status = \"\" }"); - lines.push(` if !(${vrlPredicate("upstream_status", service.fields.upstream_status, "upstream_status")}) { abort }`); + lines.push(" upstream_status_raw = \"-\""); + lines.push(" if exists(j.upstream_status) { upstream_status_raw = j.upstream_status }"); + lines.push(" if !is_string(upstream_status_raw) { abort }"); + lines.push(" upstream_status_raw = to_string!(upstream_status_raw)"); + lines.push(" if length(upstream_status_raw) > 63 { abort }"); + lines.push(" upstream_status = \"\""); + lines.push(" if upstream_status_raw != \"-\" && upstream_status_raw != \"\" {"); + lines.push(" if !match!(upstream_status_raw, r'^[1-5][0-9]{2}(, ?[1-5][0-9]{2})*$') { abort }"); + lines.push(" upstream_status = slice!(upstream_status_raw, 0, 3)"); + lines.push(" }"); + for (const line of vrlValidationLines(service.fields.upstream_status, "upstream_status")) lines.push(` ${line}`); lines.push(" out.upstream_status = upstream_status"); lines.push(" request_time = get(j, [\"request_time\"]) ?? null"); - lines.push(` if !(${vrlPredicate("request_time_seconds", service.fields.request_time_seconds, "request_time")}) { abort }`); + for (const line of vrlValidationLines(service.fields.request_time_seconds, "request_time")) lines.push(` ${line}`); lines.push(" out.request_time_seconds = request_time"); lines.push(" status_class = if status >= 500 { \"5xx\" } else if status >= 400 { \"4xx\" } else if status >= 300 { \"3xx\" } else if status >= 200 { \"2xx\" } else { \"1xx\" }"); lines.push(" out.status_class = status_class"); - lines.push(" if status >= 400 { out.level = if status >= 500 { \"error\" } else { \"warn\" }; out.event = \"ingress_request_failed\" }"); + lines.push(" if status >= 400 { out.level = if status >= 500 { \"error\" } else { \"warn\" }; out.event = \"ingress_failure_count\"; out.request_count = 1 }"); lines.push(" else if status >= 200 && status < 300 { out.level = \"info\"; out.event = \"ingress_success_count\"; out.request_count = 1 }"); lines.push(" else if request_time >= 1 { out.level = \"warn\"; out.event = \"ingress_request_slow\" }"); lines.push(" else { abort }"); lines.push("} else { abort }"); lines.push(". = out"); + lines.push("if length(encode_json(.)) > 280 { abort }"); lines.push(endMarker); return lines.join("\n"); } @@ -153,15 +187,38 @@ export function checkVectorFile(schema, yaml = readFileSync(vectorPath, "utf8")) if (!match) throw new Error("vector.yaml is missing generated allowlist markers"); const actual = match[0].split("\n").map((line) => line.replace(/^ /, "")).join("\n"); if (actual !== renderAllowlist(schema)) throw new Error("vector.yaml allowlist differs from fields.json-generated VRL"); - for (const name of ["allowlist", "slow_success", "success_minute", "success_output"]) { + for (const name of ["allowlist", "slow_success", "request_minute", "request_output", "budget_size_guard"]) { const block = yaml.match(new RegExp(`^ ${name}:\\n([\\s\\S]*?)(?=\\n [a-z][a-z0-9_]*:\\n|\\nsinks:)`, "m"))?.[1]; if (!block?.includes("type: remap") || !block.includes("drop_on_error: true")) throw new Error(`${name} remap must set drop_on_error: true`); } - const successOutput = yaml.match(/ success_output:\n([\s\S]*?)(?=\n [a-z][a-z0-9_]*:\n|$)/)?.[1]; - if (!successOutput?.includes("drop_on_abort: true") || !successOutput.includes("drop_on_error: true")) throw new Error("success_output must drop aborts and remap errors"); + const requestOutput = yaml.match(/ request_output:\n([\s\S]*?)(?=\n [a-z][a-z0-9_]*:\n|$)/)?.[1]; + if (!requestOutput?.includes("drop_on_abort: true")) throw new Error("request_output must drop aborts"); const count = schema.services["tinycloud-node"].fields.request_count; - const countGuard = `if !is_integer(count) || count < ${count.min} || count > ${count.max} { abort }`; - if (!successOutput.includes(countGuard)) throw new Error("success_output count bounds differ from fields.json"); + const countGuard = `if !is_integer(count) { abort }\n count = to_int!(count)\n if count < ${count.min} || count > ${count.max} { abort }`; + if (!requestOutput.includes(countGuard)) throw new Error("request output count bounds differ from fields.json"); + const requiredBudgetConfig = [ + "panic: .event == \"node_fatal_panic\"", + "app_error: .event == \"node_error\"", + "app_warning: .event == \"node_warning\"", + "ingress_summary: .event == \"ingress_failure_count\" || .event == \"ingress_success_count\"", + "group_by: [route, status_class, event, minute]", + "inputs: [classify_by_budget.panic]", + "threshold: 2", + "inputs: [classify_by_budget.app_error]", + "threshold: 5", + "inputs: [classify_by_budget.app_warning]", + "inputs: [request_output, classify_by_budget.ingress_slow, slow_success, throttle_app_warning]", + "threshold: 27", + "expire_after_ms: 75000", + "end_every_period_ms: 60000", + "flush_period_ms: 1000", + "if length(encode_json(.)) > 280 { abort }", + "inputs: [throttle_panic, throttle_app_error, throttle_ingress]", + ]; + if (requiredBudgetConfig.some((line) => !yaml.includes(line))) throw new Error("Vector priority lanes, counters, reduce timers, or event-size bound differ from reviewed budget"); + const monthlyBudgetBytes = (2 + 5 + 5 + 27) * 60 * 24 * 30 * 281; + if (monthlyBudgetBytes > 480000000) throw new Error(`Vector event budget exceeds 0.48 GB/month: ${monthlyBudgetBytes} bytes`); + if ((yaml.match(/threshold: 5\n/g) ?? []).length !== 2) throw new Error("node error and warning lanes must each be capped at five events/minute"); } export function renderComposeVectorConfig(vectorYaml) { return vectorYaml.trimEnd().split("\n").map((line) => ` ${line.replaceAll("$", () => "$$")}`).join("\n"); @@ -197,12 +254,16 @@ export function checkNodeSchema(schema) { if (JSON.stringify(schema.common.level) !== JSON.stringify({ type: "enum", values: ["debug", "info", "warn", "error"] })) throw new Error("common.level differs from the emitted level allowlist"); if (JSON.stringify(schema.common.ts) !== JSON.stringify({ type: "string", regex: "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$", max_len: 24 })) throw new Error("common.ts differs from the emitted timestamp format"); if (schema.common.event?.type !== "event") throw new Error("common.event must use event type"); - if (service.events.join(",") !== "node_fatal_panic,node_warning,node_error,ingress_request_failed,ingress_request_slow,ingress_success_count") throw new Error("tinycloud-node event set differs from the reviewed set"); + if (service.events.join(",") !== "node_fatal_panic,node_warning,node_error,ingress_failure_count,ingress_request_slow,ingress_success_count") throw new Error("tinycloud-node event set differs from the reviewed set"); if (Object.values(sourceMessages).some(([, code]) => !new Set(service.fields.event_code?.values ?? []).has(code))) throw new Error("source message map contains an event_code missing from fields.json"); if (Object.keys(service.fields).sort().join(",") !== [ "bytes_sent", "event_code", "method", "request_count", "request_time_seconds", "route", "status", "status_class", "svc", "upstream_status", ].sort().join(",")) throw new Error("tinycloud-node fields differ from the reviewed field set"); if (service.fields.request_count?.type !== "integer" || service.fields.request_count.min !== 1 || service.fields.request_count.max !== 1000000) throw new Error("request_count must remain bounded to 1..1000000"); + if (service.fields.route?.type !== "enum" || service.fields.route.values.length !== 36 || !service.fields.route.values.includes("")) throw new Error("route output must be a 36-value bounded enum including "); + if (JSON.stringify(service.fields.svc) !== JSON.stringify({ type: "enum", values: ["api", "ingress"] })) throw new Error("svc enum differs from emitted values"); + if (JSON.stringify(service.fields.status_class) !== JSON.stringify({ type: "enum", values: ["1xx", "2xx", "3xx", "4xx", "5xx"] })) throw new Error("status_class enum differs from emitted values"); + if (service.fields.event_code?.type !== "enum" || !service.fields.event_code.values.includes("other")) throw new Error("event_code must preserve the fixed panic other value"); } if (import.meta.url === `file://${process.argv[1]}`) { diff --git a/scripts/node-vector-schema.test.mjs b/scripts/node-vector-schema.test.mjs index 55ac346b..ecf50145 100644 --- a/scripts/node-vector-schema.test.mjs +++ b/scripts/node-vector-schema.test.mjs @@ -3,7 +3,10 @@ import { readFileSync } from "node:fs"; import test from "node:test"; import { checkVectorFile, renderAllowlist, renderComposeVectorConfig } from "./node-vector-schema.mjs"; +const workflow = readFileSync(new URL("../.github/workflows/rust.yml", import.meta.url), "utf8"); const schema = JSON.parse(readFileSync(new URL("../log-shipper/fields.json", import.meta.url), "utf8")); +const compose = readFileSync(new URL("../docker-compose.dstack-postgres.yaml", import.meta.url), "utf8"); +const deployWorkflow = readFileSync(new URL("../.github/workflows/docker.yml", import.meta.url), "utf8"); test("generated allowlist emits known fixed messages and builds output from validated fields", () => { const result = renderAllowlist(schema); @@ -13,12 +16,17 @@ test("generated allowlist emits known fixed messages and builds output from vali assert.doesNotMatch(result, /out\.message|out\.msg|\. = j/); assert.match(result, /parse_err != null \|\| !is_object\(j\) \{ abort \}/); assert.match(result, /if exists\(\.partial_message\) \{ abort \}/); + assert.match(result, /if !is_string\(route\) \{ abort \}[\s\S]*if length\(route\) > 256 \{ abort \}/); + assert.match(result, /if !includes\(\[.*\/healthz.*\], route\) \{ route = "" \}/); + assert.match(result, /length\(encode_json\(\.\)\) > 280/); + assert.match(result, /event_code = "other"/); + assert.match(result, /level = if event == "node_warning" \{ "warn" \} else \{ "error" \}/); }); test("unknown event message and malformed allowed values abort instead of forwarding", () => { const result = renderAllowlist(schema); assert.match(result, /else \{ abort \}/); - assert.match(result, /is_integer\(status\) && status >= 100 && status <= 599/); + assert.match(result, /if !is_integer\(status\) \{ abort \}[\s\S]*if status < 100 \|\| status > 599 \{ abort \}/); }); test("generator rejects schema drift outside common and tinycloud-node", () => { @@ -45,12 +53,67 @@ test("generator rejects schema drift outside common and tinycloud-node", () => { /request_count/, ); }); +test("generator rejects incompatible enums and removed event codes", () => { + const withField = (name, field) => ({ + ...schema, + services: { + ...schema.services, + "tinycloud-node": { + ...schema.services["tinycloud-node"], + fields: { ...schema.services["tinycloud-node"].fields, [name]: field }, + }, + }, + }); + assert.throws(() => renderAllowlist(withField("svc", { type: "enum", values: ["api"] })), /svc enum/); + assert.throws(() => renderAllowlist(withField("status_class", { type: "enum", values: ["2xx", "4xx"] })), /status_class enum/); + assert.throws( + () => renderAllowlist(withField("event_code", { + ...schema.services["tinycloud-node"].fields.event_code, + values: schema.services["tinycloud-node"].fields.event_code.values.filter((value) => value !== "invocation_replay_insert_failed"), + })), + /event_code missing/, + ); + assert.throws( + () => renderAllowlist(withField("event_code", { + ...schema.services["tinycloud-node"].fields.event_code, + values: schema.services["tinycloud-node"].fields.event_code.values.filter((value) => value !== "other"), + })), + /panic other value/, + ); +}); test("success counter validates its bounded output and abort policy", () => { assert.doesNotThrow(() => checkVectorFile(schema)); }); +test("CI runs native Vector validation and tests after checking the pinned release checksum", () => { + assert.match(workflow, /version=0\.59\.0/); + assert.match(workflow, /expected_sha256=a8dbc43c18ae25d0b23a712c9262f3aad904e352b1db5e7db1d9c4aecab59496/); + assert.match(workflow, /sha256sum --check --ignore-missing/); + assert.match(workflow, /validate --no-environment log-shipper\/vector\.yaml/); + assert.match(workflow, /test log-shipper\/vector\.yaml log-shipper\/vector\.tests\.yaml/); +}); test("compose embeds source Vector config without allowing Compose to interpolate its regexes", () => { assert.equal( renderComposeVectorConfig("sources:\n test: true\nregex: ^[a-z]+$"), " sources:\n test: true\n regex: ^[a-z]+$$", ); }); +test("missing Better Stack token leaves node and ingress startup independent", () => { + assert.match(compose, /BETTERSTACK_SOURCE_TOKEN: "\$\{BETTERSTACK_SOURCE_TOKEN:-\}"/); + assert.doesNotMatch(compose, /BETTERSTACK_SOURCE_TOKEN[^\n]*:\?/); + const node = compose.slice(compose.indexOf(" tinycloud:\n"), compose.indexOf(" dstack-ingress:\n")); + const ingress = compose.slice(compose.indexOf(" dstack-ingress:\n"), compose.indexOf(" ssh-gateway:\n")); + assert.doesNotMatch(node, /depends_on:[\s\S]*?vector/); + assert.doesNotMatch(ingress, /depends_on:[\s\S]*?vector/); + assert.match(node, /fluentd-async: "true"/); + assert.match(ingress, /fluentd-async: "true"/); + assert.match(compose, /Better Stack token is absent; Vector is idle/); + const requiredVarsStart = deployWorkflow.indexOf("required_vars=("); + const requiredVarsEnd = deployWorkflow.indexOf("\n )", requiredVarsStart); + assert.ok(requiredVarsStart >= 0 && requiredVarsEnd > requiredVarsStart); + assert.doesNotMatch(deployWorkflow.slice(requiredVarsStart, requiredVarsEnd), /BETTERSTACK_SOURCE_TOKEN/); + assert.ok( + deployWorkflow.indexOf("phala deploy --cvm-id tinycloud-node") < + deployWorkflow.indexOf("name: Sync production allowed_envs"), + "allowed_envs sync must not gate compose startup", + ); +}); diff --git a/tinycloud-node-server/src/runtime.rs b/tinycloud-node-server/src/runtime.rs index 757f41a9..82dbeb8b 100644 --- a/tinycloud-node-server/src/runtime.rs +++ b/tinycloud-node-server/src/runtime.rs @@ -649,6 +649,30 @@ mod tests { }) ); } + #[::core::prelude::v1::test] + fn serve_config_figment_reads_json_log_format_from_canonical_env() { + let _lock = env_lock(); + let temp = tempdir().unwrap(); + let base = temp.path().join("base.toml"); + std::fs::write(&base, "[global]\n").unwrap(); + let _storage_datadir = EnvGuard::unset("TINYCLOUD_STORAGE_DATADIR"); + let _storage_datadir_canonical = EnvGuard::unset("TINYCLOUD_STORAGE__DATADIR"); + let _old_logging_key = EnvGuard::unset("TINYCLOUD_LOGGING__FORMAT"); + let _legacy_log_key = EnvGuard::unset("TINYCLOUD_LOG_FORMAT"); + let _canonical_log_key = EnvGuard::set("TINYCLOUD_LOG__FORMAT", "Json"); + let _keys_env = clear_keys_env(); + let _rocket_address = EnvGuard::unset("ROCKET_ADDRESS"); + let _rocket_port = EnvGuard::unset("ROCKET_PORT"); + let _rocket_config = EnvGuard::unset("ROCKET_CONFIG"); + let _rocket_profile = EnvGuard::unset("ROCKET_PROFILE"); + + let cfg = serve_config_figment(&base) + .unwrap() + .extract::() + .unwrap(); + + assert_eq!(cfg.log.format, config::LoggingFormat::Json); + } #[::core::prelude::v1::test] fn serve_config_figment_defaults_to_spec_bind_without_config_file() { From c70976cb6d860b01d850f57cf2010d3980026ac0 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 09:41:20 +0000 Subject: [PATCH 03/21] Fix node log shipping review findings --- README.md | 2 +- docker-compose.dstack-postgres.yaml | 16 +++- .../tc-959-betterstack-log-shipping.md | 4 +- log-shipper/vector.tests.yaml | 92 ++++++++++++++++++- log-shipper/vector.yaml | 16 +++- scripts/node-vector-schema.mjs | 22 ++++- scripts/node-vector-schema.test.mjs | 10 +- 7 files changed, 141 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index abf3ddad..e68105c1 100644 --- a/README.md +++ b/README.md @@ -298,7 +298,7 @@ The secret MUST contain at least 32 bytes of entropy (either randomly generated The production Compose file sends redacted JSON records through Vector to Better Stack. A missing source token leaves only Vector idle; the node and ingress start independently and use asynchronous Fluentd logging. -Node output is bounded to 280 JSON bytes plus one line separator. The caps are 2 panic/min, 5 app ERROR/min, 5 WARN/min, and 27/min through the shared request lane. WARN records first pass their 5/min cap, then share the request lane with aggregates and slow requests; ERROR and panic remain isolated. The conservative monthly ceiling charges all lane caps separately: `(2 + 5 + 5 + 27) × 60 × 24 × 30 × 281 = 473,428,800 bytes` (about 0.473 GB per 30 days), below the node's 0.48 GB allocation of the shared free-plan quota. Ingress 4xx/5xx counts aggregate by route template and status class; successful requests aggregate by route. There are 36 route values, including `` for unknown paths. +Node output is bounded to 280 JSON bytes plus one line separator. The caps are 2 panic/min, 5 app ERROR/min, 5 WARN/min, and 27/min through the ingress/request lane. WARN bypasses the ingress throttle and joins panic and ERROR directly at the final size guard. The conservative monthly ceiling charges all lane caps independently: `(2 + 5 + 5 + 27) × 60 × 24 × 30 × 281 = 473,428,800 bytes` (about 0.473 GB per 30 days), below the node's 0.48 GB allocation of the shared free-plan quota. Ingress 4xx/5xx counts aggregate by route template and status class; successful requests aggregate by route. There are 36 route values, including `` for unknown paths. The Docker review check pulls the registry `dstack` image; its Linux/amd64 manifest measured 17,115,015 compressed bytes across seven layers. ## Running diff --git a/docker-compose.dstack-postgres.yaml b/docker-compose.dstack-postgres.yaml index f9b67030..d09df2ae 100644 --- a/docker-compose.dstack-postgres.yaml +++ b/docker-compose.dstack-postgres.yaml @@ -286,7 +286,7 @@ services: if length(upstream_status_raw) > 63 { abort } upstream_status = "" if upstream_status_raw != "-" && upstream_status_raw != "" { - if !match!(upstream_status_raw, r'^[1-5][0-9]{2}(, ?[1-5][0-9]{2})*$$') { abort } + if !match!(upstream_status_raw, r'^[1-5][0-9]{2}(, ?[1-5][0-9]{2}| : [1-5][0-9]{2})*$$') { abort } upstream_status = slice!(upstream_status_raw, 0, 3) } if !is_string(upstream_status) { abort } @@ -307,7 +307,6 @@ services: else { abort } } else { abort } . = out - if length(encode_json(.)) > 280 { abort } # END GENERATED FROM fields.json classify_by_budget: type: route @@ -347,7 +346,14 @@ services: inputs: [split_requests.aggregate] drop_on_error: true source: | - .minute = slice!(.ts, 0, 16) + minute = slice!(.ts, 0, 16) + . = { + "route": .route, + "status_class": .status_class, + "event": .event, + "minute": minute, + "request_count": .request_count + } request_reduce: type: reduce inputs: [request_minute] @@ -401,12 +407,12 @@ services: threshold: 5 throttle_ingress: type: throttle - inputs: [request_output, classify_by_budget.ingress_slow, slow_success, throttle_app_warning] + inputs: [request_output, classify_by_budget.ingress_slow, slow_success] window_secs: 60 threshold: 27 budget_size_guard: type: remap - inputs: [throttle_panic, throttle_app_error, throttle_ingress] + inputs: [throttle_panic, throttle_app_error, throttle_app_warning, throttle_ingress] drop_on_error: true drop_on_abort: true source: | diff --git a/docs/runbooks/tc-959-betterstack-log-shipping.md b/docs/runbooks/tc-959-betterstack-log-shipping.md index 3ae5412b..33f7a7fe 100644 --- a/docs/runbooks/tc-959-betterstack-log-shipping.md +++ b/docs/runbooks/tc-959-betterstack-log-shipping.md @@ -6,9 +6,9 @@ The node app emits JSON tracing records (`TINYCLOUD_LOG__FORMAT=Json`). The ingr The VRL remap starts from an empty object. Node records require a known fixed message; variable error text is matched only by a fixed prefix and never copied. Unknown messages, non-JSON, non-object JSON, malformed typed fields, and Docker partial records are dropped and increment the Vector remap component's internal discarded-event counters. Those counters are not exposed through an API or metrics sink. Ingress routes, methods, status, duration, response bytes, and upstream status are validated against `log-shipper/fields.json`; only route templates are emitted. Raw paths, query strings, request headers, error text, and arbitrary nested values are not forwarded. Remap failures are dropped. -Ingress failures are counted per minute by `(route, status_class)`, successful requests by route, and slow requests remain bounded warning events; a slow 2xx is both counted and marked slow. Panic has a dedicated 2/min lane and node ERROR has a dedicated 5/min lane. Node WARN is capped at 5/min and then shares the request lane with ingress aggregates and slow requests, capped at 27/min. Request traffic and WARN cannot consume the panic or ERROR lanes. The sink retains at most 1,000 events in memory and drops newest when full. There is no Vector API or exposed metrics port; Fluent input is published only on `127.0.0.1:24224`. +Ingress failures are counted per minute by `(route, status_class)` and successful requests by route; reducer input is projected to `(route, status_class, event, minute, request_count)` before aggregation. Slow requests remain bounded warning events; a slow 2xx is both counted and marked slow. Panic, node ERROR, and node WARN have dedicated 2/min, 5/min, and 5/min lanes. WARN bypasses the ingress throttle and feeds the final size guard directly; the separate 27/min ingress throttle covers request aggregates and slow requests only. The sink retains at most 1,000 events in memory and drops newest when full. There is no Vector API or exposed metrics port; Fluent input is published only on `127.0.0.1:24224`. -The ingest hostname is configured once as `BETTERSTACK_SOURCE_HOST` in the Vector service environment; the entrypoint validates it and substitutes it into the generated config's placeholder before Vector starts. The source token is the GitHub `production` environment secret `PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN`; deploy writes it into the sealed environment as `BETTERSTACK_SOURCE_TOKEN` and synchronizes `allowed_envs` after compose startup. An absent token leaves only Vector idle; the node and ingress have no dependency on it. Docker container environment metadata is visible to principals with `docker inspect` access, so restrict CVM Docker access accordingly. No token value belongs in this repository or logs. +The ingest hostname is configured once as `BETTERSTACK_SOURCE_HOST` in the Vector service environment; the entrypoint validates it and substitutes it into the generated config's placeholder before Vector starts. The source token is the GitHub `production` environment secret `PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN`; deploy writes it into the sealed environment as `BETTERSTACK_SOURCE_TOKEN` and synchronizes `allowed_envs` after compose startup. An absent token leaves only Vector idle; the node and ingress have no dependency on it. Because the token is present in Vector's container environment, `docker inspect ` exposes it to principals with Docker daemon/API access, even though Vector also writes it to a runtime secret file. Restrict Docker access accordingly. No token value belongs in this repository or logs. At go-live, create a Better Stack alert for `event = node_fatal_panic`. diff --git a/log-shipper/vector.tests.yaml b/log-shipper/vector.tests.yaml index 29f7548c..b23b21cc 100644 --- a/log-shipper/vector.tests.yaml +++ b/log-shipper/vector.tests.yaml @@ -358,7 +358,7 @@ tests: assert_eq!(.route, "/info") assert_eq!(.level, "error") assert_eq!(length(keys(.)), 7) - - name: "404 and slow 2xx floods cannot consume panic or app error lanes" + - name: "404 and slow 2xx floods cannot consume panic, app error, or WARN lanes" inputs: - &slow2xx insert_at: allowlist @@ -430,6 +430,11 @@ tests: - *failure404 - *failure404 - *failure404 + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.node" + log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"invocation replay insert failed"}}' - insert_at: allowlist type: log log_fields: @@ -453,6 +458,87 @@ tests: source: | assert_eq!(.event, "node_error") assert_eq!(.level, "error") + - extract_from: throttle_app_warning + conditions: + - type: vrl + source: | + assert_eq!(.event, "node_warning") + assert_eq!(.level, "warn") + assert_eq!(.event_code, "invocation_replay_insert_failed") + - extract_from: budget_size_guard + conditions: + - type: vrl + source: | + assert_eq!(.event, "node_warning") + assert_eq!(.level, "warn") + - name: "504 colon-separated upstream status is normalized to one status" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"POST","route":"/invoke","status":504,"bytes":0,"request_time":0.2,"upstream_status":"504 : 200"}' + outputs: + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(.event, "ingress_failure_count") + assert_eq!(.level, "error") + assert_eq!(.status_class, "5xx") + assert_eq!(.upstream_status, "504") + - name: "long-route 2xx counter input survives the final-size projection" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/encryption/networks//decrypt","status":200,"bytes":1000,"request_time":0.01,"upstream_status":"200"}' + outputs: + - extract_from: request_minute + conditions: + - type: vrl + source: | + assert_eq!(.event, "ingress_success_count") + assert_eq!(.status_class, "2xx") + assert_eq!(.route, "/encryption/networks//decrypt") + assert_eq!(.request_count, 1) + assert_eq!(length(keys(.)), 5) + - extract_from: request_output + conditions: + - type: vrl + source: | + assert_eq!(.event, "ingress_success_count") + assert_eq!(.status_class, "2xx") + assert_eq!(.route, "/encryption/networks//decrypt") + assert_eq!(.request_count, 1) + assert_eq!(length(keys(.)), 7) + - name: "long-route 5xx counter input survives the final-size projection" + inputs: + - insert_at: allowlist + type: log + log_fields: + tag: "tinycloud.ingress" + log: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/encryption/networks//decrypt","status":502,"bytes":1000,"request_time":0.01,"upstream_status":"502"}' + outputs: + - extract_from: request_minute + conditions: + - type: vrl + source: | + assert_eq!(.event, "ingress_failure_count") + assert_eq!(.status_class, "5xx") + assert_eq!(.route, "/encryption/networks//decrypt") + assert_eq!(.request_count, 1) + assert_eq!(length(keys(.)), 5) + - extract_from: request_output + conditions: + - type: vrl + source: | + assert_eq!(.event, "ingress_failure_count") + assert_eq!(.status_class, "5xx") + assert_eq!(.route, "/encryption/networks//decrypt") + assert_eq!(.request_count, 1) + assert_eq!(length(keys(.)), 7) - name: "WARN cannot forge the fatal panic alert" inputs: - insert_at: allowlist @@ -461,7 +547,7 @@ tests: tag: "tinycloud.node" log: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"node fatal panic"}}' no_outputs_from: [allowlist] - - name: "node WARN events pass through both their cap and the shared request lane" + - name: "node WARN uses its dedicated cap and reaches the final guard without ingress throttle" inputs: - insert_at: throttle_app_warning type: log @@ -472,7 +558,7 @@ tests: svc: "api" event_code: "quota_fallback" outputs: - - extract_from: throttle_ingress + - extract_from: budget_size_guard conditions: - type: vrl source: | diff --git a/log-shipper/vector.yaml b/log-shipper/vector.yaml index f972c293..e7cb164a 100644 --- a/log-shipper/vector.yaml +++ b/log-shipper/vector.yaml @@ -129,7 +129,7 @@ transforms: if length(upstream_status_raw) > 63 { abort } upstream_status = "" if upstream_status_raw != "-" && upstream_status_raw != "" { - if !match!(upstream_status_raw, r'^[1-5][0-9]{2}(, ?[1-5][0-9]{2})*$') { abort } + if !match!(upstream_status_raw, r'^[1-5][0-9]{2}(, ?[1-5][0-9]{2}| : [1-5][0-9]{2})*$') { abort } upstream_status = slice!(upstream_status_raw, 0, 3) } if !is_string(upstream_status) { abort } @@ -150,7 +150,6 @@ transforms: else { abort } } else { abort } . = out - if length(encode_json(.)) > 280 { abort } # END GENERATED FROM fields.json classify_by_budget: type: route @@ -190,7 +189,14 @@ transforms: inputs: [split_requests.aggregate] drop_on_error: true source: | - .minute = slice!(.ts, 0, 16) + minute = slice!(.ts, 0, 16) + . = { + "route": .route, + "status_class": .status_class, + "event": .event, + "minute": minute, + "request_count": .request_count + } request_reduce: type: reduce inputs: [request_minute] @@ -244,12 +250,12 @@ transforms: threshold: 5 throttle_ingress: type: throttle - inputs: [request_output, classify_by_budget.ingress_slow, slow_success, throttle_app_warning] + inputs: [request_output, classify_by_budget.ingress_slow, slow_success] window_secs: 60 threshold: 27 budget_size_guard: type: remap - inputs: [throttle_panic, throttle_app_error, throttle_ingress] + inputs: [throttle_panic, throttle_app_error, throttle_app_warning, throttle_ingress] drop_on_error: true drop_on_abort: true source: | diff --git a/scripts/node-vector-schema.mjs b/scripts/node-vector-schema.mjs index edda7bf8..d6faea97 100644 --- a/scripts/node-vector-schema.mjs +++ b/scripts/node-vector-schema.mjs @@ -161,7 +161,7 @@ export function renderAllowlist(schema) { lines.push(" if length(upstream_status_raw) > 63 { abort }"); lines.push(" upstream_status = \"\""); lines.push(" if upstream_status_raw != \"-\" && upstream_status_raw != \"\" {"); - lines.push(" if !match!(upstream_status_raw, r'^[1-5][0-9]{2}(, ?[1-5][0-9]{2})*$') { abort }"); + lines.push(" if !match!(upstream_status_raw, r'^[1-5][0-9]{2}(, ?[1-5][0-9]{2}| : [1-5][0-9]{2})*$') { abort }"); lines.push(" upstream_status = slice!(upstream_status_raw, 0, 3)"); lines.push(" }"); for (const line of vrlValidationLines(service.fields.upstream_status, "upstream_status")) lines.push(` ${line}`); @@ -177,7 +177,6 @@ export function renderAllowlist(schema) { lines.push(" else { abort }"); lines.push("} else { abort }"); lines.push(". = out"); - lines.push("if length(encode_json(.)) > 280 { abort }"); lines.push(endMarker); return lines.join("\n"); } @@ -196,6 +195,21 @@ export function checkVectorFile(schema, yaml = readFileSync(vectorPath, "utf8")) const count = schema.services["tinycloud-node"].fields.request_count; const countGuard = `if !is_integer(count) { abort }\n count = to_int!(count)\n if count < ${count.min} || count > ${count.max} { abort }`; if (!requestOutput.includes(countGuard)) throw new Error("request output count bounds differ from fields.json"); + const requestMinute = yaml.match(/ request_minute:\n([\s\S]*?)(?=\n [a-z][a-z0-9_]*:\n|$)/)?.[1]; + const projectedFields = [ + "minute = slice!(.ts, 0, 16)", + '"route": .route', + '"status_class": .status_class', + '"event": .event', + '"minute": minute', + '"request_count": .request_count', + ]; + if (!requestMinute || projectedFields.some((field) => !requestMinute.includes(field))) throw new Error("request_minute must project the five bounded reducer fields"); + const sizeGuard = "if length(encode_json(.)) > 280 { abort }"; + const sizeGuardMatches = yaml.match(/if length\(encode_json\(\.\)\) > 280 \{ abort \}/g) ?? []; + if (sizeGuardMatches.length !== 1 || !yaml.match(/ budget_size_guard:\n([\s\S]*?)(?=\n [a-z][a-z0-9_]*:\n|\nsinks:)/)?.[1].includes(sizeGuard)) { + throw new Error("the only event-size guard must be budget_size_guard"); + } const requiredBudgetConfig = [ "panic: .event == \"node_fatal_panic\"", "app_error: .event == \"node_error\"", @@ -207,13 +221,13 @@ export function checkVectorFile(schema, yaml = readFileSync(vectorPath, "utf8")) "inputs: [classify_by_budget.app_error]", "threshold: 5", "inputs: [classify_by_budget.app_warning]", - "inputs: [request_output, classify_by_budget.ingress_slow, slow_success, throttle_app_warning]", + "inputs: [request_output, classify_by_budget.ingress_slow, slow_success]", "threshold: 27", "expire_after_ms: 75000", "end_every_period_ms: 60000", "flush_period_ms: 1000", "if length(encode_json(.)) > 280 { abort }", - "inputs: [throttle_panic, throttle_app_error, throttle_ingress]", + "inputs: [throttle_panic, throttle_app_error, throttle_app_warning, throttle_ingress]", ]; if (requiredBudgetConfig.some((line) => !yaml.includes(line))) throw new Error("Vector priority lanes, counters, reduce timers, or event-size bound differ from reviewed budget"); const monthlyBudgetBytes = (2 + 5 + 5 + 27) * 60 * 24 * 30 * 281; diff --git a/scripts/node-vector-schema.test.mjs b/scripts/node-vector-schema.test.mjs index ecf50145..6f40ae5e 100644 --- a/scripts/node-vector-schema.test.mjs +++ b/scripts/node-vector-schema.test.mjs @@ -7,6 +7,7 @@ const workflow = readFileSync(new URL("../.github/workflows/rust.yml", import.me const schema = JSON.parse(readFileSync(new URL("../log-shipper/fields.json", import.meta.url), "utf8")); const compose = readFileSync(new URL("../docker-compose.dstack-postgres.yaml", import.meta.url), "utf8"); const deployWorkflow = readFileSync(new URL("../.github/workflows/docker.yml", import.meta.url), "utf8"); +const vector = readFileSync(new URL("../log-shipper/vector.yaml", import.meta.url), "utf8"); test("generated allowlist emits known fixed messages and builds output from validated fields", () => { const result = renderAllowlist(schema); @@ -18,7 +19,8 @@ test("generated allowlist emits known fixed messages and builds output from vali assert.match(result, /if exists\(\.partial_message\) \{ abort \}/); assert.match(result, /if !is_string\(route\) \{ abort \}[\s\S]*if length\(route\) > 256 \{ abort \}/); assert.match(result, /if !includes\(\[.*\/healthz.*\], route\) \{ route = "" \}/); - assert.match(result, /length\(encode_json\(\.\)\) > 280/); + assert.doesNotMatch(result, /if length\(encode_json\(\.\)\) > 280/); + assert.match(result, /upstream_status_raw, r'\^\[1-5\]\[0-9\]\{2\}\(, \?\[1-5\]\[0-9\]\{2\}\| : \[1-5\]\[0-9\]\{2\}\)\*\$'/); assert.match(result, /event_code = "other"/); assert.match(result, /level = if event == "node_warning" \{ "warn" \} else \{ "error" \}/); }); @@ -117,3 +119,9 @@ test("missing Better Stack token leaves node and ingress startup independent", ( "allowed_envs sync must not gate compose startup", ); }); +test("WARN lane bypasses the ingress throttle and stays behind the size guard", () => { + assert.match(vector, /throttle_app_warning:\n type: throttle\n inputs: \[classify_by_budget\.app_warning\]\n window_secs: 60\n threshold: 5/); + assert.match(vector, /throttle_ingress:\n type: throttle\n inputs: \[request_output, classify_by_budget\.ingress_slow, slow_success\]\n window_secs: 60\n threshold: 27/); + assert.match(vector, /budget_size_guard:\n type: remap\n inputs: \[throttle_panic, throttle_app_error, throttle_app_warning, throttle_ingress\]/); + assert.match(vector, /sinks:\n betterstack:\n type: http\n inputs: \[budget_size_guard\]/); +}); From 7173923b86b457d0badeb3fbd2c2b05620bf2102 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 09:46:15 +0000 Subject: [PATCH 04/21] Correct Better Stack warning lane budget note --- docs/runbooks/tc-959-betterstack-log-shipping.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/runbooks/tc-959-betterstack-log-shipping.md b/docs/runbooks/tc-959-betterstack-log-shipping.md index 33f7a7fe..e8dbbe16 100644 --- a/docs/runbooks/tc-959-betterstack-log-shipping.md +++ b/docs/runbooks/tc-959-betterstack-log-shipping.md @@ -21,7 +21,7 @@ The allowlist excludes signed-KV tickets, authorization headers, tokens, transcr The node CVM is tdx.medium: 2 vCPU, 4 GiB RAM, 20 GiB disk. Vector is hard-capped at 64 MiB (1.56% of RAM) and 0.25 CPU (12.5% of 2 vCPU). Measured node Vector memory and CPU peaks are **PENDING** the reviewed Docker burst check; the ptx-batch observation of 16.83 MiB under a 7,000-event burst is a reference only, not a node measurement. The Linux/amd64 `ghcr.io/tinycloudlabs/tinycloud-node:dstack` registry manifest measured 17,115,015 compressed layer bytes across seven layers. The Docker check uses that registry image; no local build is needed for the small pull. -The conservative monthly calculation charges each cap independently: `2 + 5 + 5 + 27 = 39 events/minute`. Over 30 days: `39 × 60 × 24 × 30 × 281 = 473,428,800 bytes` (about 0.473 GB, 452 MiB), below the node's 0.48 GB allocation of the shared 3 GB free quota. Per-cap arithmetic: panic `2 × 60 × 24 × 30 × 281 = 24,278,400 bytes`; app ERROR and WARN each `5 × 60 × 24 × 30 × 281 = 60,696,000 bytes`; shared request lane `27 × 60 × 24 × 30 × 281 = 327,758,400 bytes`. WARN is counted in both its own 5/min cap and the 27/min request cap in this conservative estimate. +The conservative monthly calculation charges each cap independently: `2 + 5 + 5 + 27 = 39 events/minute`. Over 30 days: `39 × 60 × 24 × 30 × 281 = 473,428,800 bytes` (about 0.473 GB, 452 MiB), below the node's 0.48 GB allocation of the shared 3 GB free quota. Per-cap arithmetic: panic `2 × 60 × 24 × 30 × 281 = 24,278,400 bytes`; app ERROR and WARN each `5 × 60 × 24 × 30 × 281 = 60,696,000 bytes`; request lane `27 × 60 × 24 × 30 × 281 = 327,758,400 bytes`. The successful-counter route enum has 36 members (35 known templates plus ``). A route string longer than 256 characters is dropped before normalization; other unknown paths map to ``. Failure counts aggregate by route and status class. The transformed event-size guard and per-lane throttles cap emitted bytes; excess events are dropped in their own lane rather than borrowing another lane's quota. These limits bound node output, not other services' shared quota usage. From f5f51b9dafb17e70f3d500174ac6d34992f58f0f Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 15:35:58 +0000 Subject: [PATCH 05/21] TC-959: switch node log shipping to GELF --- .github/workflows/rust.yml | 8 +- CHANGELOG.md | 2 +- README.md | 17 +- docker-compose.dstack-postgres.yaml | 322 +++++++++++--- .../tc-959-betterstack-log-shipping.md | 35 +- log-shipper/fields.json | 8 +- log-shipper/vector.tests.yaml | 167 +++++++- log-shipper/vector.yaml | 256 ++++++++++- scripts/node-vector-schema.mjs | 86 +++- scripts/node-vector-schema.test.mjs | 65 ++- tc-959-node-docker-check.review.sh | 396 ++++++++++++++++++ 11 files changed, 1232 insertions(+), 130 deletions(-) create mode 100755 tc-959-node-docker-check.review.sh diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 132b34a0..089e4e78 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -206,10 +206,10 @@ jobs: mkdir -p "${RUNNER_TEMP}/vector" tar -xzf "${RUNNER_TEMP}/${archive}" -C "${RUNNER_TEMP}/vector" vector="${RUNNER_TEMP}/vector/vector-x86_64-unknown-linux-musl/bin/vector" - mkdir -m 700 -p /tmp/vector-secrets - trap 'rm -rf /tmp/vector-secrets' EXIT - printf '%s' 'dummy-vector-test-token' > /tmp/vector-secrets/bs_token - chmod 600 /tmp/vector-secrets/bs_token + sudo install -d -o "$(id -u)" -g "$(id -g)" -m 700 /var/lib/vector/secrets + trap 'sudo rm -rf /var/lib/vector' EXIT + printf '%s' 'dummy-vector-test-token' > /var/lib/vector/secrets/bs_token + chmod 600 /var/lib/vector/secrets/bs_token "${vector}" validate --no-environment log-shipper/vector.yaml "${vector}" test log-shipper/vector.yaml log-shipper/vector.tests.yaml - name: Verify generated capability artifacts match capabilities.json diff --git a/CHANGELOG.md b/CHANGELOG.md index c1561227..58bc67a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog ## [Unreleased] -- TC-959: add a Vector sidecar that forwards bounded JSON node warnings/errors and redacted ingress failures, slow requests, and per-minute successful-request counts to Better Stack. VRL starts from typed allowlisted fields and drops unrecognized or malformed records; the node remains independent of Vector availability. The shipper is pinned and resource-capped; the node peak is pending the reviewed Docker burst check. See `docs/runbooks/tc-959-betterstack-log-shipping.md`. +- TC-959: ship only labelled node/ingress logs through GELF/UDP to a fixed-address, isolated Vector socket; gate labels before metadata stripping, JSON parsing, and the existing allowlist. Vector publishes UDP on host loopback only, has no Docker-directory mount or socket, and runs with capped resources. Unlabelled services remain on capped `json-file` logs. Native tests cover transport, redaction, gates, and budgets; the review-only Docker acceptance script remains unrun. See `docs/runbooks/tc-959-betterstack-log-shipping.md`. - Security (TC-1132, TC-1080 Parts C–D): webhook registration and every delivery attempt share an egress policy covering special-use IPv4/IPv6, embedded IPv4 (mapped, compatible, NAT64 and 6to4), Teredo and site-local IPv6. Ambiguous local-use NAT64 is refused if any valid IPv4 interpretation is forbidden. Corrected public controls such as `192.2.1.1` and `198.51.0.1` are no longer over-blocked. Mixed DNS answers retain only permitted addresses. - Webhooks deny the resolved addresses of the configured network database, quota URL, space allowlist and explicit S3 endpoint on every port, including aliases and embedded IPv4 forms. The service deny-set refreshes every ten minutes; validation waits for an in-progress refresh, and failed/stale resolution closes egress until recovery. Temporary DNS/policy failures retry on the existing bounded schedule; invalid or forbidden callbacks dead-letter. Existing subscriptions whose destinations become forbidden dead-letter on the next attempt. diff --git a/README.md b/README.md index e68105c1..f90d456e 100644 --- a/README.md +++ b/README.md @@ -294,13 +294,6 @@ When `keys.type` is `Static` the instance will use an array of bytes as a static The secret MUST contain at least 32 bytes of entropy (either randomly generated or derived in a cryptographically secure way). It is STRONGLY RECOMMENDED that the secret be given via environment variables and NOT in the `tinycloud.toml` config file. Additionally it is STRONGLY RECOMMENDED that the secret be backed up in a secure place if used in production. Loss of the secret will result in total loss of function for the TinyCloud Protocol instance. -## Production log shipping - -The production Compose file sends redacted JSON records through Vector to Better Stack. A missing source token leaves only Vector idle; the node and ingress start independently and use asynchronous Fluentd logging. - -Node output is bounded to 280 JSON bytes plus one line separator. The caps are 2 panic/min, 5 app ERROR/min, 5 WARN/min, and 27/min through the ingress/request lane. WARN bypasses the ingress throttle and joins panic and ERROR directly at the final size guard. The conservative monthly ceiling charges all lane caps independently: `(2 + 5 + 5 + 27) × 60 × 24 × 30 × 281 = 473,428,800 bytes` (about 0.473 GB per 30 days), below the node's 0.48 GB allocation of the shared free-plan quota. Ingress 4xx/5xx counts aggregate by route template and status class; successful requests aggregate by route. There are 36 route values, including `` for unknown paths. -The Docker review check pulls the registry `dstack` image; its Linux/amd64 manifest measured 17,115,015 compressed bytes across seven layers. - ## Running TinyCloud Protocol instances can be started via command line, e.g.: @@ -311,6 +304,16 @@ TINYCLOUD_PORT=8001 tinycloud If the TinyCloud Protocol instance is not able to find or establish a connection to the configured storage, the instance will terminate. +### Production log shipping (TC-959) + +The node and ingress use the Docker GELF driver with exact `tc.logship` labels and send UDP to host loopback `127.0.0.1:12201`. Vector runs on its own fixed-address bridge network, publishes only that loopback UDP port, and has no Docker socket or host Docker-directory mount. Its first transform gates on `_tc.logship`; only then does it strip GELF metadata and parse the app's JSON line before the existing schema allowlist. Other services, including Postgres, stay unlabelled on capped `json-file` logs. Node and ingress do not depend on Vector; a missing Better Stack token leaves Vector idle. + +The separate panic, app ERROR, app WARN, and ingress lanes are capped at 2, 5, 5, and 27 events/minute. With the 281-byte final event limit, the monthly ceiling is `(2 + 5 + 5 + 27) × 60 × 24 × 30 × 281 = 473,428,800 bytes` (0.473 GB/30d), below the node's 0.48 GB share of the 3 GB free quota. Unknown routes collapse to ``. WARN and fixed gate counters bypass the ingress throttle. + +On the 2-vCPU/4-GiB/20-GB tdx.medium CVM, Vector is capped at 64 MiB and 0.25 CPU. Its data directory and token file live on an 8 MiB tmpfs; the Vector and ssh-gateway `json-file` logs are capped at `10m × 3` each (60 MB total). The review-only Docker acceptance script measures Vector peak memory and transport loss; neither has been measured by this change. + +The production token is GitHub `production` environment secret `PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN`, written to the sealed environment as `BETTERSTACK_SOURCE_TOKEN`. The entrypoint copies it to a mode-0600 `SECRET[]` file on Vector's tmpfs and unsets the variable before executing Vector. Docker daemon/API principals can still inspect container configuration; restrict access. At go-live, create the `node_fatal_panic` alert and a per-service source-silence alert. See [`docs/runbooks/tc-959-betterstack-log-shipping.md`](docs/runbooks/tc-959-betterstack-log-shipping.md) for redaction tests, privilege boundaries, retention, and release checks. + ## Usage TinyCloud Protocol is most easily used via the TinyCloud Protocol SDK. See the example DApps and tutorials for detailed information. diff --git a/docker-compose.dstack-postgres.yaml b/docker-compose.dstack-postgres.yaml index d09df2ae..f0a51eee 100644 --- a/docker-compose.dstack-postgres.yaml +++ b/docker-compose.dstack-postgres.yaml @@ -12,18 +12,15 @@ services: # ghcr.io/tinycloudlabs/tinycloud-node:1.4.0-dstack-duckdb image: ghcr.io/tinycloudlabs/tinycloud-node:dstack restart: unless-stopped - # TC-959: asynchronously forward redacted JSON records to the local Vector - # receiver. The queue is bounded in events; Docker's daemon remains - # independent of Vector availability. + # TC-959: the GELF driver emits the exact label to Vector's UDP port on the + # host loopback; it never blocks app startup when Vector is unavailable. + labels: + tc.logship: "tinycloud-node" logging: - driver: fluentd + driver: gelf options: - fluentd-address: "127.0.0.1:24224" - fluentd-async: "true" - fluentd-buffer-limit: "1000" - tag: "tinycloud.node" - cache-max-size: "20m" - cache-max-file: "5" + gelf-address: udp://127.0.0.1:12201 + labels: "tc.logship" ports: - "8000:8000" volumes: @@ -110,18 +107,14 @@ services: - /var/run/dstack.sock:/var/run/dstack.sock - cert-data:/etc/letsencrypt restart: unless-stopped - # TC-959: only sanitized ingress JSON is forwarded; raw access-log paths - # are not forwarded because the ingress emits route templates. + # TC-959: only this service is labelled and shipped over loopback GELF. + labels: + tc.logship: "tinycloud-ingress" logging: - driver: fluentd + driver: gelf options: - fluentd-address: "127.0.0.1:24224" - fluentd-async: "true" - fluentd-buffer-limit: "1000" - tag: "tinycloud.ingress" - cache-max-size: "20m" - cache-max-file: "5" - + gelf-address: udp://127.0.0.1:12201 + labels: "tc.logship" # The dstack gateway's TLS-terminating routes (-.:443) # only forward to compose-published ports, so guest sshd on :22 is # unreachable via `phala ssh` even with authorized_keys installed. @@ -142,28 +135,242 @@ services: options: max-size: "10m" max-file: "3" - vector: image: timberio/vector:0.59.0-alpine@sha256:0e36bc2a290039e9b6a5e92f04bd2f133223506f72d09682c1df16c015172b7a restart: unless-stopped + user: "10001:10001" + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] mem_limit: 64m memswap_limit: 64m cpus: 0.25 ports: - - "127.0.0.1:24224:24224" + - "127.0.0.1:12201:12201/udp" + networks: + logship: + ipv4_address: 10.255.253.2 + tmpfs: + - /var/lib/vector:rw,noexec,nosuid,size=8m,uid=10001,gid=10001,mode=0700 environment: BETTERSTACK_SOURCE_TOKEN: "${BETTERSTACK_SOURCE_TOKEN:-}" BETTERSTACK_SOURCE_HOST: "${BETTERSTACK_SOURCE_HOST:-s2795437.us-west-2a.betterstackdata.com}" VECTOR_CONFIG_YAML: | data_dir: /var/lib/vector secret: - bs: { type: directory, path: /tmp/vector-secrets, remove_trailing_whitespace: true } + bs: { type: directory, path: /var/lib/vector/secrets, remove_trailing_whitespace: true } sources: - docker_fluent: { type: fluent, mode: tcp, address: 0.0.0.0:24224 } + gelf_in: + type: socket + mode: udp + address: 0.0.0.0:12201 + max_length: 262144 + framing: { method: chunked_gelf, chunked_gelf: { decompression: Auto } } + decoding: { codec: gelf } transforms: + logship_gate: + type: remap + inputs: [gelf_in] + drop_on_error: true + drop_on_abort: true + source: | + lbl = string(get(., ["_tc.logship"]) ?? null) ?? "" + if lbl == "tinycloud-node" || lbl == "tinycloud-ingress" { + .gate_action = "candidate" + .gate_label = lbl + } else if lbl == "" { + . = {"gate_action": "drop", "gate_reason": "unlabelled"} + } else { + . = {"gate_action": "unexpected", "gate_reason": "unexpected_label"} + } + logship_gate_route: + type: route + inputs: [logship_gate] + route: + node: .gate_action == "candidate" && .gate_label == "tinycloud-node" + ingress: .gate_action == "candidate" && .gate_label == "tinycloud-ingress" + rejected: .gate_action == "drop" || .gate_action == "unexpected" + strip_gelf_fields: + type: remap + inputs: [logship_gate_route.node, logship_gate_route.ingress] + drop_on_error: true + drop_on_abort: true + source: | + label = .gate_label + raw = get(., ["message"]) ?? null + if !is_string(raw) { + . = {"gate_action": "drop", "gate_reason": "malformed_envelope"} + } else { + tag = if label == "tinycloud-node" { "tinycloud.node" } else { "tinycloud.ingress" } + . = {"gate_action": "candidate", "gate_label": label, "tag": tag, "message": raw} + } + parse_gelf_message: + type: remap + inputs: [strip_gelf_fields] + drop_on_error: true + drop_on_abort: true + source: | + if .gate_action == "drop" { + reason = .gate_reason + . = {"gate_action": "drop", "gate_reason": reason} + } else { + parsed, parse_err = parse_json(.message) + if parse_err != null || !is_object(parsed) { + . = {"gate_action": "drop", "gate_reason": "malformed_envelope"} + } else { + tag = .tag + raw = .message + . = {"tag": tag, "log": raw, "parsed_log": parsed} + } + } + logship_inner: + type: remap + inputs: [parse_gelf_message] + drop_on_error: true + drop_on_abort: true + source: | + if exists(.gate_action) { + action = .gate_action + reason = .gate_reason + . = {"gate_action": action, "gate_reason": reason} + } else if !is_string(.log) { + . = {"gate_action": "drop", "gate_reason": "malformed_envelope"} + } else { + raw = .log + parsed = .parsed_log + if !is_object(parsed) || exists(parsed.partial_message) { + . = {"gate_action": "drop", "gate_reason": "malformed_envelope"} + } else if .tag == "tinycloud.node" { + fields = get(parsed, ["fields"]) ?? null + message = if is_object(fields) { get(fields, ["message"]) ?? null } else { null } + if !is_string(message) { + . = {"gate_action": "drop", "gate_reason": "malformed_envelope"} + } else { + message = to_string(message) ?? "" + known_message = includes([ + "invocation replay insert failed", + "invocation replay cleanup failed", + "share-v2 local-tee feature active: deriving a deterministic, non-attested TEE identity from node key material", + "retention: delivered hook delivery prune failed", + "retention: dead-letter hook delivery prune failed", + "retention: signed KV ticket prune failed", + "failed to load tunnel state; tunnel not started", + "invalid loopback API address; tunnel not started", + "failed to load link state; LAN listener not started", + "failed to bind link LAN listener; LAN listener not started", + "failed to build TLS config; LAN listener not started", + "link LAN listener exited with error", + "failed to record link listener bind state", + "failed to record link listener bind failure", + "failed to load config for auto-renew", + "link auto-renew failed", + "failed to hot-reload renewed link cert", + "failed to reload TLS material after link auto-renew", + "link disable: service delete failed; local state removed anyway", + "link connection error", + "link listener accept error", + "webhook delivery attempt failed", + "webhook dispatcher loop failed", + "addressed delegation authorization failed", + "native share authorization denied", + "native share KV put failed", + "share v2 readiness probe failed", + "share v2 is enabled but shareEmail.invitationPublicKey is unset", + "configured share invitation key does not match the node's derived signing key", + "share/v2 policy challenge denied", + "failed to read committed hook events", + "failed to load kv write hook rows", + "failed to load kv delete hook rows", + "missing committed kv hook row for invocation", + "database write committed but webhook enqueue failed", + "tunnel: failed to prepare connection attempt", + "tunnel: no frame within idle timeout; treating connection as dead", + "tunnel: read error", + "tunnel: dropping malformed frame", + "tunnel: relay sent an error frame", + "tunnel: failed to write frame", + "tunnel: failed to record runtime state", + "tunnel: failed to clear runtime state after disable", + "tunnel authentication rejected", + "node fatal panic" + ], message) + if !known_message { + known_message = starts_with(message, "dstack socket available but get_info failed: ") || + starts_with(message, "quota service returned an invalid body: ") || + starts_with(message, "quota service returned ") || + starts_with(message, "Failed to get TDX quote: ") + } + if !known_message { + . = {"gate_action": "drop", "gate_reason": "unknown_message"} + } else { + . = {"tag": .tag, "log": raw, "parsed_log": parsed} + } + } + } else if .tag == "tinycloud.ingress" { + . = {"tag": .tag, "log": raw, "parsed_log": parsed} + } else { + . = {"gate_action": "drop", "gate_reason": "unexpected_label"} + } + } + logship_result_route: + type: route + inputs: [logship_gate_route.rejected, logship_inner] + route: + dropped: .gate_action == "drop" + unexpected: .gate_action == "unexpected" + validated: is_string(.tag) && is_string(.log) + logship_drop_counter: + type: remap + inputs: [logship_result_route.dropped] + drop_on_error: true + drop_on_abort: true + source: | + reason = .gate_reason + if !includes(["malformed_envelope", "unknown_message", "unlabelled"], reason) { abort } + minute = format_timestamp!(now(), format: "%Y-%m-%dT%H:%M") + . = {"event": "logship_gate_dropped_total", "reason": reason, "minute": minute, "count": 1} + logship_unexpected_counter: + type: remap + inputs: [logship_result_route.unexpected] + drop_on_error: true + drop_on_abort: true + source: | + minute = format_timestamp!(now(), format: "%Y-%m-%dT%H:%M") + . = {"event": "logship_unexpected_label", "reason": "unexpected_label", "minute": minute, "count": 1} + logship_counter_reduce: + type: reduce + inputs: [logship_drop_counter, logship_unexpected_counter] + group_by: [event, reason, minute] + expire_after_ms: 75000 + end_every_period_ms: 60000 + flush_period_ms: 1000 + merge_strategies: + count: sum + logship_counter_output: + type: remap + inputs: [logship_counter_reduce] + drop_on_error: true + drop_on_abort: true + source: | + event = .event + reason = .reason + count = .count + if !includes(["logship_gate_dropped_total", "logship_unexpected_label"], event) { abort } + if !includes(["malformed_envelope", "unexpected_label", "unknown_message", "unlabelled"], reason) { abort } + if !is_integer(count) { abort } + count = to_int(count) ?? 0 + if count < 1 || count > 1000000000 { abort } + . = { + "level": "warn", + "ts": format_timestamp!(now(), format: "%Y-%m-%dT%H:%M:%S%.3fZ"), + "event": event, + "svc": "logship", + "reason": reason, + "count": count + } allowlist: type: remap - inputs: [docker_fluent] + inputs: [logship_result_route.validated] drop_on_abort: true drop_on_error: true source: | @@ -171,30 +378,37 @@ services: if exists(.partial_message) { abort } if !is_string(.tag) { abort } tag = .tag - if !is_string(.log) { abort } - j, parse_err = parse_json(.log) - if parse_err != null || !is_object(j) { abort } + j = null + if exists(.parsed_log) { + j = .parsed_log + if !is_object(j) { abort } + } else { + if !is_string(.log) { abort } + j, parse_err = parse_json(.log) + if parse_err != null || !is_object(j) { abort } + } out = {} if tag == "tinycloud.node" { fields = get(j, ["fields"]) ?? null if !is_object(fields) { abort } level = get(j, ["level"]) ?? null if !is_string(level) { abort } - level = downcase!(level) + level = downcase(level) ?? "" if !includes(["warn", "error"], level) { abort } msg = get(fields, ["message"]) ?? null if !is_string(msg) { abort } + msg = to_string(msg) ?? "" event = null event_code = null if msg == "invocation replay insert failed" { event = "node_warning"; event_code = "invocation_replay_insert_failed" } else if msg == "invocation replay cleanup failed" { event = "node_warning"; event_code = "invocation_replay_cleanup_failed" } - else if starts_with!(msg, "dstack socket available but get_info failed: ") { event = "node_warning"; event_code = "dstack_get_info_failed" } + else if starts_with(msg, "dstack socket available but get_info failed: ") { event = "node_warning"; event_code = "dstack_get_info_failed" } else if msg == "share-v2 local-tee feature active: deriving a deterministic, non-attested TEE identity from node key material" { event = "node_warning"; event_code = "share_local_tee_identity" } else if msg == "retention: delivered hook delivery prune failed" { event = "node_warning"; event_code = "retention_hook_delivery_prune_failed" } else if msg == "retention: dead-letter hook delivery prune failed" { event = "node_warning"; event_code = "retention_dead_letter_prune_failed" } else if msg == "retention: signed KV ticket prune failed" { event = "node_warning"; event_code = "retention_signed_ticket_prune_failed" } - else if starts_with!(msg, "quota service returned an invalid body: ") { event = "node_warning"; event_code = "quota_invalid_body" } - else if starts_with!(msg, "quota service returned ") { event = "node_warning"; event_code = "quota_fallback" } + else if starts_with(msg, "quota service returned an invalid body: ") { event = "node_warning"; event_code = "quota_invalid_body" } + else if starts_with(msg, "quota service returned ") { event = "node_warning"; event_code = "quota_fallback" } else if msg == "failed to load tunnel state; tunnel not started" { event = "node_warning"; event_code = "tunnel_state_load_failed" } else if msg == "invalid loopback API address; tunnel not started" { event = "node_warning"; event_code = "tunnel_address_invalid" } else if msg == "failed to load link state; LAN listener not started" { event = "node_warning"; event_code = "link_state_load_failed" } @@ -219,7 +433,7 @@ services: else if msg == "share v2 is enabled but shareEmail.invitationPublicKey is unset" { event = "node_error"; event_code = "share_invitation_key_missing" } else if msg == "configured share invitation key does not match the node's derived signing key" { event = "node_error"; event_code = "share_invitation_key_mismatch" } else if msg == "share/v2 policy challenge denied" { event = "node_warning"; event_code = "share_policy_denied" } - else if starts_with!(msg, "Failed to get TDX quote: ") { event = "node_error"; event_code = "attestation_quote_failed" } + else if starts_with(msg, "Failed to get TDX quote: ") { event = "node_error"; event_code = "attestation_quote_failed" } else if msg == "failed to read committed hook events" { event = "node_warning"; event_code = "hook_events_read_failed" } else if msg == "failed to load kv write hook rows" { event = "node_warning"; event_code = "hook_rows_load_failed" } else if msg == "failed to load kv delete hook rows" { event = "node_warning"; event_code = "hook_rows_load_failed" } @@ -257,46 +471,43 @@ services: out.ts = format_timestamp!(parsed_ts, format: "%Y-%m-%dT%H:%M:%S%.3fZ") route = get(j, ["route"]) ?? null if !is_string(route) { abort } - route = to_string!(route) + route = to_string(route) ?? "" if length(route) > 256 { abort } if !includes(["/healthz","/info","/version","/invoke","/delegate","/revoke","/attestation","/signed/kv","/signed/kv/","/admin/quota","/admin/quota/","/admin/usage","/hooks/events","/hooks/tickets","/hooks/webhooks","/hooks/webhooks/","/delegation/query","/delegation/status","/encryption/networks","/encryption/networks/","/encryption/networks//revoke","/encryption/networks//decrypt","/policy/v3/status","/policy/v3/status/","/policy/v3/policies","/policy/v3/challenges","/policy/v3/delegations","/policy/v3/enforcer-bindings","/policy/v3/deliveries/authorize","/.well-known/tinycloud/node-keys","/.well-known/encryption/network/","/public//kv","/public//kv/","/peer/generate/","/evidences/"], route) { route = "" } if !is_string(route) { abort } - route = to_string!(route) if !includes(["/healthz","/info","/version","/invoke","/delegate","/revoke","/attestation","/signed/kv","/signed/kv/","/admin/quota","/admin/quota/","/admin/usage","/hooks/events","/hooks/tickets","/hooks/webhooks","/hooks/webhooks/","/delegation/query","/delegation/status","/encryption/networks","/encryption/networks/","/encryption/networks//revoke","/encryption/networks//decrypt","/policy/v3/status","/policy/v3/status/","/policy/v3/policies","/policy/v3/challenges","/policy/v3/delegations","/policy/v3/enforcer-bindings","/policy/v3/deliveries/authorize","/.well-known/tinycloud/node-keys","/.well-known/encryption/network/","/public//kv","/public//kv/","/peer/generate/","/evidences/",""], route) { abort } out.route = route method = get(j, ["method"]) ?? null if !is_string(method) { abort } - method = to_string!(method) if !includes(["GET","HEAD","POST","PUT","PATCH","DELETE","OPTIONS",""], method) { abort } out.method = method status = get(j, ["status"]) ?? null if !is_integer(status) { abort } - status = to_int!(status) + status = to_int(status) ?? 0 if status < 100 || status > 599 { abort } out.status = status bytes = get(j, ["bytes"]) ?? null if !is_integer(bytes) { abort } - bytes = to_int!(bytes) + bytes = to_int(bytes) ?? 0 if bytes < 0 || bytes > 1000000000 { abort } out.bytes_sent = bytes upstream_status_raw = "-" if exists(j.upstream_status) { upstream_status_raw = j.upstream_status } if !is_string(upstream_status_raw) { abort } - upstream_status_raw = to_string!(upstream_status_raw) + upstream_status_raw = to_string(upstream_status_raw) ?? "" if length(upstream_status_raw) > 63 { abort } upstream_status = "" if upstream_status_raw != "-" && upstream_status_raw != "" { - if !match!(upstream_status_raw, r'^[1-5][0-9]{2}(, ?[1-5][0-9]{2}| : [1-5][0-9]{2})*$$') { abort } + if !match(upstream_status_raw, r'^[1-5][0-9]{2}(, ?[1-5][0-9]{2}| : [1-5][0-9]{2})*$$') { abort } upstream_status = slice!(upstream_status_raw, 0, 3) } if !is_string(upstream_status) { abort } - upstream_status = to_string!(upstream_status) if length(upstream_status) > 3 { abort } - if !match!(upstream_status, r'^(|[1-5][0-9]{2})$$') { abort } + if !match(upstream_status, r'^(|[1-5][0-9]{2})$$') { abort } out.upstream_status = upstream_status request_time = get(j, ["request_time"]) ?? null if !(is_integer(request_time) || is_float(request_time)) { abort } - request_time = to_float!(request_time) + request_time = to_float(request_time) ?? 0.0 if request_time < 0 || request_time > 86400 { abort } out.request_time_seconds = request_time status_class = if status >= 500 { "5xx" } else if status >= 400 { "4xx" } else if status >= 300 { "3xx" } else if status >= 200 { "2xx" } else { "1xx" } @@ -310,11 +521,11 @@ services: # END GENERATED FROM fields.json classify_by_budget: type: route - inputs: [allowlist] + inputs: [allowlist, logship_counter_output] route: panic: .event == "node_fatal_panic" app_error: .event == "node_error" - app_warning: .event == "node_warning" + app_warning: .event == "node_warning" || .event == "logship_gate_dropped_total" || .event == "logship_unexpected_label" ingress_summary: .event == "ingress_failure_count" || .event == "ingress_success_count" ingress_slow: .event == "ingress_request_slow" split_requests: @@ -433,23 +644,32 @@ services: - -ec - | umask 077 - if [ -z "$$BETTERSTACK_SOURCE_TOKEN" ]; then + mkdir -p /var/lib/vector/secrets + if [ -z "$${BETTERSTACK_SOURCE_TOKEN}" ]; then + rm -f /var/lib/vector/secrets/bs_token printf '%s\n' 'Better Stack token is absent; Vector is idle' while :; do sleep 3600; done fi - mkdir -p /tmp/vector-secrets - case "$$BETTERSTACK_SOURCE_HOST" in + case "$${BETTERSTACK_SOURCE_HOST}" in ''|*[!A-Za-z0-9.-]*) printf '%s\n' 'invalid Better Stack host' >&2; exit 1 ;; esac - printf '%s' "$$BETTERSTACK_SOURCE_TOKEN" > /tmp/vector-secrets/bs_token - printf '%s' "$$VECTOR_CONFIG_YAML" | sed "s/betterstack-placeholder.invalid/$$BETTERSTACK_SOURCE_HOST/g" > /etc/vector/vector.yaml + printf '%s' "$${BETTERSTACK_SOURCE_TOKEN}" > /var/lib/vector/secrets/bs_token + printf '%s' "$${VECTOR_CONFIG_YAML}" | sed "s/betterstack-placeholder.invalid/$${BETTERSTACK_SOURCE_HOST}/g" > /var/lib/vector/runtime.yaml unset BETTERSTACK_SOURCE_TOKEN BETTERSTACK_SOURCE_HOST VECTOR_CONFIG_YAML - exec vector --config /etc/vector/vector.yaml + exec vector --config /var/lib/vector/runtime.yaml logging: driver: json-file options: max-size: "10m" max-file: "3" + volumes: cert-data: + +networks: + logship: + driver: bridge + ipam: + config: + - subnet: 10.255.253.0/28 diff --git a/docs/runbooks/tc-959-betterstack-log-shipping.md b/docs/runbooks/tc-959-betterstack-log-shipping.md index e8dbbe16..c1ee2188 100644 --- a/docs/runbooks/tc-959-betterstack-log-shipping.md +++ b/docs/runbooks/tc-959-betterstack-log-shipping.md @@ -1,36 +1,37 @@ # TC-959: tinycloud-node Better Stack log shipping -## Phase 1 data path +## Transport and gate -The node app emits JSON tracing records (`TINYCLOUD_LOG__FORMAT=Json`). The ingress emits its TC-970 redacted access records. Docker forwards only these two services to the local Vector Fluent source using the asynchronous Fluentd driver. `log-shipper/vector.yaml` is the source config; the generator embeds an escaped copy into Compose and CI checks parity. +The node and `dstack-ingress` use Docker's `gelf` logging driver with the exact service labels `tc.logship=tinycloud-node` and `tc.logship=tinycloud-ingress`. Both send UDP to `127.0.0.1:12201`. Vector listens inside its own bridge network at fixed address `10.255.253.2`; that network is not shared with app or Postgres services and has egress for Better Stack. The only published Vector port is `127.0.0.1:12201:12201/udp`. Vector has no `docker.sock` or `/var/lib/docker` mount. There is no DockerRootDir pre-deploy check. -The VRL remap starts from an empty object. Node records require a known fixed message; variable error text is matched only by a fixed prefix and never copied. Unknown messages, non-JSON, non-object JSON, malformed typed fields, and Docker partial records are dropped and increment the Vector remap component's internal discarded-event counters. Those counters are not exposed through an API or metrics sink. Ingress routes, methods, status, duration, response bytes, and upstream status are validated against `log-shipper/fields.json`; only route templates are emitted. Raw paths, query strings, request headers, error text, and arbitrary nested values are not forwarded. Remap failures are dropped. +`logship_gate` is the first transform. It reads `_tc.logship` as a string and accepts only the two exact labels. Missing/empty labels increment the fixed `logship_gate_dropped_total{reason="unlabelled"}` counter without content; every other label emits one fixed `logship_unexpected_label` event, throttled to about one per minute. After the gate, `strip_gelf_fields` reconstructs the event from only the approved label and GELF `message`, dropping all underscore-prefixed fields and dockerd metadata. `parse_gelf_message` then parses `.message`, the app's JSON line. The existing inner allowlist, schema projection, event enum, priority lanes, throttle, `budget_size_guard`, and sole sink input remain unchanged. Ingress non-JSON stderr is dropped after the gate. -Ingress failures are counted per minute by `(route, status_class)` and successful requests by route; reducer input is projected to `(route, status_class, event, minute, request_count)` before aggregation. Slow requests remain bounded warning events; a slow 2xx is both counted and marked slow. Panic, node ERROR, and node WARN have dedicated 2/min, 5/min, and 5/min lanes. WARN bypasses the ingress throttle and feeds the final size guard directly; the separate 27/min ingress throttle covers request aggregates and slow requests only. The sink retains at most 1,000 events in memory and drops newest when full. There is no Vector API or exposed metrics port; Fluent input is published only on `127.0.0.1:24224`. +Docker splits lines over 16 KiB into GELF fragments. An incomplete fragment does not parse as the app JSON line and increments the bounded malformed-envelope counter; no partial text is forwarded. The GELF socket source accepts chunked and gzip-compressed messages, with no checkpoint or replay state. -The ingest hostname is configured once as `BETTERSTACK_SOURCE_HOST` in the Vector service environment; the entrypoint validates it and substitutes it into the generated config's placeholder before Vector starts. The source token is the GitHub `production` environment secret `PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN`; deploy writes it into the sealed environment as `BETTERSTACK_SOURCE_TOKEN` and synchronizes `allowed_envs` after compose startup. An absent token leaves only Vector idle; the node and ingress have no dependency on it. Because the token is present in Vector's container environment, `docker inspect ` exposes it to principals with Docker daemon/API access, even though Vector also writes it to a runtime secret file. Restrict Docker access accordingly. No token value belongs in this repository or logs. +The node and ingress do not depend on Vector. Their GELF logging driver is UDP and does not require Vector to start. A missing Better Stack token leaves Vector idle. Postgres and every non-shipped service remain unlabelled and use capped `json-file` logs. -At go-live, create a Better Stack alert for `event = node_fatal_panic`. +## Redaction and alerting -## Redaction boundary +The allowlist excludes signed-KV tickets, authorization headers, tokens, transcript text, SQL values, meeting URLs, X-Forwarded-For and CF-Connecting-IP values, IPv4/IPv6 addresses, raw paths/IDs, UCAN/SIWE/delegation strings, owner DIDs or wallet addresses, space IDs, e-mail addresses, nested objects, arrays, and all unknown keys. Unknown ingress paths collapse to `` only after the raw route string passes its 256-character bound. The panic hook emits only a fixed JSON message and does not serialize panic text or source location; input level cannot forge the panic alert. See `log-shipper/vector.tests.yaml` for forbidden-value, malformed-value, priority-lane flood, route-template, multi-upstream, fixed-panic, GELF gate, metadata-stripping, and counter tests. -The allowlist excludes signed-KV tickets, authorization headers, tokens, transcript text, SQL values, meeting URLs, X-Forwarded-For and CF-Connecting-IP values, IPv4/IPv6 addresses, raw paths/IDs, UCAN/SIWE/delegation strings, owner DIDs or wallet addresses, space IDs, e-mail addresses, nested objects, arrays, and all unknown keys. Unknown ingress paths collapse to `` only after the raw route string passes its 256-character bound. The panic hook emits only a fixed JSON message and does not serialize panic text or source location; a WARN cannot forge that alert. See `log-shipper/vector.tests.yaml` for forbidden-value, malformed-value, priority-lane flood, route-template, multi-upstream, and fixed-panic cases. +At go-live, create a Better Stack alert for `event = node_fatal_panic` and one source-silence alert per service, within free-plan limits. The lead's GELF/UDP spike passed restart/recreate recovery, 500-line outage-burst recovery, sibling forgery, and external-interface isolation. It also observed no app blocking. The full node-specific Docker acceptance script is review-only and has not been run here. -## Resource and volume budget +## Priority and monthly volume budget -The node CVM is tdx.medium: 2 vCPU, 4 GiB RAM, 20 GiB disk. Vector is hard-capped at 64 MiB (1.56% of RAM) and 0.25 CPU (12.5% of 2 vCPU). Measured node Vector memory and CPU peaks are **PENDING** the reviewed Docker burst check; the ptx-batch observation of 16.83 MiB under a 7,000-event burst is a reference only, not a node measurement. -The Linux/amd64 `ghcr.io/tinycloudlabs/tinycloud-node:dstack` registry manifest measured 17,115,015 compressed layer bytes across seven layers. The Docker check uses that registry image; no local build is needed for the small pull. +The panic lane allows 2 events/minute; node app ERROR and WARN each allow 5/minute; ingress allows 27/minute. WARN and fixed gate counters pass through their own 5/minute warning lane, then feed `budget_size_guard` directly without entering the ingress throttle. The ingress lane carries per-minute `(route, status_class)` failure counts, per-minute route-only success counts, and slow requests. A 2xx/slow request is counted and marked slow; route cardinality is bounded to 35 known templates plus ``. -The conservative monthly calculation charges each cap independently: `2 + 5 + 5 + 27 = 39 events/minute`. Over 30 days: `39 × 60 × 24 × 30 × 281 = 473,428,800 bytes` (about 0.473 GB, 452 MiB), below the node's 0.48 GB allocation of the shared 3 GB free quota. Per-cap arithmetic: panic `2 × 60 × 24 × 30 × 281 = 24,278,400 bytes`; app ERROR and WARN each `5 × 60 × 24 × 30 × 281 = 60,696,000 bytes`; request lane `27 × 60 × 24 × 30 × 281 = 327,758,400 bytes`. +At the 281-byte final event-size limit, the worst case is `(2 + 5 + 5 + 27) × 60 × 24 × 30 × 281 = 473,428,800 bytes` (0.473 GB/30d), within the node's 0.48 GB allocation of the shared 3 GB free quota. The individual lane ceilings are 24,278,400 bytes for panic, 60,696,000 bytes each for app ERROR and WARN, and 327,758,400 bytes for ingress. These are hard ceilings assuming every allowed event reaches the sink; shared quota use by other sources is separate. -The successful-counter route enum has 36 members (35 known templates plus ``). A route string longer than 256 characters is dropped before normalization; other unknown paths map to ``. Failure counts aggregate by route and status class. The transformed event-size guard and per-lane throttles cap emitted bytes; excess events are dropped in their own lane rather than borrowing another lane's quota. These limits bound node output, not other services' shared quota usage. +## CVM resource and disk budget -Each of the two Fluentd producers has a 1,000-event queue. The planning assumption of a 16 KiB raw line plus 1 KiB event/driver overhead gives `17 KiB × 1,000 × 2 = 34,816,000 bytes` (33.2 MiB). Adding Vector's 64 MiB cap gives 97.2 MiB (2.37% of 4 GiB). Docker does not enforce the assumed 16 KiB raw-line ceiling, so this is not a guaranteed producer-queue byte bound. The explicit Compose `mem_limit` sum is 64 MiB (Vector only); the node, ingress, and ssh helper have no Compose memory caps, so this is not a bounded whole-guest memory total. Do not treat remaining guest memory as guaranteed headroom. +The CVM is tdx.medium: 2 vCPU, 4 GiB RAM, 20 GB disk. Vector runs as UID 10001 with 64 MiB memory/swap and 0.25 CPU limits, read-only root, all capabilities dropped, and `no-new-privileges`. Its `data_dir` is an 8 MiB tmpfs; the token file is mode 0600 under that same tmpfs. There is no volume initializer, Docker log-tree mount, or persistent checkpoint volume. Vector and ssh-gateway each have capped `json-file` logs (`10m × 3`), a 60 MB combined retention ceiling. Node and ingress logs travel through GELF rather than a local log file. The review-only acceptance harness measures Vector peak memory and reports 5,000-line burst loss; neither value has been measured by this change. -## Local log retention exceptions +## Secrets and local-log exceptions -The node app and ingress use the asynchronous Fluentd driver with a 1,000-event limit and `20m × 5` cache settings each. The Vector sidecar and ssh gateway retain capped `json-file` logs (`10m × 3`) and are not sent to themselves; this prevents a feedback loop and keeps helper logs local. Postgres is managed outside this Compose file, so there is no database container log stream to ship. No service depends on Vector for startup or request handling. +The ingest host is configured once as `BETTERSTACK_SOURCE_HOST` and substituted into the sink URL. The source token is GitHub `production` environment secret `PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN`, written to the sealed environment as `BETTERSTACK_SOURCE_TOKEN`. The entrypoint writes `SECRET[bs.bs_token]` into `/var/lib/vector/secrets/bs_token` on Vector's tmpfs, with a restrictive umask, then unsets the token and config environment variables before executing Vector. The token is not committed or logged. Docker daemon/API principals can inspect the container's configured environment; restrict daemon/API access. + +Only node and ingress logs carry `tc.logship`. Vector and ssh-gateway retain capped, unlabelled `json-file` logs and are not shipped, preventing self-ingestion. Postgres is not mounted into Vector. ## Verification and deploy gate -Non-Docker CI checks the canonical common + `services.tinycloud-node` schema, generated VRL, generated Compose parity, and generator tests. The single `/tmp/tc-959-node-docker-check.sh` review script runs the pinned Vector VRL suite, Compose render parity, node-serving checks with Vector stopped/restarted, and a Vector memory burst; it is intentionally not run by this change. Production deployment remains Amos-owned. Compose hash, app-id, KMS attestation, and `allowed_envs` must be checked by the release gate; the node Vector peak remains pending until that window. +CI checks the canonical common + `services.tinycloud-node` schema, generated VRL, byte-identical embedded Compose config, packaging restrictions (`configs.*.file`, `env_file`, and relative bind sources), and native Vector 0.59.0 validation/tests after verifying the release tarball SHA256. The review-only `tc-959-node-docker-check.review.sh` covers 5× each Vector restart, crash/restart-policy, and recreate; 3× Vector-down 500-line bursts; app service continuity; source-metric isolation; external-interface probes; and Vector memory. It includes a test-only node override with a local Postgres, a Static test key, TEE Off, and no dstack socket. It must only run after explicit Docker authorization. Amos owns production deployment and go-live alerts; do not use this runbook to deploy or change secrets. diff --git a/log-shipper/fields.json b/log-shipper/fields.json index 94759583..364c087c 100644 --- a/log-shipper/fields.json +++ b/log-shipper/fields.json @@ -69,10 +69,12 @@ "node_error", "ingress_failure_count", "ingress_request_slow", - "ingress_success_count" + "ingress_success_count", + "logship_gate_dropped_total", + "logship_unexpected_label" ], "fields": { - "svc": { "type": "enum", "values": ["api", "ingress"] }, + "svc": { "type": "enum", "values": ["api", "ingress", "logship"] }, "route": { "type": "enum", "values": ["/healthz", "/info", "/version", "/invoke", "/delegate", "/revoke", "/attestation", "/signed/kv", "/signed/kv/", "/admin/quota", "/admin/quota/", "/admin/usage", "/hooks/events", "/hooks/tickets", "/hooks/webhooks", "/hooks/webhooks/", "/delegation/query", "/delegation/status", "/encryption/networks", "/encryption/networks/", "/encryption/networks//revoke", "/encryption/networks//decrypt", "/policy/v3/status", "/policy/v3/status/", "/policy/v3/policies", "/policy/v3/challenges", "/policy/v3/delegations", "/policy/v3/enforcer-bindings", "/policy/v3/deliveries/authorize", "/.well-known/tinycloud/node-keys", "/.well-known/encryption/network/", "/public//kv", "/public//kv/", "/peer/generate/", "/evidences/", ""] }, "method": { "type": "enum", "values": ["GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS", ""] }, "status": { "type": "integer", "min": 100, "max": 599 }, @@ -81,6 +83,8 @@ "request_count": { "type": "integer", "min": 1, "max": 1000000 }, "bytes_sent": { "type": "integer", "min": 0, "max": 1000000000 }, "upstream_status": { "type": "string", "regex": "^(|[1-5][0-9]{2})$", "max_len": 3 }, + "count": { "type": "integer", "min": 1, "max": 1000000000 }, + "reason": { "type": "enum", "values": ["malformed_envelope", "unexpected_label", "unknown_message", "unlabelled"] }, "event_code": { "type": "enum", "values": ["invocation_replay_insert_failed", "invocation_replay_cleanup_failed", "dstack_get_info_failed", "share_local_tee_identity", "retention_hook_delivery_prune_failed", "retention_dead_letter_prune_failed", "retention_signed_ticket_prune_failed", "quota_invalid_body", "quota_fallback", "tunnel_state_load_failed", "tunnel_address_invalid", "link_state_load_failed", "link_bind_invalid", "link_bind_failed", "link_bind_state_write_failed", "link_tls_config_failed", "link_listener_error", "link_listener_state_write_failed", "link_auto_renew_config_failed", "link_auto_renew_failed", "link_cert_reload_failed", "link_listener_cert_reload_failed", "link_disable_failed", "link_connection_error", "link_listener_accept_failed", "webhook_delivery_failed", "webhook_dispatcher_failed", "share_authorization_failed", "share_native_authorization_denied", "share_native_kv_put_failed", "share_v2_readiness_failed", "share_invitation_key_missing", "share_invitation_key_mismatch", "share_policy_denied", "attestation_quote_failed", "hook_events_read_failed", "hook_rows_load_failed", "missing_committed_hook_row", "webhook_enqueue_failed", "tunnel_connection_prepare_failed", "tunnel_idle_timeout", "tunnel_read_failed", "tunnel_malformed_frame", "tunnel_relay_error", "tunnel_write_failed", "tunnel_state_record_failed", "tunnel_state_clear_failed", "tunnel_auth_rejected", "other"] } } }, diff --git a/log-shipper/vector.tests.yaml b/log-shipper/vector.tests.yaml index b23b21cc..a8f2e9ab 100644 --- a/log-shipper/vector.tests.yaml +++ b/log-shipper/vector.tests.yaml @@ -252,7 +252,7 @@ tests: tag: "tinycloud.node" log: "[]" no_outputs_from: [allowlist] - - name: "Docker partial records are dropped" + - name: "partial records are dropped" inputs: - insert_at: allowlist type: log @@ -635,3 +635,168 @@ tests: assert_eq!(.event, "ingress_success_count") assert_eq!(.route, "") assert!(!contains(encode_json(.), "ticket-probe")) + - name: "labelled node GELF message passes after dockerd fields are stripped" + inputs: + - insert_at: logship_gate + type: log + log_fields: + '."_tc.logship"': "tinycloud-node" + "_container_id": "container-secret-probe" + "_container_name": "tinycloud-probe" + "_image_name": "image-secret-probe" + "_command": "private-command-probe" + "_private_probe": "custom-dockerd-probe" + host: "host-secret-probe" + version: "1" + level: 6 + timestamp: 1791972000.0 + source_type: "source-secret-probe" + peer_addr: "192.0.2.1" + port: 12345 + message: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"invocation replay insert failed"}}' + outputs: + - extract_from: logship_gate + conditions: + - type: vrl + source: | + assert_eq!(get(., ["_tc.logship"]) ?? null, "tinycloud-node") + assert_eq!(.gate_action, "candidate") + assert_eq!(.gate_label, "tinycloud-node") + - extract_from: strip_gelf_fields + conditions: + - type: vrl + source: | + assert_eq!(.gate_action, "candidate") + assert!((get(., ["_tc.logship"]) ?? null) == null) + assert!(!exists(._container_id)) + assert!(!exists(._container_name)) + assert!(!exists(._image_name)) + assert!(!exists(._command)) + assert!(!exists(._private_probe)) + assert!(!exists(.host)) + assert!(!exists(.version)) + assert!(!exists(.level)) + assert!(!exists(.timestamp)) + assert!(!exists(.source_type)) + assert!(!exists(.peer_addr)) + assert!(!exists(.port)) + assert!(!contains(encode_json(.), "secret-probe")) + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(.event, "node_warning") + assert_eq!(.event_code, "invocation_replay_insert_failed") + - name: "labelled ingress GELF message passes the allowlist" + inputs: + - insert_at: logship_gate + type: log + log_fields: + '."_tc.logship"': "tinycloud-ingress" + message: '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/info","status":404,"bytes":0,"request_time":0.02,"upstream_status":"-"}' + outputs: + - extract_from: allowlist + conditions: + - type: vrl + source: | + assert_eq!(.event, "ingress_failure_count") + assert_eq!(.route, "/info") + - name: "unlabelled GELF canaries are reduced to bounded counters" + inputs: + - &unlabelled_canary + insert_at: logship_gate + type: log + log_fields: + message: 'signed-kv-ticket-probe auth-probe did:key:owner-probe space-probe@example.invalid' + - *unlabelled_canary + no_outputs_from: [strip_gelf_fields, parse_gelf_message, allowlist] + outputs: + - extract_from: logship_counter_output + conditions: + - type: vrl + source: | + assert_eq!(.event, "logship_gate_dropped_total") + assert_eq!(.reason, "unlabelled") + assert_eq!(.count, 2) + assert!(!contains(encode_json(.), "signed-kv-ticket-probe")) + assert!(!contains(encode_json(.), "auth-probe")) + assert!(!contains(encode_json(.), "did:key:owner-probe")) + assert!(!contains(encode_json(.), "space-probe@example.invalid")) + - name: "wrong and postgres labels produce only a fixed unexpected-label event" + inputs: + - insert_at: logship_gate + type: log + log_fields: + '."_tc.logship"': "private-untrusted-label" + message: "ticket-probe auth-probe wrong-label-canary" + - insert_at: logship_gate + type: log + log_fields: + '."_tc.logship"': "postgres" + message: "sql-password-probe postgres-canary" + no_outputs_from: [strip_gelf_fields, parse_gelf_message, allowlist] + outputs: + - extract_from: logship_counter_output + conditions: + - type: vrl + source: | + assert_eq!(.event, "logship_unexpected_label") + assert_eq!(.reason, "unexpected_label") + assert_eq!(.count, 2) + assert!(!contains(encode_json(.), "private-untrusted-label")) + assert!(!contains(encode_json(.), "postgres")) + assert!(!contains(encode_json(.), "ticket-probe")) + assert!(!contains(encode_json(.), "sql-password-probe")) + - name: "label text inside a GELF message cannot forge the top-level label" + inputs: + - insert_at: logship_gate + type: log + log_fields: + message: '{"_tc.logship":"tinycloud-node","timestamp":"2026-10-14T10:00:00.000Z","level":"ERROR","fields":{"message":"node fatal panic"}}' + no_outputs_from: [strip_gelf_fields, parse_gelf_message, allowlist] + outputs: + - extract_from: logship_counter_output + conditions: + - type: vrl + source: | + assert_eq!(.event, "logship_gate_dropped_total") + assert_eq!(.reason, "unlabelled") + assert_eq!(.count, 1) + assert!(!contains(encode_json(.), "node fatal panic")) + - name: "split GELF fragments with invalid JSON are counted without content" + inputs: + - insert_at: logship_gate + type: log + log_fields: + '."_tc.logship"': "tinycloud-node" + "_container_id": "container-fragment-probe" + message: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"ERROR","fields":{"message":"token-fragment-probe' + no_outputs_from: [allowlist] + outputs: + - extract_from: logship_counter_output + conditions: + - type: vrl + source: | + assert_eq!(.event, "logship_gate_dropped_total") + assert_eq!(.reason, "malformed_envelope") + assert_eq!(.count, 1) + assert!(!contains(encode_json(.), "token-fragment-probe")) + - name: "labelled ingress non-JSON messages are dropped after the gate" + inputs: + - insert_at: logship_gate + type: log + log_fields: + '."_tc.logship"': "tinycloud-ingress" + _container_id: "container-non-json-probe" + message: "private-non-json-ingress-probe" + no_outputs_from: [allowlist] + outputs: + - extract_from: logship_counter_output + conditions: + - type: vrl + source: | + assert_eq!(.event, "logship_gate_dropped_total") + assert_eq!(.reason, "malformed_envelope") + assert_eq!(.count, 1) + assert!(!contains(encode_json(.), "private-non-json-ingress-probe")) + assert!(!contains(encode_json(.), "container-non-json-probe")) diff --git a/log-shipper/vector.yaml b/log-shipper/vector.yaml index e7cb164a..36f57e83 100644 --- a/log-shipper/vector.yaml +++ b/log-shipper/vector.yaml @@ -1,12 +1,218 @@ data_dir: /var/lib/vector secret: - bs: { type: directory, path: /tmp/vector-secrets, remove_trailing_whitespace: true } + bs: { type: directory, path: /var/lib/vector/secrets, remove_trailing_whitespace: true } sources: - docker_fluent: { type: fluent, mode: tcp, address: 0.0.0.0:24224 } + gelf_in: + type: socket + mode: udp + address: 0.0.0.0:12201 + max_length: 262144 + framing: { method: chunked_gelf, chunked_gelf: { decompression: Auto } } + decoding: { codec: gelf } transforms: + logship_gate: + type: remap + inputs: [gelf_in] + drop_on_error: true + drop_on_abort: true + source: | + lbl = string(get(., ["_tc.logship"]) ?? null) ?? "" + if lbl == "tinycloud-node" || lbl == "tinycloud-ingress" { + .gate_action = "candidate" + .gate_label = lbl + } else if lbl == "" { + . = {"gate_action": "drop", "gate_reason": "unlabelled"} + } else { + . = {"gate_action": "unexpected", "gate_reason": "unexpected_label"} + } + logship_gate_route: + type: route + inputs: [logship_gate] + route: + node: .gate_action == "candidate" && .gate_label == "tinycloud-node" + ingress: .gate_action == "candidate" && .gate_label == "tinycloud-ingress" + rejected: .gate_action == "drop" || .gate_action == "unexpected" + strip_gelf_fields: + type: remap + inputs: [logship_gate_route.node, logship_gate_route.ingress] + drop_on_error: true + drop_on_abort: true + source: | + label = .gate_label + raw = get(., ["message"]) ?? null + if !is_string(raw) { + . = {"gate_action": "drop", "gate_reason": "malformed_envelope"} + } else { + tag = if label == "tinycloud-node" { "tinycloud.node" } else { "tinycloud.ingress" } + . = {"gate_action": "candidate", "gate_label": label, "tag": tag, "message": raw} + } + parse_gelf_message: + type: remap + inputs: [strip_gelf_fields] + drop_on_error: true + drop_on_abort: true + source: | + if .gate_action == "drop" { + reason = .gate_reason + . = {"gate_action": "drop", "gate_reason": reason} + } else { + parsed, parse_err = parse_json(.message) + if parse_err != null || !is_object(parsed) { + . = {"gate_action": "drop", "gate_reason": "malformed_envelope"} + } else { + tag = .tag + raw = .message + . = {"tag": tag, "log": raw, "parsed_log": parsed} + } + } + logship_inner: + type: remap + inputs: [parse_gelf_message] + drop_on_error: true + drop_on_abort: true + source: | + if exists(.gate_action) { + action = .gate_action + reason = .gate_reason + . = {"gate_action": action, "gate_reason": reason} + } else if !is_string(.log) { + . = {"gate_action": "drop", "gate_reason": "malformed_envelope"} + } else { + raw = .log + parsed = .parsed_log + if !is_object(parsed) || exists(parsed.partial_message) { + . = {"gate_action": "drop", "gate_reason": "malformed_envelope"} + } else if .tag == "tinycloud.node" { + fields = get(parsed, ["fields"]) ?? null + message = if is_object(fields) { get(fields, ["message"]) ?? null } else { null } + if !is_string(message) { + . = {"gate_action": "drop", "gate_reason": "malformed_envelope"} + } else { + message = to_string(message) ?? "" + known_message = includes([ + "invocation replay insert failed", + "invocation replay cleanup failed", + "share-v2 local-tee feature active: deriving a deterministic, non-attested TEE identity from node key material", + "retention: delivered hook delivery prune failed", + "retention: dead-letter hook delivery prune failed", + "retention: signed KV ticket prune failed", + "failed to load tunnel state; tunnel not started", + "invalid loopback API address; tunnel not started", + "failed to load link state; LAN listener not started", + "failed to bind link LAN listener; LAN listener not started", + "failed to build TLS config; LAN listener not started", + "link LAN listener exited with error", + "failed to record link listener bind state", + "failed to record link listener bind failure", + "failed to load config for auto-renew", + "link auto-renew failed", + "failed to hot-reload renewed link cert", + "failed to reload TLS material after link auto-renew", + "link disable: service delete failed; local state removed anyway", + "link connection error", + "link listener accept error", + "webhook delivery attempt failed", + "webhook dispatcher loop failed", + "addressed delegation authorization failed", + "native share authorization denied", + "native share KV put failed", + "share v2 readiness probe failed", + "share v2 is enabled but shareEmail.invitationPublicKey is unset", + "configured share invitation key does not match the node's derived signing key", + "share/v2 policy challenge denied", + "failed to read committed hook events", + "failed to load kv write hook rows", + "failed to load kv delete hook rows", + "missing committed kv hook row for invocation", + "database write committed but webhook enqueue failed", + "tunnel: failed to prepare connection attempt", + "tunnel: no frame within idle timeout; treating connection as dead", + "tunnel: read error", + "tunnel: dropping malformed frame", + "tunnel: relay sent an error frame", + "tunnel: failed to write frame", + "tunnel: failed to record runtime state", + "tunnel: failed to clear runtime state after disable", + "tunnel authentication rejected", + "node fatal panic" + ], message) + if !known_message { + known_message = starts_with(message, "dstack socket available but get_info failed: ") || + starts_with(message, "quota service returned an invalid body: ") || + starts_with(message, "quota service returned ") || + starts_with(message, "Failed to get TDX quote: ") + } + if !known_message { + . = {"gate_action": "drop", "gate_reason": "unknown_message"} + } else { + . = {"tag": .tag, "log": raw, "parsed_log": parsed} + } + } + } else if .tag == "tinycloud.ingress" { + . = {"tag": .tag, "log": raw, "parsed_log": parsed} + } else { + . = {"gate_action": "drop", "gate_reason": "unexpected_label"} + } + } + logship_result_route: + type: route + inputs: [logship_gate_route.rejected, logship_inner] + route: + dropped: .gate_action == "drop" + unexpected: .gate_action == "unexpected" + validated: is_string(.tag) && is_string(.log) + logship_drop_counter: + type: remap + inputs: [logship_result_route.dropped] + drop_on_error: true + drop_on_abort: true + source: | + reason = .gate_reason + if !includes(["malformed_envelope", "unknown_message", "unlabelled"], reason) { abort } + minute = format_timestamp!(now(), format: "%Y-%m-%dT%H:%M") + . = {"event": "logship_gate_dropped_total", "reason": reason, "minute": minute, "count": 1} + logship_unexpected_counter: + type: remap + inputs: [logship_result_route.unexpected] + drop_on_error: true + drop_on_abort: true + source: | + minute = format_timestamp!(now(), format: "%Y-%m-%dT%H:%M") + . = {"event": "logship_unexpected_label", "reason": "unexpected_label", "minute": minute, "count": 1} + logship_counter_reduce: + type: reduce + inputs: [logship_drop_counter, logship_unexpected_counter] + group_by: [event, reason, minute] + expire_after_ms: 75000 + end_every_period_ms: 60000 + flush_period_ms: 1000 + merge_strategies: + count: sum + logship_counter_output: + type: remap + inputs: [logship_counter_reduce] + drop_on_error: true + drop_on_abort: true + source: | + event = .event + reason = .reason + count = .count + if !includes(["logship_gate_dropped_total", "logship_unexpected_label"], event) { abort } + if !includes(["malformed_envelope", "unexpected_label", "unknown_message", "unlabelled"], reason) { abort } + if !is_integer(count) { abort } + count = to_int(count) ?? 0 + if count < 1 || count > 1000000000 { abort } + . = { + "level": "warn", + "ts": format_timestamp!(now(), format: "%Y-%m-%dT%H:%M:%S%.3fZ"), + "event": event, + "svc": "logship", + "reason": reason, + "count": count + } allowlist: type: remap - inputs: [docker_fluent] + inputs: [logship_result_route.validated] drop_on_abort: true drop_on_error: true source: | @@ -14,30 +220,37 @@ transforms: if exists(.partial_message) { abort } if !is_string(.tag) { abort } tag = .tag - if !is_string(.log) { abort } - j, parse_err = parse_json(.log) - if parse_err != null || !is_object(j) { abort } + j = null + if exists(.parsed_log) { + j = .parsed_log + if !is_object(j) { abort } + } else { + if !is_string(.log) { abort } + j, parse_err = parse_json(.log) + if parse_err != null || !is_object(j) { abort } + } out = {} if tag == "tinycloud.node" { fields = get(j, ["fields"]) ?? null if !is_object(fields) { abort } level = get(j, ["level"]) ?? null if !is_string(level) { abort } - level = downcase!(level) + level = downcase(level) ?? "" if !includes(["warn", "error"], level) { abort } msg = get(fields, ["message"]) ?? null if !is_string(msg) { abort } + msg = to_string(msg) ?? "" event = null event_code = null if msg == "invocation replay insert failed" { event = "node_warning"; event_code = "invocation_replay_insert_failed" } else if msg == "invocation replay cleanup failed" { event = "node_warning"; event_code = "invocation_replay_cleanup_failed" } - else if starts_with!(msg, "dstack socket available but get_info failed: ") { event = "node_warning"; event_code = "dstack_get_info_failed" } + else if starts_with(msg, "dstack socket available but get_info failed: ") { event = "node_warning"; event_code = "dstack_get_info_failed" } else if msg == "share-v2 local-tee feature active: deriving a deterministic, non-attested TEE identity from node key material" { event = "node_warning"; event_code = "share_local_tee_identity" } else if msg == "retention: delivered hook delivery prune failed" { event = "node_warning"; event_code = "retention_hook_delivery_prune_failed" } else if msg == "retention: dead-letter hook delivery prune failed" { event = "node_warning"; event_code = "retention_dead_letter_prune_failed" } else if msg == "retention: signed KV ticket prune failed" { event = "node_warning"; event_code = "retention_signed_ticket_prune_failed" } - else if starts_with!(msg, "quota service returned an invalid body: ") { event = "node_warning"; event_code = "quota_invalid_body" } - else if starts_with!(msg, "quota service returned ") { event = "node_warning"; event_code = "quota_fallback" } + else if starts_with(msg, "quota service returned an invalid body: ") { event = "node_warning"; event_code = "quota_invalid_body" } + else if starts_with(msg, "quota service returned ") { event = "node_warning"; event_code = "quota_fallback" } else if msg == "failed to load tunnel state; tunnel not started" { event = "node_warning"; event_code = "tunnel_state_load_failed" } else if msg == "invalid loopback API address; tunnel not started" { event = "node_warning"; event_code = "tunnel_address_invalid" } else if msg == "failed to load link state; LAN listener not started" { event = "node_warning"; event_code = "link_state_load_failed" } @@ -62,7 +275,7 @@ transforms: else if msg == "share v2 is enabled but shareEmail.invitationPublicKey is unset" { event = "node_error"; event_code = "share_invitation_key_missing" } else if msg == "configured share invitation key does not match the node's derived signing key" { event = "node_error"; event_code = "share_invitation_key_mismatch" } else if msg == "share/v2 policy challenge denied" { event = "node_warning"; event_code = "share_policy_denied" } - else if starts_with!(msg, "Failed to get TDX quote: ") { event = "node_error"; event_code = "attestation_quote_failed" } + else if starts_with(msg, "Failed to get TDX quote: ") { event = "node_error"; event_code = "attestation_quote_failed" } else if msg == "failed to read committed hook events" { event = "node_warning"; event_code = "hook_events_read_failed" } else if msg == "failed to load kv write hook rows" { event = "node_warning"; event_code = "hook_rows_load_failed" } else if msg == "failed to load kv delete hook rows" { event = "node_warning"; event_code = "hook_rows_load_failed" } @@ -100,46 +313,43 @@ transforms: out.ts = format_timestamp!(parsed_ts, format: "%Y-%m-%dT%H:%M:%S%.3fZ") route = get(j, ["route"]) ?? null if !is_string(route) { abort } - route = to_string!(route) + route = to_string(route) ?? "" if length(route) > 256 { abort } if !includes(["/healthz","/info","/version","/invoke","/delegate","/revoke","/attestation","/signed/kv","/signed/kv/","/admin/quota","/admin/quota/","/admin/usage","/hooks/events","/hooks/tickets","/hooks/webhooks","/hooks/webhooks/","/delegation/query","/delegation/status","/encryption/networks","/encryption/networks/","/encryption/networks//revoke","/encryption/networks//decrypt","/policy/v3/status","/policy/v3/status/","/policy/v3/policies","/policy/v3/challenges","/policy/v3/delegations","/policy/v3/enforcer-bindings","/policy/v3/deliveries/authorize","/.well-known/tinycloud/node-keys","/.well-known/encryption/network/","/public//kv","/public//kv/","/peer/generate/","/evidences/"], route) { route = "" } if !is_string(route) { abort } - route = to_string!(route) if !includes(["/healthz","/info","/version","/invoke","/delegate","/revoke","/attestation","/signed/kv","/signed/kv/","/admin/quota","/admin/quota/","/admin/usage","/hooks/events","/hooks/tickets","/hooks/webhooks","/hooks/webhooks/","/delegation/query","/delegation/status","/encryption/networks","/encryption/networks/","/encryption/networks//revoke","/encryption/networks//decrypt","/policy/v3/status","/policy/v3/status/","/policy/v3/policies","/policy/v3/challenges","/policy/v3/delegations","/policy/v3/enforcer-bindings","/policy/v3/deliveries/authorize","/.well-known/tinycloud/node-keys","/.well-known/encryption/network/","/public//kv","/public//kv/","/peer/generate/","/evidences/",""], route) { abort } out.route = route method = get(j, ["method"]) ?? null if !is_string(method) { abort } - method = to_string!(method) if !includes(["GET","HEAD","POST","PUT","PATCH","DELETE","OPTIONS",""], method) { abort } out.method = method status = get(j, ["status"]) ?? null if !is_integer(status) { abort } - status = to_int!(status) + status = to_int(status) ?? 0 if status < 100 || status > 599 { abort } out.status = status bytes = get(j, ["bytes"]) ?? null if !is_integer(bytes) { abort } - bytes = to_int!(bytes) + bytes = to_int(bytes) ?? 0 if bytes < 0 || bytes > 1000000000 { abort } out.bytes_sent = bytes upstream_status_raw = "-" if exists(j.upstream_status) { upstream_status_raw = j.upstream_status } if !is_string(upstream_status_raw) { abort } - upstream_status_raw = to_string!(upstream_status_raw) + upstream_status_raw = to_string(upstream_status_raw) ?? "" if length(upstream_status_raw) > 63 { abort } upstream_status = "" if upstream_status_raw != "-" && upstream_status_raw != "" { - if !match!(upstream_status_raw, r'^[1-5][0-9]{2}(, ?[1-5][0-9]{2}| : [1-5][0-9]{2})*$') { abort } + if !match(upstream_status_raw, r'^[1-5][0-9]{2}(, ?[1-5][0-9]{2}| : [1-5][0-9]{2})*$') { abort } upstream_status = slice!(upstream_status_raw, 0, 3) } if !is_string(upstream_status) { abort } - upstream_status = to_string!(upstream_status) if length(upstream_status) > 3 { abort } - if !match!(upstream_status, r'^(|[1-5][0-9]{2})$') { abort } + if !match(upstream_status, r'^(|[1-5][0-9]{2})$') { abort } out.upstream_status = upstream_status request_time = get(j, ["request_time"]) ?? null if !(is_integer(request_time) || is_float(request_time)) { abort } - request_time = to_float!(request_time) + request_time = to_float(request_time) ?? 0.0 if request_time < 0 || request_time > 86400 { abort } out.request_time_seconds = request_time status_class = if status >= 500 { "5xx" } else if status >= 400 { "4xx" } else if status >= 300 { "3xx" } else if status >= 200 { "2xx" } else { "1xx" } @@ -153,11 +363,11 @@ transforms: # END GENERATED FROM fields.json classify_by_budget: type: route - inputs: [allowlist] + inputs: [allowlist, logship_counter_output] route: panic: .event == "node_fatal_panic" app_error: .event == "node_error" - app_warning: .event == "node_warning" + app_warning: .event == "node_warning" || .event == "logship_gate_dropped_total" || .event == "logship_unexpected_label" ingress_summary: .event == "ingress_failure_count" || .event == "ingress_success_count" ingress_slow: .event == "ingress_request_slow" split_requests: diff --git a/scripts/node-vector-schema.mjs b/scripts/node-vector-schema.mjs index d6faea97..b66ca03b 100644 --- a/scripts/node-vector-schema.mjs +++ b/scripts/node-vector-schema.mjs @@ -63,25 +63,23 @@ const quote = (value) => JSON.stringify(value); function vrlValidationLines(spec, accessor) { if (spec.type === "enum") return [ `if !is_string(${accessor}) { abort }`, - `${accessor} = to_string!(${accessor})`, `if !includes(${JSON.stringify(spec.values)}, ${accessor}) { abort }`, ]; if (spec.type === "integer") return [ `if !is_integer(${accessor}) { abort }`, - `${accessor} = to_int!(${accessor})`, + `${accessor} = to_int(${accessor}) ?? 0`, `if ${accessor} < ${spec.min} || ${accessor} > ${spec.max} { abort }`, ]; if (spec.type === "number") return [ `if !(is_integer(${accessor}) || is_float(${accessor})) { abort }`, - `${accessor} = to_float!(${accessor})`, + `${accessor} = to_float(${accessor}) ?? 0.0`, `if ${accessor} < ${spec.min} || ${accessor} > ${spec.max} { abort }`, ]; if (spec.type === "boolean") return [`if !is_boolean(${accessor}) { abort }`]; if (spec.type === "string") return [ `if !is_string(${accessor}) { abort }`, - `${accessor} = to_string!(${accessor})`, `if length(${accessor}) > ${spec.max_len} { abort }`, - `if !match!(${accessor}, r'${spec.regex}') { abort }`, + `if !match(${accessor}, r'${spec.regex}') { abort }`, ]; throw new Error(`unsupported type for ${spec.type}`); } @@ -95,25 +93,32 @@ export function renderAllowlist(schema) { "if exists(.partial_message) { abort }", "if !is_string(.tag) { abort }", "tag = .tag", - "if !is_string(.log) { abort }", - "j, parse_err = parse_json(.log)", - "if parse_err != null || !is_object(j) { abort }", + "j = null", + "if exists(.parsed_log) {", + " j = .parsed_log", + " if !is_object(j) { abort }", + "} else {", + " if !is_string(.log) { abort }", + " j, parse_err = parse_json(.log)", + " if parse_err != null || !is_object(j) { abort }", + "}", "out = {}", "if tag == \"tinycloud.node\" {", " fields = get(j, [\"fields\"]) ?? null", " if !is_object(fields) { abort }", " level = get(j, [\"level\"]) ?? null", " if !is_string(level) { abort }", - " level = downcase!(level)", + " level = downcase(level) ?? \"\"", " if !includes([\"warn\", \"error\"], level) { abort }", " msg = get(fields, [\"message\"]) ?? null", " if !is_string(msg) { abort }", + " msg = to_string(msg) ?? \"\"", " event = null", " event_code = null", ]; const messagePairs = Object.entries(sourceMessages); for (const [index, [message, [event, code]]] of messagePairs.entries()) { - const condition = message.endsWith(": ") || message === "quota service returned " ? `starts_with!(msg, ${quote(message)})` : `msg == ${quote(message)}`; + const condition = message.endsWith(": ") || message === "quota service returned " ? `starts_with(msg, ${quote(message)})` : `msg == ${quote(message)}`; lines.push(` ${index === 0 ? "if" : "else if"} ${condition} { event = ${quote(event)}; event_code = ${quote(code)} }`); } lines.push(" else if msg == \"node fatal panic\" { event = \"node_fatal_panic\"; event_code = \"other\" }"); @@ -139,7 +144,7 @@ export function renderAllowlist(schema) { lines.push(" out.ts = format_timestamp!(parsed_ts, format: \"%Y-%m-%dT%H:%M:%S%.3fZ\")"); lines.push(" route = get(j, [\"route\"]) ?? null"); lines.push(" if !is_string(route) { abort }"); - lines.push(" route = to_string!(route)"); + lines.push(" route = to_string(route) ?? \"\""); lines.push(" if length(route) > 256 { abort }"); const knownRoutes = service.fields.route.values.filter((value) => value !== ""); lines.push(` if !includes(${JSON.stringify(knownRoutes)}, route) { route = "" }`); @@ -157,11 +162,11 @@ export function renderAllowlist(schema) { lines.push(" upstream_status_raw = \"-\""); lines.push(" if exists(j.upstream_status) { upstream_status_raw = j.upstream_status }"); lines.push(" if !is_string(upstream_status_raw) { abort }"); - lines.push(" upstream_status_raw = to_string!(upstream_status_raw)"); + lines.push(" upstream_status_raw = to_string(upstream_status_raw) ?? \"\""); lines.push(" if length(upstream_status_raw) > 63 { abort }"); lines.push(" upstream_status = \"\""); lines.push(" if upstream_status_raw != \"-\" && upstream_status_raw != \"\" {"); - lines.push(" if !match!(upstream_status_raw, r'^[1-5][0-9]{2}(, ?[1-5][0-9]{2}| : [1-5][0-9]{2})*$') { abort }"); + lines.push(" if !match(upstream_status_raw, r'^[1-5][0-9]{2}(, ?[1-5][0-9]{2}| : [1-5][0-9]{2})*$') { abort }"); lines.push(" upstream_status = slice!(upstream_status_raw, 0, 3)"); lines.push(" }"); for (const line of vrlValidationLines(service.fields.upstream_status, "upstream_status")) lines.push(` ${line}`); @@ -186,10 +191,34 @@ export function checkVectorFile(schema, yaml = readFileSync(vectorPath, "utf8")) if (!match) throw new Error("vector.yaml is missing generated allowlist markers"); const actual = match[0].split("\n").map((line) => line.replace(/^ /, "")).join("\n"); if (actual !== renderAllowlist(schema)) throw new Error("vector.yaml allowlist differs from fields.json-generated VRL"); - for (const name of ["allowlist", "slow_success", "request_minute", "request_output", "budget_size_guard"]) { + for (const name of ["logship_gate", "strip_gelf_fields", "parse_gelf_message", "logship_inner", "logship_drop_counter", "logship_unexpected_counter", "logship_counter_output", "allowlist", "slow_success", "request_minute", "request_output", "budget_size_guard"]) { const block = yaml.match(new RegExp(`^ ${name}:\\n([\\s\\S]*?)(?=\\n [a-z][a-z0-9_]*:\\n|\\nsinks:)`, "m"))?.[1]; if (!block?.includes("type: remap") || !block.includes("drop_on_error: true")) throw new Error(`${name} remap must set drop_on_error: true`); } + const firstTransform = yaml.match(/transforms:\n ([a-z][a-z0-9_]*):/)?.[1]; + const gate = yaml.match(/ logship_gate:\n([\s\S]*?)(?=\n [a-z][a-z0-9_]*:\n|\nsinks:)/)?.[1]; + const gelfSource = yaml.match(/ gelf_in:\n([\s\S]*?)(?=\n [a-z][a-z0-9_]*:\n|\ntransforms:)/)?.[1]; + if (firstTransform !== "logship_gate" || + !gate?.includes('inputs: [gelf_in]') || + !gate.includes('lbl = string(get(., ["_tc.logship"]) ?? null) ?? ""') || + !gelfSource?.includes("type: socket") || + !gelfSource.includes("mode: udp") || + !gelfSource.includes("address: 0.0.0.0:12201") || + !gelfSource.includes("framing: { method: chunked_gelf, chunked_gelf: { decompression: Auto } }") || + !gelfSource.includes("decoding: { codec: gelf }")) { + throw new Error("GELF socket and first-transform label gate differ from the reviewed transport"); + } + const transformOrder = ["logship_gate", "strip_gelf_fields", "parse_gelf_message", "allowlist"]; + if (transformOrder.some((name, index) => index > 0 && yaml.indexOf(` ${transformOrder[index - 1]}:`) >= yaml.indexOf(` ${name}:`))) { + throw new Error("GELF gate, field-stripping, parse, and allowlist order changed"); + } + if (/docker_files|parse_docker_envelope|read_from:|\/var\/lib\/docker|fluentd/i.test(yaml)) throw new Error("obsolete Docker file and Fluentd transport remnants remain"); + const inner = yaml.match(/ logship_inner:\n([\s\S]*?)(?=\n [a-z][a-z0-9_]*:\n|$)/)?.[1]; + if (!inner) throw new Error("logship_inner message guard is missing"); + for (const message of Object.keys(sourceMessages)) { + if (!inner.includes(quote(message))) throw new Error(`logship_inner message allowlist is missing ${message}`); + } + if (!inner.includes('"node fatal panic"') || !inner.includes('"unknown_message"')) throw new Error("logship_inner must count unknown messages and preserve the fixed panic event"); const requestOutput = yaml.match(/ request_output:\n([\s\S]*?)(?=\n [a-z][a-z0-9_]*:\n|$)/)?.[1]; if (!requestOutput?.includes("drop_on_abort: true")) throw new Error("request_output must drop aborts"); const count = schema.services["tinycloud-node"].fields.request_count; @@ -213,8 +242,11 @@ export function checkVectorFile(schema, yaml = readFileSync(vectorPath, "utf8")) const requiredBudgetConfig = [ "panic: .event == \"node_fatal_panic\"", "app_error: .event == \"node_error\"", - "app_warning: .event == \"node_warning\"", - "ingress_summary: .event == \"ingress_failure_count\" || .event == \"ingress_success_count\"", + "app_warning: .event == \"node_warning\" || .event == \"logship_gate_dropped_total\" || .event == \"logship_unexpected_label\"", + "inputs: [logship_gate_route.node, logship_gate_route.ingress]", + "inputs: [logship_gate_route.rejected, logship_inner]", + "inputs: [allowlist, logship_counter_output]", + "inputs: [logship_result_route.validated]", "group_by: [route, status_class, event, minute]", "inputs: [classify_by_budget.panic]", "threshold: 2", @@ -234,11 +266,22 @@ export function checkVectorFile(schema, yaml = readFileSync(vectorPath, "utf8")) if (monthlyBudgetBytes > 480000000) throw new Error(`Vector event budget exceeds 0.48 GB/month: ${monthlyBudgetBytes} bytes`); if ((yaml.match(/threshold: 5\n/g) ?? []).length !== 2) throw new Error("node error and warning lanes must each be capped at five events/minute"); } + +export function checkComposeDeploymentPackaging(compose) { + if (/^\s*env_file\s*:/m.test(compose)) throw new Error("Compose packaging must not use env_file"); + const configs = compose.match(/^configs:\n([\s\S]*?)(?=\n\S|$)/)?.[1]; + if (configs && /^\s+file\s*:/m.test(configs)) throw new Error("Compose packaging must not use configs.*.file"); + if (/^\s*-\s*(?:\.\/|\.\.\/)/m.test(compose) || /^\s*source:\s*(?!\/|\|)[^\s]+/m.test(compose)) { + throw new Error("Compose packaging must not use relative bind sources"); + } +} + export function renderComposeVectorConfig(vectorYaml) { return vectorYaml.trimEnd().split("\n").map((line) => ` ${line.replaceAll("$", () => "$$")}`).join("\n"); } export function checkComposeVectorConfig(compose = readFileSync(`${root}docker-compose.dstack-postgres.yaml`, "utf8"), vectorYaml = readFileSync(vectorPath, "utf8")) { + checkComposeDeploymentPackaging(compose); const start = compose.indexOf(" VECTOR_CONFIG_YAML: |\n"); const end = compose.indexOf(" entrypoint:", start); if (start < 0 || end < 0) throw new Error("compose Vector config block not found"); @@ -268,16 +311,21 @@ export function checkNodeSchema(schema) { if (JSON.stringify(schema.common.level) !== JSON.stringify({ type: "enum", values: ["debug", "info", "warn", "error"] })) throw new Error("common.level differs from the emitted level allowlist"); if (JSON.stringify(schema.common.ts) !== JSON.stringify({ type: "string", regex: "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$", max_len: 24 })) throw new Error("common.ts differs from the emitted timestamp format"); if (schema.common.event?.type !== "event") throw new Error("common.event must use event type"); - if (service.events.join(",") !== "node_fatal_panic,node_warning,node_error,ingress_failure_count,ingress_request_slow,ingress_success_count") throw new Error("tinycloud-node event set differs from the reviewed set"); + if (service.events.join(",") !== "node_fatal_panic,node_warning,node_error,ingress_failure_count,ingress_request_slow,ingress_success_count,logship_gate_dropped_total,logship_unexpected_label") throw new Error("tinycloud-node event set differs from the reviewed set"); if (Object.values(sourceMessages).some(([, code]) => !new Set(service.fields.event_code?.values ?? []).has(code))) throw new Error("source message map contains an event_code missing from fields.json"); if (Object.keys(service.fields).sort().join(",") !== [ - "bytes_sent", "event_code", "method", "request_count", "request_time_seconds", "route", "status", "status_class", "svc", "upstream_status", + "bytes_sent", "count", "event_code", "method", "reason", "request_count", "request_time_seconds", "route", "status", "status_class", "svc", "upstream_status", ].sort().join(",")) throw new Error("tinycloud-node fields differ from the reviewed field set"); if (service.fields.request_count?.type !== "integer" || service.fields.request_count.min !== 1 || service.fields.request_count.max !== 1000000) throw new Error("request_count must remain bounded to 1..1000000"); + if (service.fields.count?.type !== "integer" || service.fields.count.min !== 1 || service.fields.count.max !== 1000000000) throw new Error("logship count must remain bounded to 1..1000000000"); + if (JSON.stringify(service.fields.reason) !== JSON.stringify({ type: "enum", values: ["malformed_envelope", "unexpected_label", "unknown_message", "unlabelled"] })) throw new Error("logship reason enum differs from emitted values"); if (service.fields.route?.type !== "enum" || service.fields.route.values.length !== 36 || !service.fields.route.values.includes("")) throw new Error("route output must be a 36-value bounded enum including "); - if (JSON.stringify(service.fields.svc) !== JSON.stringify({ type: "enum", values: ["api", "ingress"] })) throw new Error("svc enum differs from emitted values"); + if (JSON.stringify(service.fields.svc) !== JSON.stringify({ type: "enum", values: ["api", "ingress", "logship"] })) throw new Error("svc enum differs from emitted values"); if (JSON.stringify(service.fields.status_class) !== JSON.stringify({ type: "enum", values: ["1xx", "2xx", "3xx", "4xx", "5xx"] })) throw new Error("status_class enum differs from emitted values"); if (service.fields.event_code?.type !== "enum" || !service.fields.event_code.values.includes("other")) throw new Error("event_code must preserve the fixed panic other value"); + for (const event of ["logship_gate_dropped_total", "logship_unexpected_label"]) { + if (!service.events.includes(event)) throw new Error(`tinycloud-node is missing ${event}`); + } } if (import.meta.url === `file://${process.argv[1]}`) { diff --git a/scripts/node-vector-schema.test.mjs b/scripts/node-vector-schema.test.mjs index 6f40ae5e..6a285119 100644 --- a/scripts/node-vector-schema.test.mjs +++ b/scripts/node-vector-schema.test.mjs @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import test from "node:test"; -import { checkVectorFile, renderAllowlist, renderComposeVectorConfig } from "./node-vector-schema.mjs"; +import { checkComposeDeploymentPackaging, checkVectorFile, renderAllowlist, renderComposeVectorConfig } from "./node-vector-schema.mjs"; const workflow = readFileSync(new URL("../.github/workflows/rust.yml", import.meta.url), "utf8"); const schema = JSON.parse(readFileSync(new URL("../log-shipper/fields.json", import.meta.url), "utf8")); @@ -93,10 +93,23 @@ test("CI runs native Vector validation and tests after checking the pinned relea assert.match(workflow, /validate --no-environment log-shipper\/vector\.yaml/); assert.match(workflow, /test log-shipper\/vector\.yaml log-shipper\/vector\.tests\.yaml/); }); + +test("deployed compose rejects file-backed configs, env_file, and relative bind sources", () => { + for (const invalid of [ + "services:\n vector:\n env_file: .env", + "configs:\n vector:\n file: ./vector.yaml", + "services:\n vector:\n volumes:\n - ./vector.yaml:/etc/vector/vector.yaml:ro", + "services:\n vector:\n volumes:\n - type: bind\n source: ../vector.yaml\n target: /etc/vector/vector.yaml", + "services:\n vector:\n volumes:\n - type: bind\n source: ${LOCAL_CONFIG_PATH}\n target: /etc/vector/vector.yaml", + ]) { + assert.throws(() => checkComposeDeploymentPackaging(invalid), /Compose packaging/); + } + assert.doesNotThrow(() => checkComposeDeploymentPackaging(compose)); +}); test("compose embeds source Vector config without allowing Compose to interpolate its regexes", () => { assert.equal( - renderComposeVectorConfig("sources:\n test: true\nregex: ^[a-z]+$"), - " sources:\n test: true\n regex: ^[a-z]+$$", + renderComposeVectorConfig("sources:\n test: true\nregex: ^[a-z]+$\ntoken: ${BETTERSTACK_SOURCE_TOKEN}"), + " sources:\n test: true\n regex: ^[a-z]+$$\n token: $${BETTERSTACK_SOURCE_TOKEN}", ); }); test("missing Better Stack token leaves node and ingress startup independent", () => { @@ -106,8 +119,8 @@ test("missing Better Stack token leaves node and ingress startup independent", ( const ingress = compose.slice(compose.indexOf(" dstack-ingress:\n"), compose.indexOf(" ssh-gateway:\n")); assert.doesNotMatch(node, /depends_on:[\s\S]*?vector/); assert.doesNotMatch(ingress, /depends_on:[\s\S]*?vector/); - assert.match(node, /fluentd-async: "true"/); - assert.match(ingress, /fluentd-async: "true"/); + assert.match(node, /driver: gelf[\s\S]*gelf-address: udp:\/\/127\.0\.0\.1:12201[\s\S]*labels: "tc\.logship"/); + assert.match(ingress, /driver: gelf[\s\S]*gelf-address: udp:\/\/127\.0\.0\.1:12201[\s\S]*labels: "tc\.logship"/); assert.match(compose, /Better Stack token is absent; Vector is idle/); const requiredVarsStart = deployWorkflow.indexOf("required_vars=("); const requiredVarsEnd = deployWorkflow.indexOf("\n )", requiredVarsStart); @@ -125,3 +138,45 @@ test("WARN lane bypasses the ingress throttle and stays behind the size guard", assert.match(vector, /budget_size_guard:\n type: remap\n inputs: \[throttle_panic, throttle_app_error, throttle_app_warning, throttle_ingress\]/); assert.match(vector, /sinks:\n betterstack:\n type: http\n inputs: \[budget_size_guard\]/); }); + +test("node schema includes bounded fixed logship counter events", () => { + assert.ok(schema.services["tinycloud-node"].events.includes("logship_gate_dropped_total")); + assert.ok(schema.services["tinycloud-node"].events.includes("logship_unexpected_label")); + assert.deepEqual(schema.services["tinycloud-node"].fields.svc.values, ["api", "ingress", "logship"]); + assert.deepEqual(schema.services["tinycloud-node"].fields.reason.values, ["malformed_envelope", "unexpected_label", "unknown_message", "unlabelled"]); + assert.deepEqual(schema.services["tinycloud-node"].fields.count, { type: "integer", min: 1, max: 1000000000 }); +}); + +test("first transform gates GELF labels before field stripping and JSON parsing", () => { + const firstTransform = vector.match(/transforms:\n\s{2}([a-z][a-z0-9_]*):/); + assert.equal(firstTransform?.[1], "logship_gate"); + assert.match(vector, /gelf_in:\n type: socket\n mode: udp\n address: 0\.0\.0\.0:12201\n max_length: 262144\n framing: \{ method: chunked_gelf, chunked_gelf: \{ decompression: Auto \} \}\n decoding: \{ codec: gelf \}/); + assert.match(vector, /lbl = string\(get\(\., \["_tc\.logship"\]\) \?\? null\) \?\? ""/); + assert.ok(vector.indexOf(" logship_gate:") < vector.indexOf(" strip_gelf_fields:")); + assert.ok(vector.indexOf(" strip_gelf_fields:") < vector.indexOf(" parse_gelf_message:")); + assert.ok(vector.indexOf(" parse_gelf_message:") < vector.indexOf(" allowlist:")); + assert.doesNotMatch(vector, /docker_files|parse_docker_envelope|read_from:/); + assert.match(compose, /driver: gelf[\s\S]*gelf-address: udp:\/\/127\.0\.0\.1:12201[\s\S]*labels: "tc\.logship"/); + assert.match(compose, /127\.0\.0\.1:12201:12201\/udp/); + assert.match(compose, /ipv4_address: 10\.255\.253\.2/); + const nodeService = compose.slice(compose.indexOf(" tinycloud:\n"), compose.indexOf(" dstack-ingress:\n")); + const ingressStart = compose.indexOf(" dstack-ingress:\n"); + const ingressService = compose.slice(ingressStart, compose.indexOf(" ssh-gateway:\n", ingressStart)); + const sshStart = compose.indexOf(" ssh-gateway:\n"); + const sshService = compose.slice(sshStart, compose.indexOf(" vector:\n", sshStart)); + const vectorStart = compose.indexOf(" vector:\n"); + const vectorService = compose.slice(vectorStart, compose.indexOf("\nvolumes:\n", vectorStart)); + assert.match(nodeService, /labels:\n\s+tc\.logship: "tinycloud-node"/); + assert.match(ingressService, /labels:\n\s+tc\.logship: "tinycloud-ingress"/); + assert.match(sshService, /driver: json-file[\s\S]*max-size: "10m"[\s\S]*max-file: "3"/); + assert.doesNotMatch(sshService, /tc\.logship/); + assert.match(vectorService, /user: "10001:10001"[\s\S]*read_only: true[\s\S]*cap_drop: \[ALL\][\s\S]*mem_limit: 64m/); + assert.match(vectorService, /tmpfs:\n\s+- \/var\/lib\/vector:rw,noexec,nosuid,size=8m/); + assert.match(vectorService, /networks:\n\s+logship:\n\s+ipv4_address: 10\.255\.253\.2/); + const vectorPorts = [...vectorService.matchAll(/^\s+- "([^"]+)"$/gm)].map((match) => match[1]).filter((port) => port.includes(":12201:")); + assert.deepEqual(vectorPorts, ["127.0.0.1:12201:12201/udp"]); + assert.match(vectorService, /driver: json-file[\s\S]*max-size: "10m"[\s\S]*max-file: "3"/); + assert.doesNotMatch(vectorService, /labels:\n\s+tc\.logship:/); + assert.doesNotMatch(compose, /fluentd|\/var\/lib\/docker|docker\.sock/); + assert.match(compose, /networks:\n logship:\n driver: bridge\n ipam:\n config:\n - subnet: 10\.255\.253\.0\/28/); +}); diff --git a/tc-959-node-docker-check.review.sh b/tc-959-node-docker-check.review.sh new file mode 100755 index 00000000..788765db --- /dev/null +++ b/tc-959-node-docker-check.review.sh @@ -0,0 +1,396 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT=$(git rev-parse --show-toplevel) +BASE_COMPOSE="$ROOT/docker-compose.dstack-postgres.yaml" +VECTOR_BIN=${TC_VECTOR_BIN:-/tmp/tc959-vector-0.59.0/vector-x86_64-unknown-linux-musl/bin/vector} +DRY_RUN=${TC_DRY_RUN:-0} +WORK=$(mktemp -d "${TMPDIR:-/tmp}/tc959-gelf-check.XXXXXX") +CONTROL="$WORK/control" +mkdir -p "$CONTROL" +PROJECT="tc959-gelf-$$" +OVERRIDE="$WORK/compose.override.yaml" +VECTOR_TEST_CONFIG="$WORK/vector.acceptance.yaml" +ENV_FILE="$WORK/acceptance.env" +REPORT=${TC_REPORT:-"/tmp/tc959-node-gelf-check-$$.txt"} +COMPOSE=(docker compose --env-file "$ENV_FILE" --project-directory "$ROOT" -p "$PROJECT" -f "$BASE_COMPOSE" -f "$OVERRIDE") +VECTOR_ID="" +NODE_ID="" +PG_ID="" +STARTED=0 +SAMPLER_PID="" +SAMPLER_STOP="$WORK/sample-memory" + +record() { printf '%s\n' "$*" | tee -a "$REPORT"; } +docker_cmd() { + if [[ "$DRY_RUN" == 1 ]]; then + record "DRY-RUN docker $*" + return 0 + fi + command docker "$@" +} +compose_cmd() { + if [[ "$DRY_RUN" == 1 ]]; then + record "DRY-RUN compose $*" + return 0 + fi + "${COMPOSE[@]}" "$@" +} +memory_sampler() { + local peak=0 container running current + while [[ -f "$SAMPLER_STOP" ]]; do + container=$("${COMPOSE[@]}" ps -q vector 2>/dev/null || true) + if [[ -n "$container" ]]; then + running=$(command docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null || true) + if [[ "$running" == true ]]; then + current=$(command docker stats --no-stream --format '{{.MemUsage}}' "$container" 2>/dev/null | awk '{value=$1; amount=value; sub(/[A-Za-z].*$/, "", amount); unit=substr(value, length(amount)+1); multiplier=unit=="GiB"?1073741824:unit=="MiB"?1048576:unit=="KiB"?1024:1; printf "%.0f", amount*multiplier}' || true) + [[ "$current" =~ ^[0-9]+$ ]] || current=0 + (( current > peak )) && peak=$current + fi + fi + printf '%s\n' "$peak" >"$WORK/vector-peak-bytes" + sleep 0.2 + done +} + +cleanup() { + local status=$? + if [[ -n "$SAMPLER_PID" ]]; then + rm -f "$SAMPLER_STOP" + wait "$SAMPLER_PID" || true + fi + if [[ "$DRY_RUN" != 1 && "$STARTED" == 1 ]]; then + compose_cmd down --remove-orphans --volumes >/dev/null 2>&1 || true + fi + rm -rf "$WORK" + exit "$status" +} +trap cleanup EXIT + +fail() { record "FAIL: $*"; exit 1; } +require() { command -v "$1" >/dev/null 2>&1 || fail "missing command: $1"; } + +record "TC-959 GELF/UDP acceptance; project=$PROJECT; report=$REPORT" +record "This script is review-only. Run only after the lead authorizes Docker." + +require node +[[ -x "$VECTOR_BIN" ]] || fail "native Vector binary not executable: $VECTOR_BIN" +node "$ROOT/scripts/node-vector-schema.mjs" --check || fail "generated schema check failed" +node "$ROOT/scripts/node-vector-schema.mjs" --check-compose || fail "embedded Compose parity failed" +node --test "$ROOT/scripts/node-vector-schema.test.mjs" || fail "schema/packaging tests failed" +"$VECTOR_BIN" validate --no-environment "$ROOT/log-shipper/vector.yaml" || fail "native Vector validation failed" +"$VECTOR_BIN" test "$ROOT/log-shipper/vector.yaml" "$ROOT/log-shipper/vector.tests.yaml" || fail "native Vector tests failed" +record "PASS: native checks completed before any Docker operation" + +cat >"$ENV_FILE" <<'EOF' +DATABASE_URL=postgres://postgres@postgres:5432/postgres?sslmode=disable +S3_BUCKET=acceptance-unused +S3_ENDPOINT=http://127.0.0.1:9 +AWS_KEY=acceptance-unused +AWS_SECRET=acceptance-unused +AWS_REGION=us-east-1 +SQL_IDENTITY_FENCE=true +SHARE_TRUST_BUNDLE_BASE64=e30= +TINYCLOUD_QUOTA_URL=http://127.0.0.1:9 +TINYCLOUD_ADMIN_SECRET= +CLOUDFLARE_API_TOKEN=acceptance-unused +DSTACK_GATEWAY_DOMAIN=invalid.test +CERTBOT_EMAIL=acceptance@example.invalid +EOF + +node - "$ROOT/log-shipper/vector.yaml" "$VECTOR_TEST_CONFIG" <<'NODE' +const fs = require('node:fs'); +const [sourcePath, targetPath] = process.argv.slice(2); +let config = fs.readFileSync(sourcePath, 'utf8'); +const sinkAt = config.lastIndexOf('\nsinks:\n'); +if (sinkAt < 0) throw new Error('canonical sink block not found'); +const metricSource = ' internal_metrics:\n type: internal_metrics\n'; +if (!config.includes('\ntransforms:\n')) throw new Error('canonical transforms block not found'); +config = config.replace('\ntransforms:\n', `\n${metricSource}transforms:\n`); +const updatedSinkAt = config.lastIndexOf('\nsinks:\n'); +config = config.slice(0, updatedSinkAt) + `\nsinks:\n local_console:\n type: console\n inputs: [budget_size_guard]\n encoding: { codec: json }\n metrics_exporter:\n type: prometheus_exporter\n inputs: [internal_metrics]\n address: 0.0.0.0:9598\n`; +fs.writeFileSync(targetPath, config, { mode: 0o600 }); +NODE +"$VECTOR_BIN" validate --no-environment "$VECTOR_TEST_CONFIG" || fail "acceptance Vector config validation failed" + +cat >"$OVERRIDE" </dev/null || fail "Docker Compose v2 is required" +record "Preflight: no DockerRootDir assumption; this GELF topology has no host Docker-directory access." + +# Continue below only on an explicitly Docker-authorized acceptance host. +STARTED=1 +compose_cmd up -d --wait postgres tinycloud dstack-ingress vector app-net-probe +touch "$SAMPLER_STOP" +memory_sampler & +SAMPLER_PID=$! + +wait_vector() { + local deadline=$((SECONDS + 20)) + while (( SECONDS < deadline )); do + VECTOR_ID=$(compose_cmd ps -q vector) + if [[ -n "$VECTOR_ID" ]] && [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] && curl --fail --silent --max-time 1 http://127.0.0.1:9598/metrics >/dev/null; then return 0; fi + sleep 0.2 + done + fail "Vector did not become running within 20 seconds" +} +wait_node() { + local deadline=$((SECONDS + 30)) + while (( SECONDS < deadline )); do + if curl --fail --silent --max-time 2 http://127.0.0.1:18000/version >/dev/null; then return 0; fi + sleep 0.5 + done + fail "node did not serve /version within 30 seconds" +} +read_metric() { + curl --fail --silent http://127.0.0.1:9598/metrics | node -e 'let s="";process.stdin.on("data",d=>s+=d);process.stdin.on("end",()=>{const x=s.split("\n").find(v=>v.startsWith("vector_component_received_events_total{")&&v.includes("component_id=\"gelf_in\""));process.stdout.write(x?x.trim().split(/\s+/).at(-1):"missing")})' +} +wait_metric() { + local deadline=$((SECONDS + 20)) value + while (( SECONDS < deadline )); do + value=$(read_metric) + if [[ "$value" =~ ^[0-9]+$ ]] && (( value >= ${1:-1} )); then printf '%s\n' "$value"; return 0; fi + sleep 0.25 + done + fail "Vector GELF receive metric did not reach ${1:-1}" +} +vector_event_count() { + [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] || fail "refusing to read logs from a stopped Vector container" + docker_cmd logs "$VECTOR_ID" 2>&1 | node -e 'let s="";process.stdin.on("data",d=>s+=d);process.stdin.on("end",()=>process.stdout.write(String(s.split("\n").filter(x=>x.includes("ingress_request_slow")).length)))' +} +wait_next_line() { + local previous=${1:-0} deadline=$((SECONDS + 20)) current + touch "$CONTROL/pulse" + while (( SECONDS < deadline )); do + wait_vector + current=$(vector_event_count) + if (( current > previous )); then + sleep 0.4 + current=$(vector_event_count) + (( current == previous + 1 )) || fail "duplicate/flood output after Vector recovery: expected one line, saw $((current - previous))" + return 0 + fi + sleep 0.2 + done + fail "next labelled line not observed within 20 seconds" +} + +compose_cmd ps -q tinycloud >/dev/null +NODE_ID=$(compose_cmd ps -q tinycloud) +PG_ID=$(compose_cmd ps -q postgres) +[[ -n "$NODE_ID" && -n "$PG_ID" ]] || fail "node or Postgres container missing" +node_restarts=$(docker_cmd inspect -f '{{.RestartCount}}' "$NODE_ID") +wait_vector +wait_node +curl --fail --silent --show-error --max-time 5 http://127.0.0.1:18000/version >/dev/null || fail "node did not serve /version without dstack" +wait_next_line 0 +wait_metric 1 >/dev/null +record "PASS: node serves /version with test-only Static key, TEE Off, local Postgres, and no dstack mount" + +for mode in restart crash recreate; do + for iteration in 1 2 3 4 5; do + case "$mode" in + restart) + before=$(vector_event_count) + compose_cmd restart vector >/dev/null + wait_vector + wait_next_line "$before" + ;; + crash) + before=$(vector_event_count) + docker_cmd kill --signal KILL "$VECTOR_ID" >/dev/null + deadline=$((SECONDS + 20)) + while (( SECONDS < deadline )); do + if [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID" 2>/dev/null || true)" == true ]]; then break; fi + sleep 0.2 + done + wait_vector + wait_next_line "$before" + ;; + recreate) + compose_cmd up -d --force-recreate --no-deps vector >/dev/null + wait_vector + wait_next_line 0 + ;; + esac + record "PASS: $mode recovery $iteration/5; next labelled line <=20s; one output; app remains up" + done +done + +for iteration in 1 2 3; do + before=$(vector_event_count) + compose_cmd stop vector >/dev/null + printf '%s\n' 500 >"$CONTROL/burst" + deadline=$((SECONDS + 20)) + while [[ ! -f "$CONTROL/burst-done" ]] && (( SECONDS < deadline )); do sleep 0.1; done + [[ -f "$CONTROL/burst-done" ]] || fail "ingress 500-line outage burst did not complete" + rm -f "$CONTROL/burst-done" + compose_cmd start vector >/dev/null + wait_vector + wait_next_line "$before" + record "PASS: Vector-down 500-line burst $iteration/3; next labelled line <=20s" +done + +node_restarts_after=$(docker_cmd inspect -f '{{.RestartCount}}' "$NODE_ID") +[[ "$node_restarts" == "$node_restarts_after" ]] || fail "node restarted during Vector recovery matrix" +record "PASS: app restart count unchanged across all Vector recovery tests" + +before_metric=$(wait_metric 0) +printf '%s\n' 5000 >"$CONTROL/burst" +deadline=$((SECONDS + 30)) +while [[ ! -f "$CONTROL/burst-done" ]] && (( SECONDS < deadline )); do sleep 0.1; done +[[ -f "$CONTROL/burst-done" ]] || fail "5,000-line producer burst did not complete" +rm -f "$CONTROL/burst-done" +deadline=$((SECONDS + 20)) +after_metric=$before_metric +while (( SECONDS < deadline )); do + after_metric=$(read_metric) + [[ "$after_metric" =~ ^[0-9]+$ ]] || after_metric=$before_metric + (( after_metric >= before_metric + 5000 )) && break + sleep 0.5 +done +loss=100 +if (( after_metric >= before_metric )); then loss=$(( (5000 - (after_metric - before_metric)) * 100 / 5000 )); (( loss < 0 )) && loss=0; fi +record "INFO: 5,000-line receive count delta=$((after_metric - before_metric)); estimated transport loss=${loss}% (informational)" + +# Stop the synthetic ingress producer so the isolation counter window has no legitimate GELF traffic. +compose_cmd stop dstack-ingress >/dev/null +sleep 1 +baseline=$(wait_metric 0) +payload='{"version":"1.1","host":"tc959-isolation-probe","short_message":"{\"timestamp\":\"2026-10-14T10:00:00.000Z\",\"level\":\"WARN\",\"fields\":{\"message\":\"invocation replay insert failed\"}}","timestamp":1760436000,"level":4,"_tc.logship":"tinycloud-node"}' +vector_ip=10.255.253.2 +app_gateway=$(docker_cmd inspect -f '{{range .NetworkSettings.Networks}}{{.Gateway}}{{end}}' "$NODE_ID") +probe_targets=("$vector_ip" "$app_gateway" vector host.docker.internal 127.0.0.1) +for target in "${probe_targets[@]}"; do + compose_cmd exec -T app-net-probe sh -ec 'printf "%s\n" "$1" | nc -u -w 1 "$2" 12201 >/dev/null 2>&1 || true' sh "$payload" "$target" >/dev/null + docker_cmd exec "$PG_ID" sh -ec 'printf "%s\n" "$1" | busybox nc -u -w 1 "$2" 12201 >/dev/null 2>&1 || true' sh "$payload" "$target" >/dev/null + record "INFO: attempted forged GELF from app namespace and Postgres to $target" +done +while read -r host_ip; do + [[ -n "$host_ip" ]] || continue + printf '%s\n' "$payload" | nc -u -w 1 "$host_ip" 12201 >/dev/null 2>&1 || true + record "INFO: attempted forged GELF from host interface $host_ip" +done < <(ip -o -4 addr show scope global | awk '{split($4,a,"/"); print a[1]}') +sleep 1 +final_metric=$(wait_metric 0) +[[ "$final_metric" == "$baseline" ]] || fail "isolation leak: GELF receive counter changed from $baseline to $final_metric" +record "PASS: app/Postgres/host probes delivered zero GELF packets and zero events" + +compose_cmd stop vector >/dev/null +max_ms=0 +for _ in 1 2 3 4 5 6 7 8 9 10; do + start_ms=$(date +%s%3N) + curl --fail --silent --show-error --max-time 5 http://127.0.0.1:18000/version >/dev/null || fail "node stopped serving while Vector was down" + end_ms=$(date +%s%3N) + elapsed=$((end_ms - start_ms)) + (( elapsed > max_ms )) && max_ms=$elapsed + done +node_restarts_after=$(docker_cmd inspect -f '{{.RestartCount}}' "$NODE_ID") +[[ "$node_restarts" == "$node_restarts_after" ]] || fail "node restarted with Vector stopped" +record "PASS: node served /version 10/10 with Vector down; max sampled latency=${max_ms}ms" +compose_cmd start vector >/dev/null +wait_vector +rm -f "$SAMPLER_STOP" +wait "$SAMPLER_PID" +SAMPLER_PID="" +peak_bytes=$(cat "$WORK/vector-peak-bytes") +record "INFO: sampled Vector peak RSS=$peak_bytes bytes (64 MiB configured limit)" +(( peak_bytes <= 64 * 1024 * 1024 )) || fail "Vector sampled RSS exceeded its 64 MiB memory cap" +record "PASS: complete Docker acceptance matrix" From a2f9316d03d69aed13857b4a322c6dc8aa18b4b6 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 16:24:39 +0000 Subject: [PATCH 06/21] fix(log-shipper): isolate gate summaries and harden acceptance --- .github/workflows/rust.yml | 1 + CHANGELOG.md | 2 +- README.md | 4 +- docker-compose.dstack-postgres.yaml | 77 +++++-- .../tc-959-betterstack-log-shipping.md | 10 +- log-shipper/fields.json | 9 +- log-shipper/vector.tests.yaml | 133 ++++++++++-- log-shipper/vector.yaml | 77 +++++-- package-lock.json | 48 +++++ package.json | 3 + scripts/node-vector-schema.mjs | 75 +++++-- scripts/node-vector-schema.test.mjs | 87 ++++++-- scripts/tc-959-vector-metrics.mjs | 70 +++++++ tc-959-node-docker-check.review.sh | 173 +++++++++++++--- tinycloud-node-server/src/lib.rs | 1 + tinycloud-node-server/src/main.rs | 15 +- tinycloud-node-server/src/tracing.rs | 193 +++++++++++++++++- 17 files changed, 837 insertions(+), 141 deletions(-) create mode 100644 package-lock.json create mode 100644 scripts/tc-959-vector-metrics.mjs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 089e4e78..6678ae1b 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -189,6 +189,7 @@ jobs: node-version: 20 - name: Check generated log allowlist and compose parity run: | + npm ci --ignore-scripts --no-audit --no-fund node scripts/node-vector-schema.mjs --check node scripts/node-vector-schema.mjs --check-compose node --test scripts/node-vector-schema.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 58bc67a5..ea1ce6e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog ## [Unreleased] -- TC-959: ship only labelled node/ingress logs through GELF/UDP to a fixed-address, isolated Vector socket; gate labels before metadata stripping, JSON parsing, and the existing allowlist. Vector publishes UDP on host loopback only, has no Docker-directory mount or socket, and runs with capped resources. Unlabelled services remain on capped `json-file` logs. Native tests cover transport, redaction, gates, and budgets; the review-only Docker acceptance script remains unrun. See `docs/runbooks/tc-959-betterstack-log-shipping.md`. +- TC-959: ship only labelled node/ingress logs through GELF/UDP to a fixed-address, isolated Vector socket; cap complete node JSON lines at 15,000 bytes including newline, with the panic hook on the same path; gate labels before metadata stripping, JSON parsing, and the existing allowlist. Gate reasons are summarized without content on an independent 1/min lane; ingress is capped at 26/min to preserve the 39/min, 0.473 GB/30d budget while app WARN remains 5/min. Vector publishes UDP on host loopback only, has no Docker-directory mount or socket, and runs with capped resources. Unlabelled services remain on capped `json-file`. - Security (TC-1132, TC-1080 Parts C–D): webhook registration and every delivery attempt share an egress policy covering special-use IPv4/IPv6, embedded IPv4 (mapped, compatible, NAT64 and 6to4), Teredo and site-local IPv6. Ambiguous local-use NAT64 is refused if any valid IPv4 interpretation is forbidden. Corrected public controls such as `192.2.1.1` and `198.51.0.1` are no longer over-blocked. Mixed DNS answers retain only permitted addresses. - Webhooks deny the resolved addresses of the configured network database, quota URL, space allowlist and explicit S3 endpoint on every port, including aliases and embedded IPv4 forms. The service deny-set refreshes every ten minutes; validation waits for an in-progress refresh, and failed/stale resolution closes egress until recovery. Temporary DNS/policy failures retry on the existing bounded schedule; invalid or forbidden callbacks dead-letter. Existing subscriptions whose destinations become forbidden dead-letter on the next attempt. diff --git a/README.md b/README.md index f90d456e..762b1a19 100644 --- a/README.md +++ b/README.md @@ -306,9 +306,9 @@ If the TinyCloud Protocol instance is not able to find or establish a connection ### Production log shipping (TC-959) -The node and ingress use the Docker GELF driver with exact `tc.logship` labels and send UDP to host loopback `127.0.0.1:12201`. Vector runs on its own fixed-address bridge network, publishes only that loopback UDP port, and has no Docker socket or host Docker-directory mount. Its first transform gates on `_tc.logship`; only then does it strip GELF metadata and parse the app's JSON line before the existing schema allowlist. Other services, including Postgres, stay unlabelled on capped `json-file` logs. Node and ingress do not depend on Vector; a missing Better Stack token leaves Vector idle. +The node and ingress use the Docker GELF driver with exact `tc.logship` labels and send UDP to host loopback `127.0.0.1:12201`. Vector runs on its own fixed-address bridge network, publishes only that loopback UDP port, and has no Docker socket or host Docker-directory mount. Its first transform gates on `_tc.logship`; only then does it strip GELF metadata and parse the app's JSON line before the existing schema allowlist. Other services, including Postgres, stay unlabelled on capped `json-file` logs. Node and ingress do not depend on Vector; a missing Better Stack token leaves Vector idle. The node JSON formatter and panic hook cap each complete line, including newline, at 15,000 bytes; ingress's upstream status/time lists have no explicit size bound in the checked config. -The separate panic, app ERROR, app WARN, and ingress lanes are capped at 2, 5, 5, and 27 events/minute. With the 281-byte final event limit, the monthly ceiling is `(2 + 5 + 5 + 27) × 60 × 24 × 30 × 281 = 473,428,800 bytes` (0.473 GB/30d), below the node's 0.48 GB share of the 3 GB free quota. Unknown routes collapse to ``. WARN and fixed gate counters bypass the ingress throttle. +The panic, app ERROR, app WARN, gate-summary, and ingress lanes are capped at 2, 5, 5, 1, and 26 events/minute. The one-per-minute gate summary carries four reason counts and no content; it cannot consume a real WARN slot. With the 281-byte final event limit, the monthly ceiling is `(2 + 5 + 5 + 1 + 26) × 60 × 24 × 30 × 281 = 473,428,800 bytes` (0.473 GB/30d), below the node's 0.48 GB share of the 3 GB free quota. Unknown routes collapse to ``. On the 2-vCPU/4-GiB/20-GB tdx.medium CVM, Vector is capped at 64 MiB and 0.25 CPU. Its data directory and token file live on an 8 MiB tmpfs; the Vector and ssh-gateway `json-file` logs are capped at `10m × 3` each (60 MB total). The review-only Docker acceptance script measures Vector peak memory and transport loss; neither has been measured by this change. diff --git a/docker-compose.dstack-postgres.yaml b/docker-compose.dstack-postgres.yaml index f0a51eee..27f76ae9 100644 --- a/docker-compose.dstack-postgres.yaml +++ b/docker-compose.dstack-postgres.yaml @@ -328,7 +328,7 @@ services: reason = .gate_reason if !includes(["malformed_envelope", "unknown_message", "unlabelled"], reason) { abort } minute = format_timestamp!(now(), format: "%Y-%m-%dT%H:%M") - . = {"event": "logship_gate_dropped_total", "reason": reason, "minute": minute, "count": 1} + . = {"minute": minute, "reason": reason, "count": 1} logship_unexpected_counter: type: remap inputs: [logship_result_route.unexpected] @@ -336,37 +336,70 @@ services: drop_on_abort: true source: | minute = format_timestamp!(now(), format: "%Y-%m-%dT%H:%M") - . = {"event": "logship_unexpected_label", "reason": "unexpected_label", "minute": minute, "count": 1} + . = {"minute": minute, "reason": "unexpected_label", "count": 1} + logship_summary_input: + type: remap + inputs: [logship_drop_counter, logship_unexpected_counter] + drop_on_error: true + drop_on_abort: true + source: | + reason = .reason + minute = .minute + count = .count + if !is_string(reason) || !is_string(minute) { abort } + if !includes(["malformed_envelope", "unexpected_label", "unknown_message", "unlabelled"], reason) { abort } + if !is_integer(count) { abort } + count = to_int(count) ?? 0 + if count < 1 || count > 1000000000 { abort } + . = { + "minute": .minute, + "malformed_envelope_count": 0, + "unknown_message_count": 0, + "unlabelled_count": 0, + "unexpected_label_count": 0 + } + if reason == "malformed_envelope" { .malformed_envelope_count = count } + else if reason == "unknown_message" { .unknown_message_count = count } + else if reason == "unlabelled" { .unlabelled_count = count } + else { .unexpected_label_count = count } logship_counter_reduce: type: reduce - inputs: [logship_drop_counter, logship_unexpected_counter] - group_by: [event, reason, minute] + inputs: [logship_summary_input] + group_by: [minute] expire_after_ms: 75000 end_every_period_ms: 60000 flush_period_ms: 1000 merge_strategies: - count: sum + malformed_envelope_count: sum + unknown_message_count: sum + unlabelled_count: sum + unexpected_label_count: sum logship_counter_output: type: remap inputs: [logship_counter_reduce] drop_on_error: true drop_on_abort: true source: | - event = .event - reason = .reason - count = .count - if !includes(["logship_gate_dropped_total", "logship_unexpected_label"], event) { abort } - if !includes(["malformed_envelope", "unexpected_label", "unknown_message", "unlabelled"], reason) { abort } - if !is_integer(count) { abort } - count = to_int(count) ?? 0 - if count < 1 || count > 1000000000 { abort } + malformed = .malformed_envelope_count + unknown = .unknown_message_count + unlabelled = .unlabelled_count + unexpected = .unexpected_label_count + if !is_integer(malformed) || !is_integer(unknown) || !is_integer(unlabelled) || !is_integer(unexpected) { abort } + malformed = to_int(malformed) ?? 0 + unknown = to_int(unknown) ?? 0 + unlabelled = to_int(unlabelled) ?? 0 + unexpected = to_int(unexpected) ?? 0 + if malformed < 0 || malformed > 1000000000 || unknown < 0 || unknown > 1000000000 || unlabelled < 0 || unlabelled > 1000000000 || unexpected < 0 || unexpected > 1000000000 { abort } + if malformed + unknown + unlabelled + unexpected < 1 { abort } . = { "level": "warn", "ts": format_timestamp!(now(), format: "%Y-%m-%dT%H:%M:%S%.3fZ"), - "event": event, + "event": "logship_gate_summary_total", "svc": "logship", - "reason": reason, - "count": count + "malformed_envelope_count": malformed, + "unknown_message_count": unknown, + "unlabelled_count": unlabelled, + "unexpected_label_count": unexpected } allowlist: type: remap @@ -525,7 +558,8 @@ services: route: panic: .event == "node_fatal_panic" app_error: .event == "node_error" - app_warning: .event == "node_warning" || .event == "logship_gate_dropped_total" || .event == "logship_unexpected_label" + app_warning: .event == "node_warning" + logship_summary: .event == "logship_gate_summary_total" ingress_summary: .event == "ingress_failure_count" || .event == "ingress_success_count" ingress_slow: .event == "ingress_request_slow" split_requests: @@ -616,14 +650,19 @@ services: inputs: [classify_by_budget.app_warning] window_secs: 60 threshold: 5 + throttle_logship_summary: + type: throttle + inputs: [classify_by_budget.logship_summary] + window_secs: 60 + threshold: 1 throttle_ingress: type: throttle inputs: [request_output, classify_by_budget.ingress_slow, slow_success] window_secs: 60 - threshold: 27 + threshold: 26 budget_size_guard: type: remap - inputs: [throttle_panic, throttle_app_error, throttle_app_warning, throttle_ingress] + inputs: [throttle_panic, throttle_app_error, throttle_app_warning, throttle_logship_summary, throttle_ingress] drop_on_error: true drop_on_abort: true source: | diff --git a/docs/runbooks/tc-959-betterstack-log-shipping.md b/docs/runbooks/tc-959-betterstack-log-shipping.md index c1ee2188..8bf34cd7 100644 --- a/docs/runbooks/tc-959-betterstack-log-shipping.md +++ b/docs/runbooks/tc-959-betterstack-log-shipping.md @@ -4,9 +4,9 @@ The node and `dstack-ingress` use Docker's `gelf` logging driver with the exact service labels `tc.logship=tinycloud-node` and `tc.logship=tinycloud-ingress`. Both send UDP to `127.0.0.1:12201`. Vector listens inside its own bridge network at fixed address `10.255.253.2`; that network is not shared with app or Postgres services and has egress for Better Stack. The only published Vector port is `127.0.0.1:12201:12201/udp`. Vector has no `docker.sock` or `/var/lib/docker` mount. There is no DockerRootDir pre-deploy check. -`logship_gate` is the first transform. It reads `_tc.logship` as a string and accepts only the two exact labels. Missing/empty labels increment the fixed `logship_gate_dropped_total{reason="unlabelled"}` counter without content; every other label emits one fixed `logship_unexpected_label` event, throttled to about one per minute. After the gate, `strip_gelf_fields` reconstructs the event from only the approved label and GELF `message`, dropping all underscore-prefixed fields and dockerd metadata. `parse_gelf_message` then parses `.message`, the app's JSON line. The existing inner allowlist, schema projection, event enum, priority lanes, throttle, `budget_size_guard`, and sole sink input remain unchanged. Ingress non-JSON stderr is dropped after the gate. +`logship_gate` is the first transform. It reads `_tc.logship` as a string and accepts only the two exact labels. Unlabelled, malformed, and unknown-message inputs increment bounded counters; wrong labels increment a separate bounded counter. One `logship_gate_summary_total` event per minute carries the four reason counts without source content. After the gate, `strip_gelf_fields` reconstructs the event from only the approved label and GELF `message`, dropping all underscore-prefixed fields and dockerd metadata. `parse_gelf_message` then parses `.message`, the app's JSON line. The existing inner allowlist and schema projection feed the existing priority lanes and `budget_size_guard`; gate summaries have an independent lane, and the guard remains the sink's sole input. Ingress non-JSON stderr is dropped after the gate. -Docker splits lines over 16 KiB into GELF fragments. An incomplete fragment does not parse as the app JSON line and increments the bounded malformed-envelope counter; no partial text is forwarded. The GELF socket source accepts chunked and gzip-compressed messages, with no checkpoint or replay state. +The node JSON formatter caps the complete serialized line, including its newline, at 15,000 bytes by truncating the longest string values with a fixed marker; if a line still cannot fit or parse, it emits one fixed valid JSON marker record. The panic hook uses the same path. This keeps node lines below Docker's 16 KiB GELF fragmentation threshold. Ingress logs only fixed route/method maps, protocol, and numeric request data rather than raw paths or headers. Its `$upstream_status` and `$upstream_*_time` variables can contain comma-separated retry values; the checked ingress files set no explicit retry/list bound, so a strict <16 KiB ingress-line bound is not proven. GELF reassembles complete chunks; incomplete frames are rejected rather than forwarding partial text. There is no checkpoint or replay state. The node and ingress do not depend on Vector. Their GELF logging driver is UDP and does not require Vector to start. A missing Better Stack token leaves Vector idle. Postgres and every non-shipped service remain unlabelled and use capped `json-file` logs. @@ -18,9 +18,9 @@ At go-live, create a Better Stack alert for `event = node_fatal_panic` and one s ## Priority and monthly volume budget -The panic lane allows 2 events/minute; node app ERROR and WARN each allow 5/minute; ingress allows 27/minute. WARN and fixed gate counters pass through their own 5/minute warning lane, then feed `budget_size_guard` directly without entering the ingress throttle. The ingress lane carries per-minute `(route, status_class)` failure counts, per-minute route-only success counts, and slow requests. A 2xx/slow request is counted and marked slow; route cardinality is bounded to 35 known templates plus ``. +The panic lane allows 2 events/minute; node app ERROR and WARN each allow 5/minute; the gate summary allows 1/minute; ingress allows 26/minute. The four gate-reason counts share one content-free summary event and cannot consume the 5/minute app WARN allowance. Each lane feeds `budget_size_guard` directly without entering the ingress throttle. The ingress lane carries per-minute `(route, status_class)` failure counts, per-minute route-only success counts, and slow requests. A 2xx/slow request is counted and marked slow; route cardinality is bounded to 35 known templates plus ``. -At the 281-byte final event-size limit, the worst case is `(2 + 5 + 5 + 27) × 60 × 24 × 30 × 281 = 473,428,800 bytes` (0.473 GB/30d), within the node's 0.48 GB allocation of the shared 3 GB free quota. The individual lane ceilings are 24,278,400 bytes for panic, 60,696,000 bytes each for app ERROR and WARN, and 327,758,400 bytes for ingress. These are hard ceilings assuming every allowed event reaches the sink; shared quota use by other sources is separate. +At the 281-byte final event-size limit, the worst case is `(2 + 5 + 5 + 1 + 26) × 60 × 24 × 30 × 281 = 473,428,800 bytes` (0.473 GB/30d), within the node's 0.48 GB allocation of the shared 3 GB free quota. The per-lane ceilings are 24,278,400 bytes for panic, 60,696,000 bytes each for app ERROR and WARN, 12,139,200 bytes for the gate summary, and 315,619,200 bytes for ingress. These are hard ceilings assuming every allowed event reaches the sink; shared quota use by other sources is separate. ## CVM resource and disk budget @@ -34,4 +34,4 @@ Only node and ingress logs carry `tc.logship`. Vector and ssh-gateway retain cap ## Verification and deploy gate -CI checks the canonical common + `services.tinycloud-node` schema, generated VRL, byte-identical embedded Compose config, packaging restrictions (`configs.*.file`, `env_file`, and relative bind sources), and native Vector 0.59.0 validation/tests after verifying the release tarball SHA256. The review-only `tc-959-node-docker-check.review.sh` covers 5× each Vector restart, crash/restart-policy, and recreate; 3× Vector-down 500-line bursts; app service continuity; source-metric isolation; external-interface probes; and Vector memory. It includes a test-only node override with a local Postgres, a Static test key, TEE Off, and no dstack socket. It must only run after explicit Docker authorization. Amos owns production deployment and go-live alerts; do not use this runbook to deploy or change secrets. +CI checks the canonical common + `services.tinycloud-node` schema, generated VRL, byte-identical embedded Compose config, structural packaging restrictions (`configs.*.file`, `env_file`, and relative bind sources), and native Vector 0.59.0 validation/tests after verifying the release tarball SHA256. Native preflight uses a disposable dummy token fixture only; the acceptance override config is mode 0644 for Vector UID 10001, and the rendered ingress producer command asserts `$$` escaping. The review-only `tc-959-node-docker-check.review.sh` covers 5× each Vector restart, crash/restart-policy, and recreate; 3× Vector-down 500-line bursts; app service continuity; a host-loopback and a logship-network positive control (each source +1 and shipped +1); then exact zero source/shipped deltas for node, ingress, Postgres and host-interface probes; `/version` while Vector is stopped; and Vector memory. Its Prometheus parser matches the exact GELF source label set and reads the value field before any timestamp. The test-only node override uses local Postgres, a Static test key, TEE Off, and no dstack socket. Only the native preflight and Docker-free dry run were executed; Docker acceptance remains unverified and requires explicit Docker authorization. Amos owns production deployment and go-live alerts; do not use this runbook to deploy or change secrets. diff --git a/log-shipper/fields.json b/log-shipper/fields.json index 364c087c..bf0d6990 100644 --- a/log-shipper/fields.json +++ b/log-shipper/fields.json @@ -70,8 +70,7 @@ "ingress_failure_count", "ingress_request_slow", "ingress_success_count", - "logship_gate_dropped_total", - "logship_unexpected_label" + "logship_gate_summary_total" ], "fields": { "svc": { "type": "enum", "values": ["api", "ingress", "logship"] }, @@ -83,8 +82,10 @@ "request_count": { "type": "integer", "min": 1, "max": 1000000 }, "bytes_sent": { "type": "integer", "min": 0, "max": 1000000000 }, "upstream_status": { "type": "string", "regex": "^(|[1-5][0-9]{2})$", "max_len": 3 }, - "count": { "type": "integer", "min": 1, "max": 1000000000 }, - "reason": { "type": "enum", "values": ["malformed_envelope", "unexpected_label", "unknown_message", "unlabelled"] }, + "malformed_envelope_count": { "type": "integer", "min": 0, "max": 1000000000 }, + "unknown_message_count": { "type": "integer", "min": 0, "max": 1000000000 }, + "unlabelled_count": { "type": "integer", "min": 0, "max": 1000000000 }, + "unexpected_label_count": { "type": "integer", "min": 0, "max": 1000000000 }, "event_code": { "type": "enum", "values": ["invocation_replay_insert_failed", "invocation_replay_cleanup_failed", "dstack_get_info_failed", "share_local_tee_identity", "retention_hook_delivery_prune_failed", "retention_dead_letter_prune_failed", "retention_signed_ticket_prune_failed", "quota_invalid_body", "quota_fallback", "tunnel_state_load_failed", "tunnel_address_invalid", "link_state_load_failed", "link_bind_invalid", "link_bind_failed", "link_bind_state_write_failed", "link_tls_config_failed", "link_listener_error", "link_listener_state_write_failed", "link_auto_renew_config_failed", "link_auto_renew_failed", "link_cert_reload_failed", "link_listener_cert_reload_failed", "link_disable_failed", "link_connection_error", "link_listener_accept_failed", "webhook_delivery_failed", "webhook_dispatcher_failed", "share_authorization_failed", "share_native_authorization_denied", "share_native_kv_put_failed", "share_v2_readiness_failed", "share_invitation_key_missing", "share_invitation_key_mismatch", "share_policy_denied", "attestation_quote_failed", "hook_events_read_failed", "hook_rows_load_failed", "missing_committed_hook_row", "webhook_enqueue_failed", "tunnel_connection_prepare_failed", "tunnel_idle_timeout", "tunnel_read_failed", "tunnel_malformed_frame", "tunnel_relay_error", "tunnel_write_failed", "tunnel_state_record_failed", "tunnel_state_clear_failed", "tunnel_auth_rejected", "other"] } } }, diff --git a/log-shipper/vector.tests.yaml b/log-shipper/vector.tests.yaml index a8f2e9ab..0d511aa4 100644 --- a/log-shipper/vector.tests.yaml +++ b/log-shipper/vector.tests.yaml @@ -701,7 +701,7 @@ tests: source: | assert_eq!(.event, "ingress_failure_count") assert_eq!(.route, "/info") - - name: "unlabelled GELF canaries are reduced to bounded counters" + - name: "unlabelled GELF canaries are reduced into a bounded summary" inputs: - &unlabelled_canary insert_at: logship_gate @@ -715,14 +715,16 @@ tests: conditions: - type: vrl source: | - assert_eq!(.event, "logship_gate_dropped_total") - assert_eq!(.reason, "unlabelled") - assert_eq!(.count, 2) + assert_eq!(.event, "logship_gate_summary_total") + assert_eq!(.unlabelled_count, 2) + assert_eq!(.malformed_envelope_count, 0) + assert_eq!(.unknown_message_count, 0) + assert_eq!(.unexpected_label_count, 0) assert!(!contains(encode_json(.), "signed-kv-ticket-probe")) assert!(!contains(encode_json(.), "auth-probe")) assert!(!contains(encode_json(.), "did:key:owner-probe")) assert!(!contains(encode_json(.), "space-probe@example.invalid")) - - name: "wrong and postgres labels produce only a fixed unexpected-label event" + - name: "wrong and postgres labels increment only the fixed unexpected-label summary" inputs: - insert_at: logship_gate type: log @@ -740,9 +742,11 @@ tests: conditions: - type: vrl source: | - assert_eq!(.event, "logship_unexpected_label") - assert_eq!(.reason, "unexpected_label") - assert_eq!(.count, 2) + assert_eq!(.event, "logship_gate_summary_total") + assert_eq!(.unexpected_label_count, 2) + assert_eq!(.malformed_envelope_count, 0) + assert_eq!(.unknown_message_count, 0) + assert_eq!(.unlabelled_count, 0) assert!(!contains(encode_json(.), "private-untrusted-label")) assert!(!contains(encode_json(.), "postgres")) assert!(!contains(encode_json(.), "ticket-probe")) @@ -759,9 +763,11 @@ tests: conditions: - type: vrl source: | - assert_eq!(.event, "logship_gate_dropped_total") - assert_eq!(.reason, "unlabelled") - assert_eq!(.count, 1) + assert_eq!(.event, "logship_gate_summary_total") + assert_eq!(.unlabelled_count, 1) + assert_eq!(.malformed_envelope_count, 0) + assert_eq!(.unknown_message_count, 0) + assert_eq!(.unexpected_label_count, 0) assert!(!contains(encode_json(.), "node fatal panic")) - name: "split GELF fragments with invalid JSON are counted without content" inputs: @@ -777,9 +783,11 @@ tests: conditions: - type: vrl source: | - assert_eq!(.event, "logship_gate_dropped_total") - assert_eq!(.reason, "malformed_envelope") - assert_eq!(.count, 1) + assert_eq!(.event, "logship_gate_summary_total") + assert_eq!(.malformed_envelope_count, 1) + assert_eq!(.unknown_message_count, 0) + assert_eq!(.unlabelled_count, 0) + assert_eq!(.unexpected_label_count, 0) assert!(!contains(encode_json(.), "token-fragment-probe")) - name: "labelled ingress non-JSON messages are dropped after the gate" inputs: @@ -795,8 +803,99 @@ tests: conditions: - type: vrl source: | - assert_eq!(.event, "logship_gate_dropped_total") - assert_eq!(.reason, "malformed_envelope") - assert_eq!(.count, 1) + assert_eq!(.event, "logship_gate_summary_total") + assert_eq!(.malformed_envelope_count, 1) + assert_eq!(.unknown_message_count, 0) + assert_eq!(.unlabelled_count, 0) + assert_eq!(.unexpected_label_count, 0) assert!(!contains(encode_json(.), "private-non-json-ingress-probe")) assert!(!contains(encode_json(.), "container-non-json-probe")) + - name: "all gate reasons use their summary lane without consuming five WARN slots" + inputs: + - insert_at: logship_gate + type: log + log_fields: + message: "unlabelled-probe" + - insert_at: logship_gate + type: log + log_fields: + '."_tc.logship"': "wrong-label-probe" + message: "wrong-label-probe" + - insert_at: logship_gate + type: log + log_fields: + '."_tc.logship"': "tinycloud-node" + message: "malformed-probe" + - insert_at: logship_gate + type: log + log_fields: + '."_tc.logship"': "tinycloud-node" + message: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"unknown-message-probe"}}' + - &summary_warn + insert_at: logship_gate + type: log + log_fields: + '."_tc.logship"': "tinycloud-node" + message: '{"timestamp":"2026-10-14T10:00:00.000Z","level":"WARN","fields":{"message":"invocation replay insert failed"}}' + - *summary_warn + - *summary_warn + - *summary_warn + - *summary_warn + outputs: + - extract_from: logship_counter_output + conditions: + - type: vrl + source: | + assert_eq!(.event, "logship_gate_summary_total") + assert_eq!(.svc, "logship") + assert_eq!(.level, "warn") + assert_eq!(.malformed_envelope_count, 1) + assert_eq!(.unexpected_label_count, 1) + assert_eq!(.unknown_message_count, 1) + assert_eq!(.unlabelled_count, 1) + assert!(!exists(.reason)) + assert!(!exists(.message)) + - extract_from: throttle_logship_summary + conditions: + - type: vrl + source: | + assert_eq!(.event, "logship_gate_summary_total") + assert_eq!(.malformed_envelope_count, 1) + assert_eq!(.unexpected_label_count, 1) + assert_eq!(.unknown_message_count, 1) + assert_eq!(.unlabelled_count, 1) + - extract_from: throttle_app_warning + conditions: + - type: vrl + source: | + assert_eq!(.event, "node_warning") + assert_eq!(.event_code, "invocation_replay_insert_failed") + assert_eq!(.level, "warn") + - extract_from: throttle_app_warning + conditions: + - type: vrl + source: | + assert_eq!(.event, "node_warning") + assert_eq!(.event_code, "invocation_replay_insert_failed") + assert_eq!(.level, "warn") + - extract_from: throttle_app_warning + conditions: + - type: vrl + source: | + assert_eq!(.event, "node_warning") + assert_eq!(.event_code, "invocation_replay_insert_failed") + assert_eq!(.level, "warn") + - extract_from: throttle_app_warning + conditions: + - type: vrl + source: | + assert_eq!(.event, "node_warning") + assert_eq!(.event_code, "invocation_replay_insert_failed") + assert_eq!(.level, "warn") + - extract_from: throttle_app_warning + conditions: + - type: vrl + source: | + assert_eq!(.event, "node_warning") + assert_eq!(.event_code, "invocation_replay_insert_failed") + assert_eq!(.level, "warn") diff --git a/log-shipper/vector.yaml b/log-shipper/vector.yaml index 36f57e83..4b90e0a3 100644 --- a/log-shipper/vector.yaml +++ b/log-shipper/vector.yaml @@ -170,7 +170,7 @@ transforms: reason = .gate_reason if !includes(["malformed_envelope", "unknown_message", "unlabelled"], reason) { abort } minute = format_timestamp!(now(), format: "%Y-%m-%dT%H:%M") - . = {"event": "logship_gate_dropped_total", "reason": reason, "minute": minute, "count": 1} + . = {"minute": minute, "reason": reason, "count": 1} logship_unexpected_counter: type: remap inputs: [logship_result_route.unexpected] @@ -178,37 +178,70 @@ transforms: drop_on_abort: true source: | minute = format_timestamp!(now(), format: "%Y-%m-%dT%H:%M") - . = {"event": "logship_unexpected_label", "reason": "unexpected_label", "minute": minute, "count": 1} + . = {"minute": minute, "reason": "unexpected_label", "count": 1} + logship_summary_input: + type: remap + inputs: [logship_drop_counter, logship_unexpected_counter] + drop_on_error: true + drop_on_abort: true + source: | + reason = .reason + minute = .minute + count = .count + if !is_string(reason) || !is_string(minute) { abort } + if !includes(["malformed_envelope", "unexpected_label", "unknown_message", "unlabelled"], reason) { abort } + if !is_integer(count) { abort } + count = to_int(count) ?? 0 + if count < 1 || count > 1000000000 { abort } + . = { + "minute": .minute, + "malformed_envelope_count": 0, + "unknown_message_count": 0, + "unlabelled_count": 0, + "unexpected_label_count": 0 + } + if reason == "malformed_envelope" { .malformed_envelope_count = count } + else if reason == "unknown_message" { .unknown_message_count = count } + else if reason == "unlabelled" { .unlabelled_count = count } + else { .unexpected_label_count = count } logship_counter_reduce: type: reduce - inputs: [logship_drop_counter, logship_unexpected_counter] - group_by: [event, reason, minute] + inputs: [logship_summary_input] + group_by: [minute] expire_after_ms: 75000 end_every_period_ms: 60000 flush_period_ms: 1000 merge_strategies: - count: sum + malformed_envelope_count: sum + unknown_message_count: sum + unlabelled_count: sum + unexpected_label_count: sum logship_counter_output: type: remap inputs: [logship_counter_reduce] drop_on_error: true drop_on_abort: true source: | - event = .event - reason = .reason - count = .count - if !includes(["logship_gate_dropped_total", "logship_unexpected_label"], event) { abort } - if !includes(["malformed_envelope", "unexpected_label", "unknown_message", "unlabelled"], reason) { abort } - if !is_integer(count) { abort } - count = to_int(count) ?? 0 - if count < 1 || count > 1000000000 { abort } + malformed = .malformed_envelope_count + unknown = .unknown_message_count + unlabelled = .unlabelled_count + unexpected = .unexpected_label_count + if !is_integer(malformed) || !is_integer(unknown) || !is_integer(unlabelled) || !is_integer(unexpected) { abort } + malformed = to_int(malformed) ?? 0 + unknown = to_int(unknown) ?? 0 + unlabelled = to_int(unlabelled) ?? 0 + unexpected = to_int(unexpected) ?? 0 + if malformed < 0 || malformed > 1000000000 || unknown < 0 || unknown > 1000000000 || unlabelled < 0 || unlabelled > 1000000000 || unexpected < 0 || unexpected > 1000000000 { abort } + if malformed + unknown + unlabelled + unexpected < 1 { abort } . = { "level": "warn", "ts": format_timestamp!(now(), format: "%Y-%m-%dT%H:%M:%S%.3fZ"), - "event": event, + "event": "logship_gate_summary_total", "svc": "logship", - "reason": reason, - "count": count + "malformed_envelope_count": malformed, + "unknown_message_count": unknown, + "unlabelled_count": unlabelled, + "unexpected_label_count": unexpected } allowlist: type: remap @@ -367,7 +400,8 @@ transforms: route: panic: .event == "node_fatal_panic" app_error: .event == "node_error" - app_warning: .event == "node_warning" || .event == "logship_gate_dropped_total" || .event == "logship_unexpected_label" + app_warning: .event == "node_warning" + logship_summary: .event == "logship_gate_summary_total" ingress_summary: .event == "ingress_failure_count" || .event == "ingress_success_count" ingress_slow: .event == "ingress_request_slow" split_requests: @@ -458,14 +492,19 @@ transforms: inputs: [classify_by_budget.app_warning] window_secs: 60 threshold: 5 + throttle_logship_summary: + type: throttle + inputs: [classify_by_budget.logship_summary] + window_secs: 60 + threshold: 1 throttle_ingress: type: throttle inputs: [request_output, classify_by_budget.ingress_slow, slow_success] window_secs: 60 - threshold: 27 + threshold: 26 budget_size_guard: type: remap - inputs: [throttle_panic, throttle_app_error, throttle_app_warning, throttle_ingress] + inputs: [throttle_panic, throttle_app_error, throttle_app_warning, throttle_logship_summary, throttle_ingress] drop_on_error: true drop_on_abort: true source: | diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..ce9a7fc2 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,48 @@ +{ + "name": "tinycloud-node", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tinycloud-node", + "dependencies": { + "zod": "3.22.4", + "zod-to-json-schema": "3.22.4" + }, + "devDependencies": { + "yaml": "2.8.1" + } + }, + "node_modules/yaml": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.1.tgz", + "integrity": "sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==", + "dev": true, + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + } + }, + "node_modules/zod": { + "version": "3.22.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.22.4.tgz", + "integrity": "sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.22.4", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.22.4.tgz", + "integrity": "sha512-2Ed5dJ+n/O3cU383xSY28cuVi0BCQhF8nYqWU5paEpl7fVdqdAmiLdqLyfblbNdfOFwFfi/mqU4O1pwc60iBhQ==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.22.4" + } + } + } +} diff --git a/package.json b/package.json index cb35143d..fbb98368 100644 --- a/package.json +++ b/package.json @@ -11,5 +11,8 @@ "dependencies": { "zod": "3.22.4", "zod-to-json-schema": "3.22.4" + }, + "devDependencies": { + "yaml": "2.8.1" } } diff --git a/scripts/node-vector-schema.mjs b/scripts/node-vector-schema.mjs index b66ca03b..263227ff 100644 --- a/scripts/node-vector-schema.mjs +++ b/scripts/node-vector-schema.mjs @@ -1,5 +1,7 @@ import { readFileSync, writeFileSync } from "node:fs"; import { fileURLToPath } from "node:url"; +import { isAbsolute } from "node:path"; +import { parse } from "yaml"; const root = fileURLToPath(new URL("../", import.meta.url)); const schemaPath = `${root}log-shipper/fields.json`; @@ -242,37 +244,75 @@ export function checkVectorFile(schema, yaml = readFileSync(vectorPath, "utf8")) const requiredBudgetConfig = [ "panic: .event == \"node_fatal_panic\"", "app_error: .event == \"node_error\"", - "app_warning: .event == \"node_warning\" || .event == \"logship_gate_dropped_total\" || .event == \"logship_unexpected_label\"", + "app_warning: .event == \"node_warning\"", + "logship_summary: .event == \"logship_gate_summary_total\"", "inputs: [logship_gate_route.node, logship_gate_route.ingress]", "inputs: [logship_gate_route.rejected, logship_inner]", + "inputs: [logship_drop_counter, logship_unexpected_counter]", + "inputs: [logship_summary_input]", + "group_by: [minute]", + "malformed_envelope_count: sum", + "unknown_message_count: sum", + "unlabelled_count: sum", + "unexpected_label_count: sum", "inputs: [allowlist, logship_counter_output]", "inputs: [logship_result_route.validated]", - "group_by: [route, status_class, event, minute]", "inputs: [classify_by_budget.panic]", "threshold: 2", "inputs: [classify_by_budget.app_error]", "threshold: 5", "inputs: [classify_by_budget.app_warning]", + "inputs: [classify_by_budget.logship_summary]", + "threshold: 1", "inputs: [request_output, classify_by_budget.ingress_slow, slow_success]", - "threshold: 27", + "threshold: 26", "expire_after_ms: 75000", "end_every_period_ms: 60000", "flush_period_ms: 1000", "if length(encode_json(.)) > 280 { abort }", - "inputs: [throttle_panic, throttle_app_error, throttle_app_warning, throttle_ingress]", + "inputs: [throttle_panic, throttle_app_error, throttle_app_warning, throttle_logship_summary, throttle_ingress]", ]; if (requiredBudgetConfig.some((line) => !yaml.includes(line))) throw new Error("Vector priority lanes, counters, reduce timers, or event-size bound differ from reviewed budget"); - const monthlyBudgetBytes = (2 + 5 + 5 + 27) * 60 * 24 * 30 * 281; - if (monthlyBudgetBytes > 480000000) throw new Error(`Vector event budget exceeds 0.48 GB/month: ${monthlyBudgetBytes} bytes`); - if ((yaml.match(/threshold: 5\n/g) ?? []).length !== 2) throw new Error("node error and warning lanes must each be capped at five events/minute"); + const monthlyBudgetBytes = (2 + 5 + 5 + 1 + 26) * 60 * 24 * 30 * 281; +} + +function isComposeMapping(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); } export function checkComposeDeploymentPackaging(compose) { - if (/^\s*env_file\s*:/m.test(compose)) throw new Error("Compose packaging must not use env_file"); - const configs = compose.match(/^configs:\n([\s\S]*?)(?=\n\S|$)/)?.[1]; - if (configs && /^\s+file\s*:/m.test(configs)) throw new Error("Compose packaging must not use configs.*.file"); - if (/^\s*-\s*(?:\.\/|\.\.\/)/m.test(compose) || /^\s*source:\s*(?!\/|\|)[^\s]+/m.test(compose)) { - throw new Error("Compose packaging must not use relative bind sources"); + let definition; + try { + definition = parse(compose); + } catch (error) { + throw new Error("Compose packaging must be valid YAML", { cause: error }); + } + if (!isComposeMapping(definition)) throw new Error("Compose packaging must be a YAML mapping"); + + for (const config of Object.values(definition.configs ?? {})) { + if (isComposeMapping(config) && Object.hasOwn(config, "file")) { + throw new Error("Compose packaging must not use configs.*.file"); + } + } + + const namedVolumes = definition.volumes ?? {}; + for (const service of Object.values(definition.services ?? {})) { + if (!isComposeMapping(service)) continue; + if (Object.hasOwn(service, "env_file")) throw new Error("Compose packaging must not use env_file"); + if (!Array.isArray(service.volumes)) continue; + + for (const volume of service.volumes) { + if (typeof volume === "string") { + const separator = volume.indexOf(":"); + if (separator < 0) continue; + const source = volume.slice(0, separator); + if (source && (isAbsolute(source) || Object.hasOwn(namedVolumes, source))) continue; + throw new Error("Compose packaging must not use relative bind sources"); + } + if (isComposeMapping(volume) && volume.type === "bind" && (typeof volume.source !== "string" || !isAbsolute(volume.source))) { + throw new Error("Compose packaging must not use relative bind sources"); + } + } } } @@ -311,19 +351,20 @@ export function checkNodeSchema(schema) { if (JSON.stringify(schema.common.level) !== JSON.stringify({ type: "enum", values: ["debug", "info", "warn", "error"] })) throw new Error("common.level differs from the emitted level allowlist"); if (JSON.stringify(schema.common.ts) !== JSON.stringify({ type: "string", regex: "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z$", max_len: 24 })) throw new Error("common.ts differs from the emitted timestamp format"); if (schema.common.event?.type !== "event") throw new Error("common.event must use event type"); - if (service.events.join(",") !== "node_fatal_panic,node_warning,node_error,ingress_failure_count,ingress_request_slow,ingress_success_count,logship_gate_dropped_total,logship_unexpected_label") throw new Error("tinycloud-node event set differs from the reviewed set"); + if (service.events.join(",") !== "node_fatal_panic,node_warning,node_error,ingress_failure_count,ingress_request_slow,ingress_success_count,logship_gate_summary_total") throw new Error("tinycloud-node event set differs from the reviewed set"); if (Object.values(sourceMessages).some(([, code]) => !new Set(service.fields.event_code?.values ?? []).has(code))) throw new Error("source message map contains an event_code missing from fields.json"); if (Object.keys(service.fields).sort().join(",") !== [ - "bytes_sent", "count", "event_code", "method", "reason", "request_count", "request_time_seconds", "route", "status", "status_class", "svc", "upstream_status", + "bytes_sent", "event_code", "malformed_envelope_count", "method", "request_count", "request_time_seconds", "route", "status", "status_class", "svc", "unknown_message_count", "unlabelled_count", "unexpected_label_count", "upstream_status", ].sort().join(",")) throw new Error("tinycloud-node fields differ from the reviewed field set"); if (service.fields.request_count?.type !== "integer" || service.fields.request_count.min !== 1 || service.fields.request_count.max !== 1000000) throw new Error("request_count must remain bounded to 1..1000000"); - if (service.fields.count?.type !== "integer" || service.fields.count.min !== 1 || service.fields.count.max !== 1000000000) throw new Error("logship count must remain bounded to 1..1000000000"); - if (JSON.stringify(service.fields.reason) !== JSON.stringify({ type: "enum", values: ["malformed_envelope", "unexpected_label", "unknown_message", "unlabelled"] })) throw new Error("logship reason enum differs from emitted values"); + for (const field of ["malformed_envelope_count", "unknown_message_count", "unlabelled_count", "unexpected_label_count"]) { + if (JSON.stringify(service.fields[field]) !== JSON.stringify({ type: "integer", min: 0, max: 1000000000 })) throw new Error(`${field} must be bounded to 0..1000000000`); + } if (service.fields.route?.type !== "enum" || service.fields.route.values.length !== 36 || !service.fields.route.values.includes("")) throw new Error("route output must be a 36-value bounded enum including "); if (JSON.stringify(service.fields.svc) !== JSON.stringify({ type: "enum", values: ["api", "ingress", "logship"] })) throw new Error("svc enum differs from emitted values"); if (JSON.stringify(service.fields.status_class) !== JSON.stringify({ type: "enum", values: ["1xx", "2xx", "3xx", "4xx", "5xx"] })) throw new Error("status_class enum differs from emitted values"); if (service.fields.event_code?.type !== "enum" || !service.fields.event_code.values.includes("other")) throw new Error("event_code must preserve the fixed panic other value"); - for (const event of ["logship_gate_dropped_total", "logship_unexpected_label"]) { + for (const event of ["logship_gate_summary_total"]) { if (!service.events.includes(event)) throw new Error(`tinycloud-node is missing ${event}`); } } diff --git a/scripts/node-vector-schema.test.mjs b/scripts/node-vector-schema.test.mjs index 6a285119..8ad7d007 100644 --- a/scripts/node-vector-schema.test.mjs +++ b/scripts/node-vector-schema.test.mjs @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import test from "node:test"; -import { checkComposeDeploymentPackaging, checkVectorFile, renderAllowlist, renderComposeVectorConfig } from "./node-vector-schema.mjs"; +import { checkComposeDeploymentPackaging, checkComposeVectorConfig, checkVectorFile, renderAllowlist, renderComposeVectorConfig } from "./node-vector-schema.mjs"; const workflow = readFileSync(new URL("../.github/workflows/rust.yml", import.meta.url), "utf8"); const schema = JSON.parse(readFileSync(new URL("../log-shipper/fields.json", import.meta.url), "utf8")); @@ -94,16 +94,44 @@ test("CI runs native Vector validation and tests after checking the pinned relea assert.match(workflow, /test log-shipper\/vector\.yaml log-shipper\/vector\.tests\.yaml/); }); -test("deployed compose rejects file-backed configs, env_file, and relative bind sources", () => { - for (const invalid of [ - "services:\n vector:\n env_file: .env", - "configs:\n vector:\n file: ./vector.yaml", - "services:\n vector:\n volumes:\n - ./vector.yaml:/etc/vector/vector.yaml:ro", - "services:\n vector:\n volumes:\n - type: bind\n source: ../vector.yaml\n target: /etc/vector/vector.yaml", - "services:\n vector:\n volumes:\n - type: bind\n source: ${LOCAL_CONFIG_PATH}\n target: /etc/vector/vector.yaml", - ]) { - assert.throws(() => checkComposeDeploymentPackaging(invalid), /Compose packaging/); - } +const minimalCompose = `services: + vector: + environment: + VECTOR_CONFIG_YAML: | + sources: {} + entrypoint: ["vector"] +`; +test("rejects an appended block-style configs.file entry", () => { + assert.throws( + () => checkComposeVectorConfig(`${minimalCompose}\nconfigs:\n x:\n file: ./outside.yaml\n`, "sources: {}"), + /configs\.\*\.file/, + ); +}); +test("rejects flow-style configs.file entries", () => { + assert.throws( + () => checkComposeVectorConfig(`${minimalCompose}\nconfigs: {x: {file: ./outside.yaml}}\n`, "sources: {}"), + /configs\.\*\.file/, + ); +}); +test("rejects quoted service env_file values", () => { + assert.throws( + () => checkComposeVectorConfig(`${minimalCompose} probe:\n env_file: '.env'\n`, "sources: {}"), + /env_file/, + ); +}); +test("rejects quoted relative short bind sources", () => { + assert.throws( + () => checkComposeVectorConfig(`${minimalCompose} volumes:\n - './vector.yaml:/etc/vector/vector.yaml:ro'\n`, "sources: {}"), + /relative bind sources/, + ); +}); +test("rejects flow-style relative long bind sources", () => { + assert.throws( + () => checkComposeVectorConfig(`${minimalCompose} volumes: [{type: bind, source: ../vector.yaml, target: /etc/vector/vector.yaml}]\n`, "sources: {}"), + /relative bind sources/, + ); +}); +test("accepts canonical Compose packaging", () => { assert.doesNotThrow(() => checkComposeDeploymentPackaging(compose)); }); test("compose embeds source Vector config without allowing Compose to interpolate its regexes", () => { @@ -132,19 +160,22 @@ test("missing Better Stack token leaves node and ingress startup independent", ( "allowed_envs sync must not gate compose startup", ); }); -test("WARN lane bypasses the ingress throttle and stays behind the size guard", () => { +test("WARN, gate-summary, and ingress lanes stay independent behind the size guard", () => { assert.match(vector, /throttle_app_warning:\n type: throttle\n inputs: \[classify_by_budget\.app_warning\]\n window_secs: 60\n threshold: 5/); - assert.match(vector, /throttle_ingress:\n type: throttle\n inputs: \[request_output, classify_by_budget\.ingress_slow, slow_success\]\n window_secs: 60\n threshold: 27/); - assert.match(vector, /budget_size_guard:\n type: remap\n inputs: \[throttle_panic, throttle_app_error, throttle_app_warning, throttle_ingress\]/); + assert.match(vector, /throttle_logship_summary:\n type: throttle\n inputs: \[classify_by_budget\.logship_summary\]\n window_secs: 60\n threshold: 1/); + assert.match(vector, /throttle_ingress:\n type: throttle\n inputs: \[request_output, classify_by_budget\.ingress_slow, slow_success\]\n window_secs: 60\n threshold: 26/); + assert.match(vector, /budget_size_guard:\n type: remap\n inputs: \[throttle_panic, throttle_app_error, throttle_app_warning, throttle_logship_summary, throttle_ingress\]/); assert.match(vector, /sinks:\n betterstack:\n type: http\n inputs: \[budget_size_guard\]/); + assert.equal((2 + 5 + 5 + 1 + 26) * 60 * 24 * 30 * 281, 473428800); }); -test("node schema includes bounded fixed logship counter events", () => { - assert.ok(schema.services["tinycloud-node"].events.includes("logship_gate_dropped_total")); - assert.ok(schema.services["tinycloud-node"].events.includes("logship_unexpected_label")); - assert.deepEqual(schema.services["tinycloud-node"].fields.svc.values, ["api", "ingress", "logship"]); - assert.deepEqual(schema.services["tinycloud-node"].fields.reason.values, ["malformed_envelope", "unexpected_label", "unknown_message", "unlabelled"]); - assert.deepEqual(schema.services["tinycloud-node"].fields.count, { type: "integer", min: 1, max: 1000000000 }); +test("node schema bounds the fixed per-minute gate summary counts", () => { + const node = schema.services["tinycloud-node"]; + assert.ok(node.events.includes("logship_gate_summary_total")); + for (const field of ["malformed_envelope_count", "unknown_message_count", "unlabelled_count", "unexpected_label_count"]) { + assert.deepEqual(node.fields[field], { type: "integer", min: 0, max: 1000000000 }); + } + assert.deepEqual(node.fields.svc.values, ["api", "ingress", "logship"]); }); test("first transform gates GELF labels before field stripping and JSON parsing", () => { @@ -180,3 +211,19 @@ test("first transform gates GELF labels before field stripping and JSON parsing" assert.doesNotMatch(compose, /fluentd|\/var\/lib\/docker|docker\.sock/); assert.match(compose, /networks:\n logship:\n driver: bridge\n ipam:\n config:\n - subnet: 10\.255\.253\.0\/28/); }); +const acceptance = readFileSync(new URL("../tc-959-node-docker-check.review.sh", import.meta.url), "utf8"); +test("GELF source metric parser reads the value before its timestamp", async () => { + const metrics = await import("./tc-959-vector-metrics.mjs"); + const sample = 'vector_component_received_events_total{component_id="gelf_in",component_kind="source",component_type="socket",host="node-1",mode="udp"} 1 1791992950070'; + assert.equal(metrics.parseGelfReceivedCounter(sample), 1); + assert.equal(metrics.parseGelfReceivedCounter(sample.replace('component_id="gelf_in"', 'component_id="other"')), null); + assert.equal(metrics.parseGelfReceivedCounter(sample.replace(',host="node-1"', "")), null); + assert.equal(metrics.countShippedEvents('{"event":"node_warning"}\n{"event":"logship_gate_summary_total"}\nnot-json'), 2); + assert.equal(metrics.countShippedEvent('{"event":"node_warning"}\n{"event":"logship_gate_summary_total"}', "node_warning"), 1); +}); + +test("acceptance producer asserts the rendered Compose-escaped shell command", () => { + assert.ok(acceptance.includes("count=\\$\\$(cat /control/burst)")); + assert.ok(acceptance.includes('while [ "\\$\\$i" -lt "\\$\\$count" ]')); + assert.ok(acceptance.includes("rendered producer command")); +}); diff --git a/scripts/tc-959-vector-metrics.mjs b/scripts/tc-959-vector-metrics.mjs new file mode 100644 index 00000000..06689b6b --- /dev/null +++ b/scripts/tc-959-vector-metrics.mjs @@ -0,0 +1,70 @@ +import { fileURLToPath } from "node:url"; + +const expectedLabels = ["component_id", "component_kind", "component_type", "host", "mode"]; + +function parseLabels(serialized) { + const labels = Object.create(null); + let remaining = serialized; + while (remaining.length > 0) { + const match = remaining.match(/^([a-zA-Z_][a-zA-Z0-9_]*)="((?:\\.|[^"\\])*)"/); + if (!match || Object.hasOwn(labels, match[1])) return null; + labels[match[1]] = match[2]; + remaining = remaining.slice(match[0].length); + if (remaining.length === 0) break; + if (remaining[0] !== ",") return null; + remaining = remaining.slice(1); + } + return labels; +} + +export function parseGelfReceivedCounter(metrics) { + for (const line of metrics.split(/\r?\n/)) { + const sample = line.match(/^vector_component_received_events_total\{([^}]*)\}\s+(\d+)(?:\s+\d+)?\s*$/); + if (!sample) continue; + const labels = parseLabels(sample[1]); + if (!labels) continue; + const names = Object.keys(labels).sort(); + if (names.length !== expectedLabels.length || names.some((name, index) => name !== expectedLabels[index])) continue; + if (labels.component_id !== "gelf_in" || labels.component_kind !== "source" || labels.component_type !== "socket" || labels.mode !== "udp" || labels.host.length === 0) continue; + const value = Number(sample[2]); + if (Number.isSafeInteger(value)) return value; + } + return null; +} + +function parsedEvents(logs) { + const records = []; + for (const line of logs.split(/\r?\n/)) { + try { + const record = JSON.parse(line); + if (record && typeof record.event === "string") records.push(record); + } catch { + // Vector's own text logs are not console-sink events. + } + } + return records; +} + +export function countShippedEvents(logs) { + return parsedEvents(logs).length; +} + +export function countShippedEvent(logs, event) { + return parsedEvents(logs).filter((record) => record.event === event).length; +} + +if (fileURLToPath(import.meta.url) === process.argv[1]) { + const [operation, event] = process.argv.slice(2); + let input = ""; + for await (const chunk of process.stdin) input += chunk; + if (operation === "received-counter") { + const value = parseGelfReceivedCounter(input); + process.stdout.write(value === null ? "missing" : String(value)); + } else if (operation === "shipped-count") { + process.stdout.write(String(countShippedEvents(input))); + } else if (operation === "event-count" && event) { + process.stdout.write(String(countShippedEvent(input, event))); + } else { + throw new Error("usage: tc-959-vector-metrics.mjs received-counter|shipped-count|event-count EVENT"); + } +} diff --git a/tc-959-node-docker-check.review.sh b/tc-959-node-docker-check.review.sh index 788765db..34d13ea0 100755 --- a/tc-959-node-docker-check.review.sh +++ b/tc-959-node-docker-check.review.sh @@ -11,6 +11,10 @@ mkdir -p "$CONTROL" PROJECT="tc959-gelf-$$" OVERRIDE="$WORK/compose.override.yaml" VECTOR_TEST_CONFIG="$WORK/vector.acceptance.yaml" +NATIVE_VECTOR_CONFIG="$WORK/vector.native.yaml" +NATIVE_SECRET_DIR="$WORK/native-secrets" +NATIVE_DATA_DIR="$WORK/native-data" +SENDER="$WORK/send-gelf.mjs" ENV_FILE="$WORK/acceptance.env" REPORT=${TC_REPORT:-"/tmp/tc959-node-gelf-check-$$.txt"} COMPOSE=(docker compose --env-file "$ENV_FILE" --project-directory "$ROOT" -p "$PROJECT" -f "$BASE_COMPOSE" -f "$OVERRIDE") @@ -75,12 +79,25 @@ record "This script is review-only. Run only after the lead authorizes Docker." require node [[ -x "$VECTOR_BIN" ]] || fail "native Vector binary not executable: $VECTOR_BIN" +mkdir -p "$NATIVE_SECRET_DIR" "$NATIVE_DATA_DIR" +printf '%s\n' 'tc959-dummy-token-only' >"$NATIVE_SECRET_DIR/bs_token" +chmod 0644 "$NATIVE_SECRET_DIR/bs_token" +node - "$ROOT/log-shipper/vector.yaml" "$NATIVE_VECTOR_CONFIG" "$NATIVE_SECRET_DIR" "$NATIVE_DATA_DIR" <<'NODE' +const fs = require('node:fs'); +const [sourcePath, targetPath, secretDir, dataDir] = process.argv.slice(2); +let config = fs.readFileSync(sourcePath, 'utf8'); +for (const [from, to] of [['/var/lib/vector/secrets', secretDir], ['/var/lib/vector', dataDir]]) { + if (!config.includes(from)) throw new Error(`canonical Vector config is missing ${from}`); + config = config.replaceAll(from, to); +} +fs.writeFileSync(targetPath, config, { mode: 0o600 }); +NODE node "$ROOT/scripts/node-vector-schema.mjs" --check || fail "generated schema check failed" node "$ROOT/scripts/node-vector-schema.mjs" --check-compose || fail "embedded Compose parity failed" node --test "$ROOT/scripts/node-vector-schema.test.mjs" || fail "schema/packaging tests failed" -"$VECTOR_BIN" validate --no-environment "$ROOT/log-shipper/vector.yaml" || fail "native Vector validation failed" -"$VECTOR_BIN" test "$ROOT/log-shipper/vector.yaml" "$ROOT/log-shipper/vector.tests.yaml" || fail "native Vector tests failed" -record "PASS: native checks completed before any Docker operation" +"$VECTOR_BIN" validate --no-environment "$NATIVE_VECTOR_CONFIG" || fail "native Vector validation failed" +"$VECTOR_BIN" test "$NATIVE_VECTOR_CONFIG" "$ROOT/log-shipper/vector.tests.yaml" || fail "native Vector tests failed" +record "PASS: native checks used only a disposable dummy token fixture before any Docker operation" cat >"$ENV_FILE" <<'EOF' DATABASE_URL=postgres://postgres@postgres:5432/postgres?sslmode=disable @@ -109,10 +126,26 @@ if (!config.includes('\ntransforms:\n')) throw new Error('canonical transforms b config = config.replace('\ntransforms:\n', `\n${metricSource}transforms:\n`); const updatedSinkAt = config.lastIndexOf('\nsinks:\n'); config = config.slice(0, updatedSinkAt) + `\nsinks:\n local_console:\n type: console\n inputs: [budget_size_guard]\n encoding: { codec: json }\n metrics_exporter:\n type: prometheus_exporter\n inputs: [internal_metrics]\n address: 0.0.0.0:9598\n`; -fs.writeFileSync(targetPath, config, { mode: 0o600 }); +fs.writeFileSync(targetPath, config, { mode: 0o644 }); +fs.chmodSync(targetPath, 0o644); NODE "$VECTOR_BIN" validate --no-environment "$VECTOR_TEST_CONFIG" || fail "acceptance Vector config validation failed" +cat >"$SENDER" <<'NODE' +import { createSocket } from 'node:dgram'; +const [host, portText, payload] = process.argv.slice(2); +if (!host || !portText || !payload) throw new Error('usage: send-gelf HOST PORT JSON'); +const socket = createSocket('udp4'); +socket.send(Buffer.from(payload), Number(portText), host, (error) => { + socket.close(); + if (error) { + console.error(`GELF UDP send failed to ${host}:${portText}: ${error.code ?? error.message}`); + process.exitCode = 1; + } +}); +NODE +chmod 0644 "$SENDER" + cat >"$OVERRIDE" < /var/lib/vector/secrets/bs_token && chmod 0644 /var/lib/vector/secrets/bs_token && exec vector --config /etc/vector/acceptance.yaml" volumes: - "$VECTOR_TEST_CONFIG:/etc/vector/acceptance.yaml:ro" ports: @@ -195,15 +228,27 @@ services: max-size: "10m" max-file: "3" EOF +node - "$OVERRIDE" <<'NODE' +const fs = require('node:fs'); +const override = fs.readFileSync(process.argv[2], 'utf8'); +for (const fragment of [ + 'count=$$(cat /control/burst)', + 'while [ "$$i" -lt "$$count" ]', + 'i=$$((i + 1))', +]) { + if (!override.includes(fragment)) throw new Error(`rendered producer command lost Compose escaping: ${fragment}`); +} +NODE if [[ "$DRY_RUN" == 1 ]]; then - record "DRY-RUN: Docker acceptance scenarios planned; no Docker command invoked" - record "DRY-RUN: start local Postgres, static-key/TEE-Off node, labelled ingress GELF producer, isolated Vector and app-network probe" + record "DRY-RUN: no Docker command invoked" + record "DRY-RUN: start local Postgres, test-only Static-key/TEE-Off node, labelled ingress producer, isolated Vector and network probes" record "DRY-RUN: 5x restart, 5x SIGKILL/restart-policy recovery, 5x Vector recreate; each next labelled line <=20s and no duplicate output" - record "DRY-RUN: 3x Vector-down 500-line burst; start and verify next labelled line <=20s" - record "DRY-RUN: app restart count unchanged; app /version remains 200 with Vector stopped and latency sampled" - record "DRY-RUN: forged GELF from app network/Postgres to Vector IP, app gateway, service DNS, host.docker.internal, loopback and host non-loopback interfaces; require zero received/shipped" - record "DRY-RUN: sample Vector peak RSS; 5000-line delivery-loss percentage is informational" + record "DRY-RUN: 3x Vector-down 500-line bursts; recover and verify next labelled line <=20s" + record "DRY-RUN: app restart count unchanged; /version responds 10/10 while Vector is stopped" + record "DRY-RUN: positive host-loopback and logship-control-container probes require source +1 and shipped +1" + record "DRY-RUN: node/ingress/Postgres sibling and host external-interface probes require source +0 and shipped +0, using parsed Prometheus values" + record "DRY-RUN: sample Vector peak RSS; 5,000-line delivery-loss percentage is informational" record "DRY-RUN: actual Docker, network and memory checks NOT run" exit 0 fi @@ -211,7 +256,6 @@ fi require docker require curl require ip -require nc docker compose version >/dev/null || fail "Docker Compose v2 is required" record "Preflight: no DockerRootDir assumption; this GELF topology has no host Docker-directory access." @@ -240,7 +284,7 @@ wait_node() { fail "node did not serve /version within 30 seconds" } read_metric() { - curl --fail --silent http://127.0.0.1:9598/metrics | node -e 'let s="";process.stdin.on("data",d=>s+=d);process.stdin.on("end",()=>{const x=s.split("\n").find(v=>v.startsWith("vector_component_received_events_total{")&&v.includes("component_id=\"gelf_in\""));process.stdout.write(x?x.trim().split(/\s+/).at(-1):"missing")})' + curl --fail --silent http://127.0.0.1:9598/metrics | node "$ROOT/scripts/tc-959-vector-metrics.mjs" received-counter } wait_metric() { local deadline=$((SECONDS + 20)) value @@ -251,9 +295,13 @@ wait_metric() { done fail "Vector GELF receive metric did not reach ${1:-1}" } +vector_shipped_count() { + [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] || fail "refusing to read events from a stopped Vector container" + docker_cmd logs "$VECTOR_ID" 2>&1 | node "$ROOT/scripts/tc-959-vector-metrics.mjs" shipped-count +} vector_event_count() { - [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] || fail "refusing to read logs from a stopped Vector container" - docker_cmd logs "$VECTOR_ID" 2>&1 | node -e 'let s="";process.stdin.on("data",d=>s+=d);process.stdin.on("end",()=>process.stdout.write(String(s.split("\n").filter(x=>x.includes("ingress_request_slow")).length)))' + [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] || fail "refusing to read events from a stopped Vector container" + docker_cmd logs "$VECTOR_ID" 2>&1 | node "$ROOT/scripts/tc-959-vector-metrics.mjs" event-count ingress_request_slow } wait_next_line() { local previous=${1:-0} deadline=$((SECONDS + 20)) current @@ -350,28 +398,93 @@ loss=100 if (( after_metric >= before_metric )); then loss=$(( (5000 - (after_metric - before_metric)) * 100 / 5000 )); (( loss < 0 )) && loss=0; fi record "INFO: 5,000-line receive count delta=$((after_metric - before_metric)); estimated transport loss=${loss}% (informational)" -# Stop the synthetic ingress producer so the isolation counter window has no legitimate GELF traffic. -compose_cmd stop dstack-ingress >/dev/null +# Keep the idle synthetic ingress up so its network namespace is available for sibling probes. sleep 1 -baseline=$(wait_metric 0) payload='{"version":"1.1","host":"tc959-isolation-probe","short_message":"{\"timestamp\":\"2026-10-14T10:00:00.000Z\",\"level\":\"WARN\",\"fields\":{\"message\":\"invocation replay insert failed\"}}","timestamp":1760436000,"level":4,"_tc.logship":"tinycloud-node"}' vector_ip=10.255.253.2 app_gateway=$(docker_cmd inspect -f '{{range .NetworkSettings.Networks}}{{.Gateway}}{{end}}' "$NODE_ID") +INGRESS_ID=$(compose_cmd ps -q dstack-ingress) +[[ -n "$INGRESS_ID" ]] || fail "ingress control container missing" + +send_from_container() { + local network=$1 target=$2 + docker_cmd run --rm --network "$network" --mount "type=bind,src=$SENDER,dst=/sender.mjs,readonly" node:20-alpine node /sender.mjs "$target" 12201 "$payload" +} +assert_positive_delivery() { + local source_before=$1 shipped_before=$2 description=$3 deadline=$((SECONDS + 20)) source_after shipped_after source_delta shipped_delta + while (( SECONDS < deadline )); do + source_after=$(read_metric) + shipped_after=$(vector_shipped_count) + [[ "$source_after" =~ ^[0-9]+$ && "$shipped_after" =~ ^[0-9]+$ ]] || fail "invalid Vector counters during $description" + source_delta=$((source_after - source_before)) + shipped_delta=$((shipped_after - shipped_before)) + if (( source_delta == 1 && shipped_delta == 1 )); then + record "PASS: $description positive control; source +1 and shipped +1" + return 0 + fi + (( source_delta <= 1 && shipped_delta <= 1 )) || fail "$description produced duplicate source/shipped events: source +$source_delta, shipped +$shipped_delta" + sleep 0.2 + done + fail "$description positive control failed: source +$source_delta, shipped +$shipped_delta" +} + +# Positive controls prove both paths before attempting forbidden sibling/external sources. +source_before=$(wait_metric 0) +shipped_before=$(vector_shipped_count) +node "$SENDER" 127.0.0.1 12201 "$payload" +assert_positive_delivery "$source_before" "$shipped_before" "host loopback" + +source_before=$(wait_metric 0) +shipped_before=$(vector_shipped_count) +send_from_container "${PROJECT}_logship" "$vector_ip" +assert_positive_delivery "$source_before" "$shipped_before" "logship-network control container" + +assert_isolated_probe() { + local description=$1 network=$2 target=$3 source_before shipped_before send_status=0 source_after shipped_after + source_before=$(wait_metric 0) + shipped_before=$(vector_shipped_count) + if send_from_container "$network" "$target"; then + record "INFO: $description sender completed" + else + send_status=$? + record "INFO: $description sender reported transport error exit=$send_status" + fi + sleep 0.5 + source_after=$(read_metric) + shipped_after=$(vector_shipped_count) + [[ "$source_after" =~ ^[0-9]+$ && "$shipped_after" =~ ^[0-9]+$ ]] || fail "invalid Vector counters during $description" + [[ "$source_after" == "$source_before" ]] || fail "$description isolation leak: source counter changed $source_before -> $source_after" + [[ "$shipped_after" == "$shipped_before" ]] || fail "$description isolation leak: shipped count changed $shipped_before -> $shipped_after" + record "PASS: $description isolation; source +0 and shipped +0" +} + probe_targets=("$vector_ip" "$app_gateway" vector host.docker.internal 127.0.0.1) for target in "${probe_targets[@]}"; do - compose_cmd exec -T app-net-probe sh -ec 'printf "%s\n" "$1" | nc -u -w 1 "$2" 12201 >/dev/null 2>&1 || true' sh "$payload" "$target" >/dev/null - docker_cmd exec "$PG_ID" sh -ec 'printf "%s\n" "$1" | busybox nc -u -w 1 "$2" 12201 >/dev/null 2>&1 || true' sh "$payload" "$target" >/dev/null - record "INFO: attempted forged GELF from app namespace and Postgres to $target" + assert_isolated_probe "node->$target" "container:$NODE_ID" "$target" + assert_isolated_probe "ingress->$target" "container:$INGRESS_ID" "$target" + assert_isolated_probe "postgres->$target" "container:$PG_ID" "$target" done + +host_ips=$(ip -o -4 addr show scope global | awk '{split($4,a,"/"); print a[1]}') +[[ -n "$host_ips" ]] || fail "no non-loopback IPv4 interface available for external-interface isolation probes" while read -r host_ip; do [[ -n "$host_ip" ]] || continue - printf '%s\n' "$payload" | nc -u -w 1 "$host_ip" 12201 >/dev/null 2>&1 || true - record "INFO: attempted forged GELF from host interface $host_ip" -done < <(ip -o -4 addr show scope global | awk '{split($4,a,"/"); print a[1]}') -sleep 1 -final_metric=$(wait_metric 0) -[[ "$final_metric" == "$baseline" ]] || fail "isolation leak: GELF receive counter changed from $baseline to $final_metric" -record "PASS: app/Postgres/host probes delivered zero GELF packets and zero events" + source_before=$(wait_metric 0) + shipped_before=$(vector_shipped_count) + if node "$SENDER" "$host_ip" 12201 "$payload"; then + record "INFO: host external-interface sender completed to $host_ip" + else + send_status=$? + record "INFO: host external-interface sender reported transport error exit=$send_status for $host_ip" + fi + sleep 0.5 + source_after=$(read_metric) + shipped_after=$(vector_shipped_count) + [[ "$source_after" =~ ^[0-9]+$ && "$shipped_after" =~ ^[0-9]+$ ]] || fail "invalid Vector counters during host interface $host_ip probe" + [[ "$source_after" == "$source_before" ]] || fail "host interface $host_ip isolation leak: source $source_before -> $source_after" + [[ "$shipped_after" == "$shipped_before" ]] || fail "host interface $host_ip isolation leak: shipped $shipped_before -> $shipped_after" + record "PASS: host external interface $host_ip isolation; source +0 and shipped +0" +done <<<"$host_ips" compose_cmd stop vector >/dev/null max_ms=0 diff --git a/tinycloud-node-server/src/lib.rs b/tinycloud-node-server/src/lib.rs index 7e7b8def..a29fe285 100644 --- a/tinycloud-node-server/src/lib.rs +++ b/tinycloud-node-server/src/lib.rs @@ -60,6 +60,7 @@ pub mod signed_urls; pub mod storage; pub mod tee; mod tracing; +pub use tracing::{cap_json_log_line, MAX_JSON_LOG_LINE_BYTES}; pub mod tunnel; pub mod webhook_dispatcher; diff --git a/tinycloud-node-server/src/main.rs b/tinycloud-node-server/src/main.rs index c1155ee7..73c37083 100644 --- a/tinycloud-node-server/src/main.rs +++ b/tinycloud-node-server/src/main.rs @@ -10,7 +10,7 @@ use rocket::{ figment::providers::{Env, Format, Serialized, Toml}, tokio, }; -use tinycloud::{app, config, prometheus, resolve_runtime_config}; +use tinycloud::{app, cap_json_log_line, config, prometheus, resolve_runtime_config}; fn build_config_figment() -> rocket::figment::Figment { let config_file = @@ -32,17 +32,19 @@ fn panic_log_line() -> String { let timestamp = time::OffsetDateTime::now_utc() .format(&time::format_description::well_known::Rfc3339) .expect("current UTC timestamp is RFC3339-formattable"); - serde_json::json!({ + let mut json = serde_json::json!({ "timestamp": timestamp, "level": "ERROR", "fields": { "message": "node fatal panic" }, }) - .to_string() + .to_string(); + json.push('\n'); + String::from_utf8(cap_json_log_line(json.as_bytes())).expect("capped panic line remains UTF-8") } fn install_panic_hook() { std::panic::set_hook(Box::new(|_info| { - eprintln!("{}", panic_log_line()); + eprint!("{}", panic_log_line()); })); } @@ -262,7 +264,10 @@ mod tests { #[test] fn panic_log_record_is_json_and_contains_only_a_fixed_message() { let line = panic_log_line(); - let record: serde_json::Value = serde_json::from_str(&line).expect("panic record JSON"); + assert!(line.len() <= tinycloud::MAX_JSON_LOG_LINE_BYTES); + assert_eq!(line.lines().count(), 1); + let record: serde_json::Value = + serde_json::from_str(line.trim_end()).expect("panic record JSON"); assert!(record["timestamp"].as_str().is_some()); assert_eq!(record["level"], "ERROR"); assert_eq!(record["fields"]["message"], "node fatal panic"); diff --git a/tinycloud-node-server/src/tracing.rs b/tinycloud-node-server/src/tracing.rs index b1605141..cccf8dcb 100644 --- a/tinycloud-node-server/src/tracing.rs +++ b/tinycloud-node-server/src/tracing.rs @@ -7,7 +7,7 @@ use rocket::{ request::{FromRequest, Outcome}, Data, Request, Response, }; -use serde::Serialize; +use serde::{de::Deserialize, Serialize}; use serde_json::{Map, Value}; use std::{ borrow::Cow, @@ -41,6 +41,128 @@ const UNMATCHED_ROUTE_LABEL: &str = ""; /// Target of the one log line per request (method, route template, status, /// latency). It replaces Rocket's own request logging, which prints the raw URI. const REQUEST_LOG_TARGET: &str = "tinycloud::request"; +pub const MAX_JSON_LOG_LINE_BYTES: usize = 15_000; +const TRUNCATION_MARKER: &str = "[truncated]"; +const JSON_LOG_FALLBACK: &[u8] = b"{\"fields\":{\"message\":\"log line exceeded 15000 bytes\"}}\n"; + +fn collect_string_fields<'a>(value: &'a mut Value, strings: &mut Vec<&'a mut String>) { + match value { + Value::String(string) => strings.push(string), + Value::Array(values) => { + for value in values { + collect_string_fields(value, strings); + } + } + Value::Object(fields) => { + for value in fields.values_mut() { + collect_string_fields(value, strings); + } + } + _ => {} + } +} + +fn is_valid_json(json: &[u8]) -> bool { + let mut deserializer = serde_json::Deserializer::from_slice(json); + serde::de::IgnoredAny::deserialize(&mut deserializer).is_ok() && deserializer.end().is_ok() +} + +pub fn cap_json_log_line(line: &[u8]) -> Vec { + cap_json_log_buffer(line.to_vec()) +} + +fn cap_json_log_buffer(mut line: Vec) -> Vec { + let content_len = line.len() - usize::from(line.ends_with(b"\n")); + let json = &line[..content_len]; + if json.contains(&b'\n') || !is_valid_json(json) { + return JSON_LOG_FALLBACK.to_vec(); + } + if line.len() <= MAX_JSON_LOG_LINE_BYTES { + if content_len == line.len() { + line.push(b'\n'); + } + return line; + } + + let Ok(mut value) = serde_json::from_slice::(json) else { + return JSON_LOG_FALLBACK.to_vec(); + }; + loop { + let Ok(mut encoded) = serde_json::to_vec(&value) else { + return JSON_LOG_FALLBACK.to_vec(); + }; + encoded.push(b'\n'); + if encoded.len() <= MAX_JSON_LOG_LINE_BYTES { + return encoded; + } + + let mut strings = Vec::new(); + collect_string_fields(&mut value, &mut strings); + let mut longest_index = None; + let mut longest_len = TRUNCATION_MARKER.len(); + for (index, string) in strings.iter().enumerate() { + if string.len() > longest_len { + longest_index = Some(index); + longest_len = string.len(); + } + } + let Some(longest_index) = longest_index else { + return JSON_LOG_FALLBACK.to_vec(); + }; + let longest = strings + .into_iter() + .nth(longest_index) + .expect("selected field exists"); + let excess = encoded.len() - MAX_JSON_LOG_LINE_BYTES; + let mut keep = longest + .len() + .saturating_sub(excess + TRUNCATION_MARKER.len()); + while !longest.is_char_boundary(keep) { + keep -= 1; + } + longest.truncate(keep); + longest.push_str(TRUNCATION_MARKER); + } +} + +struct CappedJsonMakeWriter(M); + +struct CappedJsonWriter { + inner: W, + bytes: Vec, +} + +impl std::io::Write for CappedJsonWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl Drop for CappedJsonWriter { + fn drop(&mut self) { + let line = cap_json_log_buffer(std::mem::take(&mut self.bytes)); + let _ = self.inner.write_all(&line); + } +} + +impl<'a, M> tracing_subscriber::fmt::MakeWriter<'a> for CappedJsonMakeWriter +where + M: tracing_subscriber::fmt::MakeWriter<'a>, +{ + type Writer = CappedJsonWriter; + + fn make_writer(&'a self) -> Self::Writer { + CappedJsonWriter { + inner: self.0.make_writer(), + bytes: Vec::with_capacity(512), + } + } +} /// Rocket modules that log only at ignite, liftoff or shutdown, never request /// data. Rocket's other modules log per request. @@ -391,7 +513,10 @@ pub fn tracing_try_init( let subscriber = tracing_subscriber::fmt::layer(); let log = match config.format { config::LoggingFormat::Text => subscriber.boxed(), - config::LoggingFormat::Json => subscriber.json().boxed(), + config::LoggingFormat::Json => subscriber + .json() + .with_writer(CappedJsonMakeWriter(|| std::io::stderr())) + .boxed(), }; let telemetry = if config.tracing.enabled { let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() @@ -607,4 +732,68 @@ mod tests { "trace header should still be set (as the all-zero id) when tracing is disabled" ); } + #[test] + fn json_formatter_caps_each_complete_serialized_event() { + #[derive(Clone)] + struct CaptureWriter(Arc>>); + + impl std::io::Write for CaptureWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0 + .lock() + .expect("capture mutex") + .extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + let output = Arc::new(Mutex::new(Vec::new())); + let capture = output.clone(); + let subscriber = tracing_subscriber::fmt() + .json() + .with_writer(CappedJsonMakeWriter(move || CaptureWriter(capture.clone()))) + .finish(); + let over_16k = "a".repeat(17 * 1024); + let several_a = "b".repeat(7 * 1024); + let several_b = "c".repeat(7 * 1024); + let several_c = "d".repeat(7 * 1024); + + tracing::subscriber::with_default(subscriber, || { + tracing::info!(oversized = %over_16k, "one oversized field"); + tracing::info!( + large_a = %several_a, + large_b = %several_b, + large_c = %several_c, + "several large fields" + ); + }); + + let bytes = output.lock().expect("capture mutex"); + let lines: Vec<&[u8]> = bytes.split_inclusive(|byte| *byte == b'\n').collect(); + assert_eq!(lines.len(), 2, "each event must produce exactly one line"); + for line in lines { + assert!( + line.len() <= MAX_JSON_LOG_LINE_BYTES, + "line size: {}", + line.len() + ); + assert_eq!(line.last(), Some(&b'\n'), "line must be newline terminated"); + let record: Value = + serde_json::from_slice(&line[..line.len() - 1]).expect("capped line must be JSON"); + assert!( + record["fields"] + .as_object() + .expect("JSON fields") + .values() + .any(|value| value + .as_str() + .is_some_and(|text| text.ends_with(TRUNCATION_MARKER))), + "large string fields should carry the fixed truncation marker" + ); + } + } } From db18d13a01dd17f8ae2954fedba19d56c0e729b3 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 16:47:35 +0000 Subject: [PATCH 07/21] Fix GELF acceptance freshness and Compose checks --- scripts/node-vector-schema.mjs | 2 +- scripts/node-vector-schema.test.mjs | 24 +++ scripts/tc-959-vector-metrics.mjs | 15 +- scripts/tc-959-vector-metrics.test.mjs | 12 ++ tc-959-node-docker-check.review.sh | 239 +++++++++++++++---------- 5 files changed, 193 insertions(+), 99 deletions(-) create mode 100644 scripts/tc-959-vector-metrics.test.mjs diff --git a/scripts/node-vector-schema.mjs b/scripts/node-vector-schema.mjs index 263227ff..e4b530b3 100644 --- a/scripts/node-vector-schema.mjs +++ b/scripts/node-vector-schema.mjs @@ -283,7 +283,7 @@ function isComposeMapping(value) { export function checkComposeDeploymentPackaging(compose) { let definition; try { - definition = parse(compose); + definition = parse(compose, { merge: true }); } catch (error) { throw new Error("Compose packaging must be valid YAML", { cause: error }); } diff --git a/scripts/node-vector-schema.test.mjs b/scripts/node-vector-schema.test.mjs index 8ad7d007..589ee1e1 100644 --- a/scripts/node-vector-schema.test.mjs +++ b/scripts/node-vector-schema.test.mjs @@ -131,6 +131,30 @@ test("rejects flow-style relative long bind sources", () => { /relative bind sources/, ); }); +test("rejects aliased service env_file values", () => { + const compose = `x-review-env: &review-env ./.review.env\n${minimalCompose} probe:\n env_file: *review-env\n`; + assert.throws(() => checkComposeDeploymentPackaging(compose), /env_file/); +}); +test("rejects merged service env_file values", () => { + const compose = `x-review-service: &review-service\n env_file: ./.review.env\n${minimalCompose} probe:\n <<: *review-service\n`; + assert.throws(() => checkComposeDeploymentPackaging(compose), /env_file/); +}); +test("rejects aliased configs.file values", () => { + const compose = `x-review-config: &review-config {file: ./outside.yaml}\n${minimalCompose}\nconfigs:\n review: *review-config\n`; + assert.throws(() => checkComposeDeploymentPackaging(compose), /configs\.\*\.file/); +}); +test("rejects merged configs.file values", () => { + const compose = `x-review-config: &review-config {file: ./outside.yaml}\n${minimalCompose}\nconfigs:\n review:\n <<: *review-config\n`; + assert.throws(() => checkComposeDeploymentPackaging(compose), /configs\.\*\.file/); +}); +test("rejects aliased relative short bind sources", () => { + const compose = `x-review-volume: &review-volume ./relative:/etc/review.yaml:ro\n${minimalCompose} volumes:\n - *review-volume\n`; + assert.throws(() => checkComposeDeploymentPackaging(compose), /relative bind sources/); +}); +test("rejects merged relative long bind sources", () => { + const compose = `x-review-volume: &review-volume {type: bind, source: ./relative, target: /etc/review.yaml}\n${minimalCompose} volumes:\n - <<: *review-volume\n`; + assert.throws(() => checkComposeDeploymentPackaging(compose), /relative bind sources/); +}); test("accepts canonical Compose packaging", () => { assert.doesNotThrow(() => checkComposeDeploymentPackaging(compose)); }); diff --git a/scripts/tc-959-vector-metrics.mjs b/scripts/tc-959-vector-metrics.mjs index 06689b6b..5e4c52c2 100644 --- a/scripts/tc-959-vector-metrics.mjs +++ b/scripts/tc-959-vector-metrics.mjs @@ -17,9 +17,9 @@ function parseLabels(serialized) { return labels; } -export function parseGelfReceivedCounter(metrics) { +export function parseGelfReceivedSample(metrics) { for (const line of metrics.split(/\r?\n/)) { - const sample = line.match(/^vector_component_received_events_total\{([^}]*)\}\s+(\d+)(?:\s+\d+)?\s*$/); + const sample = line.match(/^vector_component_received_events_total\{([^}]*)\}\s+(\d+)\s+(\d+)\s*$/); if (!sample) continue; const labels = parseLabels(sample[1]); if (!labels) continue; @@ -27,11 +27,16 @@ export function parseGelfReceivedCounter(metrics) { if (names.length !== expectedLabels.length || names.some((name, index) => name !== expectedLabels[index])) continue; if (labels.component_id !== "gelf_in" || labels.component_kind !== "source" || labels.component_type !== "socket" || labels.mode !== "udp" || labels.host.length === 0) continue; const value = Number(sample[2]); - if (Number.isSafeInteger(value)) return value; + const timestamp = Number(sample[3]); + if (Number.isSafeInteger(value) && Number.isSafeInteger(timestamp)) return { value, timestamp }; } return null; } +export function parseGelfReceivedCounter(metrics) { + return parseGelfReceivedSample(metrics)?.value ?? null; +} + function parsedEvents(logs) { const records = []; for (const line of logs.split(/\r?\n/)) { @@ -60,11 +65,13 @@ if (fileURLToPath(import.meta.url) === process.argv[1]) { if (operation === "received-counter") { const value = parseGelfReceivedCounter(input); process.stdout.write(value === null ? "missing" : String(value)); + } else if (operation === "received-sample") { + process.stdout.write(sample === null ? "missing" : `${sample.value} ${sample.timestamp}`); } else if (operation === "shipped-count") { process.stdout.write(String(countShippedEvents(input))); } else if (operation === "event-count" && event) { process.stdout.write(String(countShippedEvent(input, event))); } else { - throw new Error("usage: tc-959-vector-metrics.mjs received-counter|shipped-count|event-count EVENT"); + throw new Error("usage: tc-959-vector-metrics.mjs received-counter|received-sample|shipped-count|event-count EVENT"); } } diff --git a/scripts/tc-959-vector-metrics.test.mjs b/scripts/tc-959-vector-metrics.test.mjs new file mode 100644 index 00000000..7e54ce78 --- /dev/null +++ b/scripts/tc-959-vector-metrics.test.mjs @@ -0,0 +1,12 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { parseGelfReceivedSample } from "./tc-959-vector-metrics.mjs"; + +test("parses the counter value and exporter sample timestamp separately", () => { + const sample = parseGelfReceivedSample(`# HELP vector_component_received_events_total Events received\nvector_component_received_events_total{component_id="gelf_in",component_kind="source",component_type="socket",host="vector",mode="udp"} 1 1791992950070\n`); + assert.deepEqual(sample, { value: 1, timestamp: 1791992950070 }); +}); + +test("requires a timestamp for freshness-sensitive source samples", () => { + assert.equal(parseGelfReceivedSample(`vector_component_received_events_total{component_id="gelf_in",component_kind="source",component_type="socket",host="vector",mode="udp"} 1\n`), null); +}); diff --git a/tc-959-node-docker-check.review.sh b/tc-959-node-docker-check.review.sh index 34d13ea0..05595035 100755 --- a/tc-959-node-docker-check.review.sh +++ b/tc-959-node-docker-check.review.sh @@ -23,6 +23,8 @@ NODE_ID="" PG_ID="" STARTED=0 SAMPLER_PID="" +METRICS_SCRAPE_INTERVAL_SECONDS=1 +ISOLATION_SETTLE_SECONDS=2 SAMPLER_STOP="$WORK/sample-memory" record() { printf '%s\n' "$*" | tee -a "$REPORT"; } @@ -94,7 +96,7 @@ fs.writeFileSync(targetPath, config, { mode: 0o600 }); NODE node "$ROOT/scripts/node-vector-schema.mjs" --check || fail "generated schema check failed" node "$ROOT/scripts/node-vector-schema.mjs" --check-compose || fail "embedded Compose parity failed" -node --test "$ROOT/scripts/node-vector-schema.test.mjs" || fail "schema/packaging tests failed" +node --test "$ROOT/scripts/node-vector-schema.test.mjs" "$ROOT/scripts/tc-959-vector-metrics.test.mjs" || fail "schema, packaging, and metrics tests failed" "$VECTOR_BIN" validate --no-environment "$NATIVE_VECTOR_CONFIG" || fail "native Vector validation failed" "$VECTOR_BIN" test "$NATIVE_VECTOR_CONFIG" "$ROOT/log-shipper/vector.tests.yaml" || fail "native Vector tests failed" record "PASS: native checks used only a disposable dummy token fixture before any Docker operation" @@ -121,7 +123,7 @@ const [sourcePath, targetPath] = process.argv.slice(2); let config = fs.readFileSync(sourcePath, 'utf8'); const sinkAt = config.lastIndexOf('\nsinks:\n'); if (sinkAt < 0) throw new Error('canonical sink block not found'); -const metricSource = ' internal_metrics:\n type: internal_metrics\n'; +const metricSource = ' internal_metrics:\n type: internal_metrics\n scrape_interval_secs: 1\n'; if (!config.includes('\ntransforms:\n')) throw new Error('canonical transforms block not found'); config = config.replace('\ntransforms:\n', `\n${metricSource}transforms:\n`); const updatedSinkAt = config.lastIndexOf('\nsinks:\n'); @@ -171,28 +173,32 @@ services: - "host.docker.internal:host-gateway" dstack-ingress: image: alpine:3.21 - entrypoint: ["/bin/sh", "-ec"] - command: | - emit() { printf '%s\n' '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/info","protocol":"HTTP/1.1","status":200,"bytes":32,"request_time":1.25,"upstream_status":"200"}; } - while :; do - if [ -f /control/burst ]; then - count=\$\$(cat /control/burst) - rm -f /control/burst - i=0 - while [ "\$\$i" -lt "\$\$count" ]; do emit; i=\$\$((i + 1)); done - touch /control/burst-done - elif [ -f /control/pulse ]; then - rm -f /control/pulse - emit - fi - sleep 0.05 - done + command: + - /bin/sh + - -ec + - | + emit() { printf '%s\n' '{"time":"2026-10-14T10:00:00+00:00","method":"GET","route":"/info","protocol":"HTTP/1.1","status":200,"bytes":32,"request_time":1.25,"upstream_status":"200"}'; } + while :; do + if [ -f /control/burst ]; then + count=\$\$(cat /control/burst) + rm -f /control/burst + i=0 + while [ "\$\$i" -lt "\$\$count" ]; do emit; i=\$\$((i + 1)); done + touch /control/burst-done + elif [ -f /control/pulse ]; then + rm -f /control/pulse + emit + fi + sleep 0.05 + done volumes: !override - "$CONTROL:/control" ports: !override [] vector: - entrypoint: ["/bin/sh", "-ec"] - command: "mkdir -p /var/lib/vector/secrets && printf '%s\\n' tc959-dummy-token-only > /var/lib/vector/secrets/bs_token && chmod 0644 /var/lib/vector/secrets/bs_token && exec vector --config /etc/vector/acceptance.yaml" + command: + - /bin/sh + - -ec + - mkdir -p /var/lib/vector/secrets && printf '%s\\n' tc959-dummy-token-only > /var/lib/vector/secrets/bs_token && chmod 0644 /var/lib/vector/secrets/bs_token && exec vector --config /etc/vector/acceptance.yaml volumes: - "$VECTOR_TEST_CONFIG:/etc/vector/acceptance.yaml:ro" ports: @@ -214,40 +220,41 @@ services: options: max-size: "10m" max-file: "3" - app-net-probe: - image: alpine:3.21 - network_mode: "service:tinycloud" - depends_on: - - tinycloud - command: ["/bin/sh", "-ec", "while :; do sleep 3600; done"] - extra_hosts: - - "host.docker.internal:host-gateway" - logging: - driver: json-file - options: - max-size: "10m" - max-file: "3" EOF -node - "$OVERRIDE" <<'NODE' +node - "$OVERRIDE" "$ROOT" <<'NODE' const fs = require('node:fs'); -const override = fs.readFileSync(process.argv[2], 'utf8'); +const { spawnSync } = require('node:child_process'); +const { createRequire } = require('node:module'); +const requireFromRoot = createRequire(`${process.argv[3]}/package.json`); +const { parse } = requireFromRoot('yaml'); +const path = process.argv[2]; +const text = fs.readFileSync(path, 'utf8'); +const compose = parse(text.replaceAll('!override', '')); +for (const service of ['dstack-ingress', 'vector']) { + const argv = compose.services?.[service]?.command; + if (!Array.isArray(argv) || argv.length !== 3 || argv[0] !== '/bin/sh' || argv[1] !== '-ec' || typeof argv[2] !== 'string') { + throw new Error(`${service} rendered Compose argv must be [/bin/sh, -ec, script]`); + } + const renderedArgv = argv.map((argument) => argument.replaceAll('$$', '$')); + if (renderedArgv[0] !== '/bin/sh' || renderedArgv[1] !== '-ec') throw new Error(`${service} rendered argv changed`); + const syntax = spawnSync('sh', ['-n'], { input: renderedArgv[2], encoding: 'utf8' }); + if (syntax.status !== 0) throw new Error(`${service} generated shell is invalid: ${syntax.stderr}`); +} +const ingress = compose.services['dstack-ingress'].command[2]; for (const fragment of [ 'count=$$(cat /control/burst)', 'while [ "$$i" -lt "$$count" ]', 'i=$$((i + 1))', ]) { - if (!override.includes(fragment)) throw new Error(`rendered producer command lost Compose escaping: ${fragment}`); + if (!ingress.includes(fragment)) throw new Error(`rendered producer command lost Compose escaping: ${fragment}`); } NODE - if [[ "$DRY_RUN" == 1 ]]; then record "DRY-RUN: no Docker command invoked" - record "DRY-RUN: start local Postgres, test-only Static-key/TEE-Off node, labelled ingress producer, isolated Vector and network probes" - record "DRY-RUN: 5x restart, 5x SIGKILL/restart-policy recovery, 5x Vector recreate; each next labelled line <=20s and no duplicate output" - record "DRY-RUN: 3x Vector-down 500-line bursts; recover and verify next labelled line <=20s" - record "DRY-RUN: app restart count unchanged; /version responds 10/10 while Vector is stopped" - record "DRY-RUN: positive host-loopback and logship-control-container probes require source +1 and shipped +1" - record "DRY-RUN: node/ingress/Postgres sibling and host external-interface probes require source +0 and shipped +0, using parsed Prometheus values" + record "DRY-RUN: acceptance self-test expects an allowed host-loopback packet to make the negative check fail" + record "DRY-RUN: positive host-loopback and logship-control-container probes require fresh source +1 and shipped +1" + record "DRY-RUN: node/ingress/Postgres sibling and host external-interface probes require two newer stable source samples and shipped +0" + record "DRY-RUN: metric freshness internal_metrics=${METRICS_SCRAPE_INTERVAL_SECONDS}s; isolation settle=${ISOLATION_SETTLE_SECONDS}s" record "DRY-RUN: sample Vector peak RSS; 5,000-line delivery-loss percentage is informational" record "DRY-RUN: actual Docker, network and memory checks NOT run" exit 0 @@ -257,11 +264,12 @@ require docker require curl require ip docker compose version >/dev/null || fail "Docker Compose v2 is required" -record "Preflight: no DockerRootDir assumption; this GELF topology has no host Docker-directory access." +record "Preflight: no DockerRootDir assumption; no host Docker-directory access." +record "Metrics freshness: internal_metrics scrape interval=${METRICS_SCRAPE_INTERVAL_SECONDS}s; isolation settle=${ISOLATION_SETTLE_SECONDS}s; require two strictly newer timestamps with stable source value." # Continue below only on an explicitly Docker-authorized acceptance host. STARTED=1 -compose_cmd up -d --wait postgres tinycloud dstack-ingress vector app-net-probe +compose_cmd up -d --wait postgres tinycloud dstack-ingress vector touch "$SAMPLER_STOP" memory_sampler & SAMPLER_PID=$! @@ -275,17 +283,35 @@ wait_vector() { done fail "Vector did not become running within 20 seconds" } -wait_node() { - local deadline=$((SECONDS + 30)) - while (( SECONDS < deadline )); do - if curl --fail --silent --max-time 2 http://127.0.0.1:18000/version >/dev/null; then return 0; fi - sleep 0.5 - done - fail "node did not serve /version within 30 seconds" -} read_metric() { curl --fail --silent http://127.0.0.1:9598/metrics | node "$ROOT/scripts/tc-959-vector-metrics.mjs" received-counter } +read_metric_sample() { + curl --fail --silent http://127.0.0.1:9598/metrics | node "$ROOT/scripts/tc-959-vector-metrics.mjs" received-sample +} +metric_now_ms() { node -p 'Date.now()'; } +wait_fresh_stable_metric() { + local after_ms=$1 deadline=$((SECONDS + 20)) settle_until=$((after_ms + ISOLATION_SETTLE_SECONDS * 1000)) + local sample value timestamp previous_timestamp=-1 previous_value=-1 now + while (( SECONDS < deadline )); do + sample=$(read_metric_sample) + if [[ "$sample" =~ ^([0-9]+)[[:space:]]([0-9]+)$ ]]; then + value=${BASH_REMATCH[1]} + timestamp=${BASH_REMATCH[2]} + if (( timestamp > after_ms && timestamp > previous_timestamp )); then + now=$(metric_now_ms) + if (( value == previous_value && now >= settle_until )); then + printf '%s\n' "$value" + return 0 + fi + previous_value=$value + previous_timestamp=$timestamp + fi + fi + sleep 0.2 + done + fail "Vector source metric did not produce two fresh, stable samples after timestamp $after_ms" +} wait_metric() { local deadline=$((SECONDS + 20)) value while (( SECONDS < deadline )); do @@ -410,80 +436,105 @@ send_from_container() { local network=$1 target=$2 docker_cmd run --rm --network "$network" --mount "type=bind,src=$SENDER,dst=/sender.mjs,readonly" node:20-alpine node /sender.mjs "$target" 12201 "$payload" } +fresh_metric_baseline() { + local started + started=$(metric_now_ms) + wait_fresh_stable_metric "$started" +} +send_probe() { + local sender=$1 network=$2 target=$3 + if [[ "$sender" == host ]]; then + node "$SENDER" "$target" 12201 "$payload" + else + send_from_container "$network" "$target" + fi +} assert_positive_delivery() { - local source_before=$1 shipped_before=$2 description=$3 deadline=$((SECONDS + 20)) source_after shipped_after source_delta shipped_delta + local source_before=$1 shipped_before=$2 sent_at=$3 description=$4 + local source_after source_delta deadline=$((SECONDS + 20)) shipped_after shipped_delta + source_after=$(wait_fresh_stable_metric "$sent_at") + source_delta=$((source_after - source_before)) + (( source_delta == 1 )) || fail "$description positive control source delta was $source_delta, expected +1" while (( SECONDS < deadline )); do - source_after=$(read_metric) shipped_after=$(vector_shipped_count) - [[ "$source_after" =~ ^[0-9]+$ && "$shipped_after" =~ ^[0-9]+$ ]] || fail "invalid Vector counters during $description" - source_delta=$((source_after - source_before)) shipped_delta=$((shipped_after - shipped_before)) - if (( source_delta == 1 && shipped_delta == 1 )); then - record "PASS: $description positive control; source +1 and shipped +1" + if (( shipped_delta == 1 )); then + record "PASS: $description positive control; fresh source +1 and shipped +1" return 0 fi - (( source_delta <= 1 && shipped_delta <= 1 )) || fail "$description produced duplicate source/shipped events: source +$source_delta, shipped +$shipped_delta" + (( shipped_delta <= 1 )) || fail "$description produced duplicate shipped events: +$shipped_delta" sleep 0.2 done - fail "$description positive control failed: source +$source_delta, shipped +$shipped_delta" + fail "$description positive control shipped delta was $shipped_delta, expected +1" } -# Positive controls prove both paths before attempting forbidden sibling/external sources. -source_before=$(wait_metric 0) +# Positive controls precede the leak self-test and every forbidden-source assertion. +source_before=$(fresh_metric_baseline) shipped_before=$(vector_shipped_count) +sent_at=$(metric_now_ms) node "$SENDER" 127.0.0.1 12201 "$payload" -assert_positive_delivery "$source_before" "$shipped_before" "host loopback" +assert_positive_delivery "$source_before" "$shipped_before" "$sent_at" "host loopback" -source_before=$(wait_metric 0) +source_before=$(fresh_metric_baseline) shipped_before=$(vector_shipped_count) +sent_at=$(metric_now_ms) send_from_container "${PROJECT}_logship" "$vector_ip" -assert_positive_delivery "$source_before" "$shipped_before" "logship-network control container" +assert_positive_delivery "$source_before" "$shipped_before" "$sent_at" "logship-network control container" -assert_isolated_probe() { - local description=$1 network=$2 target=$3 source_before shipped_before send_status=0 source_after shipped_after - source_before=$(wait_metric 0) +check_isolated_probe() { + local description=$1 sender=$2 network=$3 target=$4 + local source_before shipped_before sent_at source_after shipped_after source_delta shipped_delta send_status=0 + source_before=$(fresh_metric_baseline) shipped_before=$(vector_shipped_count) - if send_from_container "$network" "$target"; then + sent_at=$(metric_now_ms) + if send_probe "$sender" "$network" "$target"; then record "INFO: $description sender completed" else send_status=$? record "INFO: $description sender reported transport error exit=$send_status" fi - sleep 0.5 - source_after=$(read_metric) + source_after=$(wait_fresh_stable_metric "$sent_at") shipped_after=$(vector_shipped_count) - [[ "$source_after" =~ ^[0-9]+$ && "$shipped_after" =~ ^[0-9]+$ ]] || fail "invalid Vector counters during $description" - [[ "$source_after" == "$source_before" ]] || fail "$description isolation leak: source counter changed $source_before -> $source_after" - [[ "$shipped_after" == "$shipped_before" ]] || fail "$description isolation leak: shipped count changed $shipped_before -> $shipped_after" - record "PASS: $description isolation; source +0 and shipped +0" + source_delta=$((source_after - source_before)) + shipped_delta=$((shipped_after - shipped_before)) + record "INFO: $description fresh deltas source=$source_delta shipped=$shipped_delta" + (( send_status == 0 )) || return 2 + if (( source_delta == 0 && shipped_delta == 0 )); then + return 0 + fi + return 1 +} +assert_isolated_probe() { + local description=$1 sender=$2 network=$3 target=$4 status + if check_isolated_probe "$description" "$sender" "$network" "$target"; then + record "PASS: $description isolation; fresh source +0 and shipped +0" + return 0 + else + status=$? + fail "$description isolation check failed (status=$status)" + fi } +if check_isolated_probe "intentional host-loopback leak self-test" host host 127.0.0.1; then + fail "isolation leak self-test passed unexpectedly" +else + status=$? + (( status == 1 )) || fail "isolation leak self-test could not deliver its allowed host-loopback packet (status=$status)" + record "PASS: negative isolation check rejected a deliberately allowed host-loopback packet" +fi + probe_targets=("$vector_ip" "$app_gateway" vector host.docker.internal 127.0.0.1) for target in "${probe_targets[@]}"; do - assert_isolated_probe "node->$target" "container:$NODE_ID" "$target" - assert_isolated_probe "ingress->$target" "container:$INGRESS_ID" "$target" - assert_isolated_probe "postgres->$target" "container:$PG_ID" "$target" + assert_isolated_probe "node->$target" container "container:$NODE_ID" "$target" + assert_isolated_probe "ingress->$target" container "container:$INGRESS_ID" "$target" + assert_isolated_probe "postgres->$target" container "container:$PG_ID" "$target" done host_ips=$(ip -o -4 addr show scope global | awk '{split($4,a,"/"); print a[1]}') [[ -n "$host_ips" ]] || fail "no non-loopback IPv4 interface available for external-interface isolation probes" while read -r host_ip; do [[ -n "$host_ip" ]] || continue - source_before=$(wait_metric 0) - shipped_before=$(vector_shipped_count) - if node "$SENDER" "$host_ip" 12201 "$payload"; then - record "INFO: host external-interface sender completed to $host_ip" - else - send_status=$? - record "INFO: host external-interface sender reported transport error exit=$send_status for $host_ip" - fi - sleep 0.5 - source_after=$(read_metric) - shipped_after=$(vector_shipped_count) - [[ "$source_after" =~ ^[0-9]+$ && "$shipped_after" =~ ^[0-9]+$ ]] || fail "invalid Vector counters during host interface $host_ip probe" - [[ "$source_after" == "$source_before" ]] || fail "host interface $host_ip isolation leak: source $source_before -> $source_after" - [[ "$shipped_after" == "$shipped_before" ]] || fail "host interface $host_ip isolation leak: shipped $shipped_before -> $shipped_after" - record "PASS: host external interface $host_ip isolation; source +0 and shipped +0" + assert_isolated_probe "host external interface $host_ip" host host "$host_ip" done <<<"$host_ips" compose_cmd stop vector >/dev/null From 5cc0b36558c37470e04ca660796afc0674ab61d7 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 17:00:17 +0000 Subject: [PATCH 08/21] Make GELF acceptance checks executable in native tests --- .github/workflows/rust.yml | 3 +- scripts/tc-959-isolation.sh | 140 ++++++++++++++++++ scripts/tc-959-vector-metrics.mjs | 1 + scripts/tc-959-vector-metrics.test.mjs | 11 ++ scripts/test-tc-959-isolation.sh | 85 +++++++++++ tc-959-node-docker-check.review.sh | 197 +++++++------------------ 6 files changed, 294 insertions(+), 143 deletions(-) create mode 100644 scripts/tc-959-isolation.sh create mode 100755 scripts/test-tc-959-isolation.sh diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 6678ae1b..09c0f971 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -192,7 +192,8 @@ jobs: npm ci --ignore-scripts --no-audit --no-fund node scripts/node-vector-schema.mjs --check node scripts/node-vector-schema.mjs --check-compose - node --test scripts/node-vector-schema.test.mjs + node --test scripts/node-vector-schema.test.mjs scripts/tc-959-vector-metrics.test.mjs + bash scripts/test-tc-959-isolation.sh - name: Test Vector allowlist (0.59.0, checksum pinned) run: | set -eu diff --git a/scripts/tc-959-isolation.sh b/scripts/tc-959-isolation.sh new file mode 100644 index 00000000..3fa56ce5 --- /dev/null +++ b/scripts/tc-959-isolation.sh @@ -0,0 +1,140 @@ +wait_vector() { + local deadline=$((SECONDS + 20)) + while (( SECONDS < deadline )); do + VECTOR_ID=$(compose_cmd ps -q vector) + if [[ -n "$VECTOR_ID" ]] && [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] && curl --fail --silent --max-time 1 http://127.0.0.1:9598/metrics >/dev/null; then return 0; fi + sleep 0.2 + done + fail "Vector did not become running within 20 seconds" +} + +wait_node() { + local deadline=$((SECONDS + 30)) + while (( SECONDS < deadline )); do + if curl --fail --silent --max-time 2 http://127.0.0.1:18000/version >/dev/null; then return 0; fi + sleep 0.5 + done + fail "node did not serve /version within 30 seconds" +} + +wait_metric() { + local deadline=$((SECONDS + 20)) value + while (( SECONDS < deadline )); do + value=$(read_metric) + if [[ "$value" =~ ^[0-9]+$ ]] && (( value >= ${1:-1} )); then printf '%s\n' "$value"; return 0; fi + sleep 0.25 + done + fail "Vector GELF receive metric did not reach ${1:-1}" +} + +wait_next_line() { + local previous=${1:-0} deadline=$((SECONDS + 20)) current + touch "$CONTROL/pulse" + while (( SECONDS < deadline )); do + wait_vector + current=$(vector_event_count) + if (( current > previous )); then + sleep 0.4 + current=$(vector_event_count) + (( current == previous + 1 )) || fail "duplicate/flood output after Vector recovery: expected one line, saw $((current - previous))" + return 0 + fi + sleep 0.2 + done + fail "next labelled line not observed within 20 seconds" +} + +wait_fresh_stable_metric() { + local after_ms=$1 + local deadline=$((SECONDS + 20)) + local settle_until + settle_until=$((after_ms + ISOLATION_SETTLE_SECONDS * 1000)) + local sample value timestamp previous_timestamp=-1 previous_value=-1 now + while (( SECONDS < deadline )); do + sample=$(read_metric_sample) + if [[ "$sample" =~ ^([0-9]+)[[:space:]]([0-9]+)$ ]]; then + value=${BASH_REMATCH[1]} + timestamp=${BASH_REMATCH[2]} + if (( timestamp > after_ms && timestamp > previous_timestamp )); then + now=$(metric_now_ms) + if (( value == previous_value && now >= settle_until )); then + printf '%s\n' "$value" + return 0 + fi + previous_value=$value + previous_timestamp=$timestamp + fi + fi + sleep 0.2 + done + fail "Vector source metric did not produce two fresh, stable samples after timestamp $after_ms" +} + +fresh_sample() { + local started + started=$(metric_now_ms) + wait_fresh_stable_metric "$started" +} + +assert_positive_delivery() { + local source_before=$1 shipped_before=$2 sent_at=$3 description=$4 + local source_after source_delta deadline=$((SECONDS + 20)) shipped_after shipped_delta + source_after=$(wait_fresh_stable_metric "$sent_at") + source_delta=$((source_after - source_before)) + (( source_delta == 1 )) || fail "$description positive control source delta was $source_delta, expected +1" + while (( SECONDS < deadline )); do + shipped_after=$(vector_shipped_count) + shipped_delta=$((shipped_after - shipped_before)) + if (( shipped_delta == 1 )); then + record "PASS: $description positive control; fresh source +1 and shipped +1" + return 0 + fi + (( shipped_delta <= 1 )) || fail "$description produced duplicate shipped events: +$shipped_delta" + sleep 0.2 + done + fail "$description positive control shipped delta was $shipped_delta, expected +1" +} + +check_isolated_probe() { + local description=$1 sender=$2 network=$3 target=$4 + local source_before shipped_before sent_at source_after shipped_after source_delta shipped_delta send_status=0 + source_before=$(fresh_sample) + shipped_before=$(vector_shipped_count) + sent_at=$(metric_now_ms) + if send_probe "$sender" "$network" "$target"; then + record "INFO: $description sender completed" + else + send_status=$? + record "INFO: $description sender reported transport error exit=$send_status" + fi + source_after=$(wait_fresh_stable_metric "$sent_at") + shipped_after=$(vector_shipped_count) + source_delta=$((source_after - source_before)) + shipped_delta=$((shipped_after - shipped_before)) + record "INFO: $description fresh deltas source=$source_delta shipped=$shipped_delta" + (( send_status == 0 )) || return 2 + if (( source_delta == 0 && shipped_delta == 0 )); then return 0; fi + return 1 +} + +assert_isolated_probe() { + local description=$1 sender=$2 network=$3 target=$4 status + if check_isolated_probe "$description" "$sender" "$network" "$target"; then + record "PASS: $description isolation; fresh source +0 and shipped +0" + return 0 + else + status=$? + fail "$description isolation check failed (status=$status)" + fi +} + +run_isolation_leak_self_test() { + local sender=$1 network=$2 target=$3 status + if check_isolated_probe "intentional host-loopback leak self-test" "$sender" "$network" "$target"; then + fail "isolation leak self-test passed unexpectedly" + else + status=$? + (( status == 1 )) || fail "isolation leak self-test could not deliver its allowed packet (status=$status)" + record "PASS: negative isolation check rejected a deliberately allowed host-loopback packet" + fi +} diff --git a/scripts/tc-959-vector-metrics.mjs b/scripts/tc-959-vector-metrics.mjs index 5e4c52c2..2451c9b9 100644 --- a/scripts/tc-959-vector-metrics.mjs +++ b/scripts/tc-959-vector-metrics.mjs @@ -66,6 +66,7 @@ if (fileURLToPath(import.meta.url) === process.argv[1]) { const value = parseGelfReceivedCounter(input); process.stdout.write(value === null ? "missing" : String(value)); } else if (operation === "received-sample") { + const sample = parseGelfReceivedSample(input); process.stdout.write(sample === null ? "missing" : `${sample.value} ${sample.timestamp}`); } else if (operation === "shipped-count") { process.stdout.write(String(countShippedEvents(input))); diff --git a/scripts/tc-959-vector-metrics.test.mjs b/scripts/tc-959-vector-metrics.test.mjs index 7e54ce78..d92eccbb 100644 --- a/scripts/tc-959-vector-metrics.test.mjs +++ b/scripts/tc-959-vector-metrics.test.mjs @@ -1,3 +1,5 @@ +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; import assert from "node:assert/strict"; import test from "node:test"; import { parseGelfReceivedSample } from "./tc-959-vector-metrics.mjs"; @@ -10,3 +12,12 @@ test("parses the counter value and exporter sample timestamp separately", () => test("requires a timestamp for freshness-sensitive source samples", () => { assert.equal(parseGelfReceivedSample(`vector_component_received_events_total{component_id="gelf_in",component_kind="source",component_type="socket",host="vector",mode="udp"} 1\n`), null); }); +test("received-sample CLI prints value and timestamp for a captured exporter sample", () => { + const input = `vector_component_received_events_total{component_id="gelf_in",component_kind="source",component_type="socket",host="vector",mode="udp"} 1 1791992950070\n`; + const result = spawnSync(process.execPath, [fileURLToPath(new URL("./tc-959-vector-metrics.mjs", import.meta.url)), "received-sample"], { + input, + encoding: "utf8", + }); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, "1 1791992950070"); +}); diff --git a/scripts/test-tc-959-isolation.sh b/scripts/test-tc-959-isolation.sh new file mode 100755 index 00000000..f09d8cdf --- /dev/null +++ b/scripts/test-tc-959-isolation.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT=$(cd "$(dirname "$0")/.." && pwd) +source "$ROOT/scripts/tc-959-isolation.sh" + +METRICS_SCRAPE_INTERVAL_SECONDS=1 +ISOLATION_SETTLE_SECONDS=2 +TEST_NOW_MS=3000 +METRIC_SAMPLES=() +SHIPPED_SAMPLES=() +SEND_COUNT=0 + +read_metric_sample() { + local sample + IFS= read -r sample <&3 || { printf 'no captured metric sample\n' >&2; return 1; } + printf '%s\n' "$sample" +} +metric_now_ms() { printf '%s\n' "$TEST_NOW_MS"; } +vector_shipped_count() { + local sample + IFS= read -r sample <&4 || { printf 'no captured shipped count\n' >&2; return 1; } + printf '%s\n' "$sample" +} +compose_cmd() { printf '%s\n' test-vector; } +docker_cmd() { [[ "$1" == inspect ]] && printf '%s\n' true; } +curl() { return 0; } +read_metric() { printf '%s\n' 3; } +vector_event_count() { + local sample + IFS= read -r sample <&5 || return 1 + printf '%s\n' "$sample" +} +sleep() { :; } +CONTROL=$(mktemp -d) +trap 'rm -rf "$CONTROL"' EXIT + +send_probe() { SEND_COUNT=$((SEND_COUNT + 1)); } +record() { :; } +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } +assert_eq() { [[ "$1" == "$2" ]] || fail "expected '$2', got '$1'"; } +reset_fixtures() { + SEND_COUNT=0 + exec 3< <(printf '%s\n' "${METRIC_SAMPLES[@]}") + exec 4< <(printf '%s\n' "${SHIPPED_SAMPLES[@]}") +} +wait_vector +wait_node +assert_eq "$(wait_metric 3)" 3 +exec 5< <(printf '%s\n' 1 1) +wait_next_line 0 + +# Fresh samples must be newer than the send and stable, after the 2s settle. +reset_fixtures +METRIC_SAMPLES=('10 2000' '10 3000') +reset_fixtures +assert_eq "$(wait_fresh_stable_metric 1000)" 10 + +# Positive controls accept exactly one source and one shipped event. +reset_fixtures +TEST_NOW_MS=6000 +ISOLATION_SETTLE_SECONDS=0 +METRIC_SAMPLES=('6 5000' '6 6000') +SHIPPED_SAMPLES=(5) +reset_fixtures +assert_positive_delivery 5 4 4000 'captured positive control' + +# A negative probe remains isolated when fresh source and sink counts do not move. +reset_fixtures +TEST_NOW_MS=7000 +METRIC_SAMPLES=('10 8000' '10 9000' '10 10000' '10 11000') +SHIPPED_SAMPLES=(5 5) +reset_fixtures +assert_isolated_probe 'captured isolated probe' container test-network 127.0.0.1 +assert_eq "$SEND_COUNT" 1 + +# The same negative check must reject an allowed host-loopback delivery. +reset_fixtures +TEST_NOW_MS=12000 +METRIC_SAMPLES=('10 13000' '10 14000' '11 15000' '11 16000') +SHIPPED_SAMPLES=(5 6) +reset_fixtures +run_isolation_leak_self_test host host 127.0.0.1 +assert_eq "$SEND_COUNT" 1 +printf '%s\n' 'PASS: isolation freshness, positive controls, and leak self-test' diff --git a/tc-959-node-docker-check.review.sh b/tc-959-node-docker-check.review.sh index 05595035..8c6c9156 100755 --- a/tc-959-node-docker-check.review.sh +++ b/tc-959-node-docker-check.review.sh @@ -75,6 +75,7 @@ trap cleanup EXIT fail() { record "FAIL: $*"; exit 1; } require() { command -v "$1" >/dev/null 2>&1 || fail "missing command: $1"; } +source "$ROOT/scripts/tc-959-isolation.sh" record "TC-959 GELF/UDP acceptance; project=$PROJECT; report=$REPORT" record "This script is review-only. Run only after the lead authorizes Docker." @@ -97,6 +98,7 @@ NODE node "$ROOT/scripts/node-vector-schema.mjs" --check || fail "generated schema check failed" node "$ROOT/scripts/node-vector-schema.mjs" --check-compose || fail "embedded Compose parity failed" node --test "$ROOT/scripts/node-vector-schema.test.mjs" "$ROOT/scripts/tc-959-vector-metrics.test.mjs" || fail "schema, packaging, and metrics tests failed" +bash "$ROOT/scripts/test-tc-959-isolation.sh" || fail "freshness and isolation helper tests failed" "$VECTOR_BIN" validate --no-environment "$NATIVE_VECTOR_CONFIG" || fail "native Vector validation failed" "$VECTOR_BIN" test "$NATIVE_VECTOR_CONFIG" "$ROOT/log-shipper/vector.tests.yaml" || fail "native Vector tests failed" record "PASS: native checks used only a disposable dummy token fixture before any Docker operation" @@ -195,9 +197,8 @@ services: - "$CONTROL:/control" ports: !override [] vector: + entrypoint: !override ["/bin/sh", "-ec"] command: - - /bin/sh - - -ec - mkdir -p /var/lib/vector/secrets && printf '%s\\n' tc959-dummy-token-only > /var/lib/vector/secrets/bs_token && chmod 0644 /var/lib/vector/secrets/bs_token && exec vector --config /etc/vector/acceptance.yaml volumes: - "$VECTOR_TEST_CONFIG:/etc/vector/acceptance.yaml:ro" @@ -221,36 +222,66 @@ services: max-size: "10m" max-file: "3" EOF -node - "$OVERRIDE" "$ROOT" <<'NODE' +node - "$OVERRIDE" "$ROOT" "$ROOT/tc-959-node-docker-check.review.sh" "$VECTOR_TEST_CONFIG" <<'NODE' const fs = require('node:fs'); const { spawnSync } = require('node:child_process'); const { createRequire } = require('node:module'); const requireFromRoot = createRequire(`${process.argv[3]}/package.json`); const { parse } = requireFromRoot('yaml'); -const path = process.argv[2]; -const text = fs.readFileSync(path, 'utf8'); -const compose = parse(text.replaceAll('!override', '')); -for (const service of ['dstack-ingress', 'vector']) { - const argv = compose.services?.[service]?.command; - if (!Array.isArray(argv) || argv.length !== 3 || argv[0] !== '/bin/sh' || argv[1] !== '-ec' || typeof argv[2] !== 'string') { - throw new Error(`${service} rendered Compose argv must be [/bin/sh, -ec, script]`); - } - const renderedArgv = argv.map((argument) => argument.replaceAll('$$', '$')); - if (renderedArgv[0] !== '/bin/sh' || renderedArgv[1] !== '-ec') throw new Error(`${service} rendered argv changed`); - const syntax = spawnSync('sh', ['-n'], { input: renderedArgv[2], encoding: 'utf8' }); - if (syntax.status !== 0) throw new Error(`${service} generated shell is invalid: ${syntax.stderr}`); +const [overridePath, root, harnessPath, vectorConfigPath] = process.argv.slice(2); +const composeText = fs.readFileSync(overridePath, 'utf8'); +const compose = parse(composeText.replaceAll('!override', '')); +const ingressCommand = compose.services?.['dstack-ingress']?.command; +if (!Array.isArray(ingressCommand) || ingressCommand.length !== 3 || + ingressCommand[0] !== '/bin/sh' || ingressCommand[1] !== '-ec' || typeof ingressCommand[2] !== 'string') { + throw new Error('dstack-ingress effective Compose argv must be [/bin/sh, -ec, script]'); +} +const vector = compose.services?.vector; +if (JSON.stringify(vector?.entrypoint) !== JSON.stringify(['/bin/sh', '-ec']) || + !Array.isArray(vector.command) || vector.command.length !== 1 || typeof vector.command[0] !== 'string') { + throw new Error('vector must override the production entrypoint and use a one-script command'); +} +const effectiveArgv = [...vector.entrypoint, ...vector.command].map((argument) => argument.replaceAll('$$', '$')); +const expectedVectorScript = String.raw`mkdir -p /var/lib/vector/secrets && printf '%s\n' tc959-dummy-token-only > /var/lib/vector/secrets/bs_token && chmod 0644 /var/lib/vector/secrets/bs_token && exec vector --config /etc/vector/acceptance.yaml`; +const expectedVectorArgv = ['/bin/sh', '-ec', expectedVectorScript]; +if (JSON.stringify(effectiveArgv) !== JSON.stringify(expectedVectorArgv)) { + throw new Error(`vector effective argv mismatch: ${JSON.stringify(effectiveArgv)}`); } -const ingress = compose.services['dstack-ingress'].command[2]; -for (const fragment of [ - 'count=$$(cat /control/burst)', - 'while [ "$$i" -lt "$$count" ]', - 'i=$$((i + 1))', -]) { +const vectorConfig = fs.readFileSync(vectorConfigPath, 'utf8'); +for (const fragment of ['allowlist:', 'budget_size_guard:', 'local_console:', 'metrics_exporter:']) { + if (!vectorConfig.includes(fragment)) throw new Error(`acceptance Vector config is missing ${fragment}`); +} +for (const [name, argv] of [['dstack-ingress', ingressCommand.map((argument) => argument.replaceAll('$$', '$'))], ['vector', effectiveArgv]]) { + const syntax = spawnSync('sh', ['-n'], { input: argv[2], encoding: 'utf8' }); + if (syntax.status !== 0) throw new Error(`${name} generated shell is invalid: ${syntax.stderr}`); +} +const ingress = ingressCommand[2]; +for (const fragment of ['count=$$(cat /control/burst)', 'while [ "$$i" -lt "$$count" ]', 'i=$$((i + 1))']) { if (!ingress.includes(fragment)) throw new Error(`rendered producer command lost Compose escaping: ${fragment}`); } +const libraryPath = `${root}/scripts/tc-959-isolation.sh`; +const shellSources = [fs.readFileSync(harnessPath, 'utf8'), fs.readFileSync(libraryPath, 'utf8')]; +const declared = new Set(shellSources.flatMap((source) => + [...source.matchAll(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*\(\)\s*\{/gm)].map((match) => match[1]))); +const calledHelpers = [ + 'record', 'docker_cmd', 'compose_cmd', 'memory_sampler', 'cleanup', 'fail', 'require', + 'wait_vector', 'wait_node', 'read_metric', 'read_metric_sample', 'metric_now_ms', + 'wait_fresh_stable_metric', 'fresh_sample', 'wait_metric', 'vector_shipped_count', + 'vector_event_count', 'wait_next_line', 'send_from_container', 'send_probe', + 'assert_positive_delivery', 'check_isolated_probe', 'assert_isolated_probe', + 'run_isolation_leak_self_test', +]; +const missing = calledHelpers.filter((name) => !declared.has(name)); +if (missing.length) throw new Error(`harness calls undefined shell functions: ${missing.join(', ')}`); +for (const shellFile of [harnessPath, libraryPath, `${root}/scripts/test-tc-959-isolation.sh`]) { + const syntax = spawnSync('bash', ['-n', shellFile], { encoding: 'utf8' }); + if (syntax.status !== 0) throw new Error(`${shellFile} has invalid Bash syntax: ${syntax.stderr}`); +} NODE if [[ "$DRY_RUN" == 1 ]]; then record "DRY-RUN: no Docker command invoked" + docker_cmd version + compose_cmd config record "DRY-RUN: acceptance self-test expects an allowed host-loopback packet to make the negative check fail" record "DRY-RUN: positive host-loopback and logship-control-container probes require fresh source +1 and shipped +1" record "DRY-RUN: node/ingress/Postgres sibling and host external-interface probes require two newer stable source samples and shipped +0" @@ -274,15 +305,6 @@ touch "$SAMPLER_STOP" memory_sampler & SAMPLER_PID=$! -wait_vector() { - local deadline=$((SECONDS + 20)) - while (( SECONDS < deadline )); do - VECTOR_ID=$(compose_cmd ps -q vector) - if [[ -n "$VECTOR_ID" ]] && [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] && curl --fail --silent --max-time 1 http://127.0.0.1:9598/metrics >/dev/null; then return 0; fi - sleep 0.2 - done - fail "Vector did not become running within 20 seconds" -} read_metric() { curl --fail --silent http://127.0.0.1:9598/metrics | node "$ROOT/scripts/tc-959-vector-metrics.mjs" received-counter } @@ -290,37 +312,6 @@ read_metric_sample() { curl --fail --silent http://127.0.0.1:9598/metrics | node "$ROOT/scripts/tc-959-vector-metrics.mjs" received-sample } metric_now_ms() { node -p 'Date.now()'; } -wait_fresh_stable_metric() { - local after_ms=$1 deadline=$((SECONDS + 20)) settle_until=$((after_ms + ISOLATION_SETTLE_SECONDS * 1000)) - local sample value timestamp previous_timestamp=-1 previous_value=-1 now - while (( SECONDS < deadline )); do - sample=$(read_metric_sample) - if [[ "$sample" =~ ^([0-9]+)[[:space:]]([0-9]+)$ ]]; then - value=${BASH_REMATCH[1]} - timestamp=${BASH_REMATCH[2]} - if (( timestamp > after_ms && timestamp > previous_timestamp )); then - now=$(metric_now_ms) - if (( value == previous_value && now >= settle_until )); then - printf '%s\n' "$value" - return 0 - fi - previous_value=$value - previous_timestamp=$timestamp - fi - fi - sleep 0.2 - done - fail "Vector source metric did not produce two fresh, stable samples after timestamp $after_ms" -} -wait_metric() { - local deadline=$((SECONDS + 20)) value - while (( SECONDS < deadline )); do - value=$(read_metric) - if [[ "$value" =~ ^[0-9]+$ ]] && (( value >= ${1:-1} )); then printf '%s\n' "$value"; return 0; fi - sleep 0.25 - done - fail "Vector GELF receive metric did not reach ${1:-1}" -} vector_shipped_count() { [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] || fail "refusing to read events from a stopped Vector container" docker_cmd logs "$VECTOR_ID" 2>&1 | node "$ROOT/scripts/tc-959-vector-metrics.mjs" shipped-count @@ -329,22 +320,6 @@ vector_event_count() { [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] || fail "refusing to read events from a stopped Vector container" docker_cmd logs "$VECTOR_ID" 2>&1 | node "$ROOT/scripts/tc-959-vector-metrics.mjs" event-count ingress_request_slow } -wait_next_line() { - local previous=${1:-0} deadline=$((SECONDS + 20)) current - touch "$CONTROL/pulse" - while (( SECONDS < deadline )); do - wait_vector - current=$(vector_event_count) - if (( current > previous )); then - sleep 0.4 - current=$(vector_event_count) - (( current == previous + 1 )) || fail "duplicate/flood output after Vector recovery: expected one line, saw $((current - previous))" - return 0 - fi - sleep 0.2 - done - fail "next labelled line not observed within 20 seconds" -} compose_cmd ps -q tinycloud >/dev/null NODE_ID=$(compose_cmd ps -q tinycloud) @@ -436,11 +411,6 @@ send_from_container() { local network=$1 target=$2 docker_cmd run --rm --network "$network" --mount "type=bind,src=$SENDER,dst=/sender.mjs,readonly" node:20-alpine node /sender.mjs "$target" 12201 "$payload" } -fresh_metric_baseline() { - local started - started=$(metric_now_ms) - wait_fresh_stable_metric "$started" -} send_probe() { local sender=$1 network=$2 target=$3 if [[ "$sender" == host ]]; then @@ -449,79 +419,22 @@ send_probe() { send_from_container "$network" "$target" fi } -assert_positive_delivery() { - local source_before=$1 shipped_before=$2 sent_at=$3 description=$4 - local source_after source_delta deadline=$((SECONDS + 20)) shipped_after shipped_delta - source_after=$(wait_fresh_stable_metric "$sent_at") - source_delta=$((source_after - source_before)) - (( source_delta == 1 )) || fail "$description positive control source delta was $source_delta, expected +1" - while (( SECONDS < deadline )); do - shipped_after=$(vector_shipped_count) - shipped_delta=$((shipped_after - shipped_before)) - if (( shipped_delta == 1 )); then - record "PASS: $description positive control; fresh source +1 and shipped +1" - return 0 - fi - (( shipped_delta <= 1 )) || fail "$description produced duplicate shipped events: +$shipped_delta" - sleep 0.2 - done - fail "$description positive control shipped delta was $shipped_delta, expected +1" -} # Positive controls precede the leak self-test and every forbidden-source assertion. -source_before=$(fresh_metric_baseline) +source_before=$(fresh_sample) shipped_before=$(vector_shipped_count) sent_at=$(metric_now_ms) node "$SENDER" 127.0.0.1 12201 "$payload" assert_positive_delivery "$source_before" "$shipped_before" "$sent_at" "host loopback" -source_before=$(fresh_metric_baseline) +source_before=$(fresh_sample) shipped_before=$(vector_shipped_count) sent_at=$(metric_now_ms) send_from_container "${PROJECT}_logship" "$vector_ip" assert_positive_delivery "$source_before" "$shipped_before" "$sent_at" "logship-network control container" -check_isolated_probe() { - local description=$1 sender=$2 network=$3 target=$4 - local source_before shipped_before sent_at source_after shipped_after source_delta shipped_delta send_status=0 - source_before=$(fresh_metric_baseline) - shipped_before=$(vector_shipped_count) - sent_at=$(metric_now_ms) - if send_probe "$sender" "$network" "$target"; then - record "INFO: $description sender completed" - else - send_status=$? - record "INFO: $description sender reported transport error exit=$send_status" - fi - source_after=$(wait_fresh_stable_metric "$sent_at") - shipped_after=$(vector_shipped_count) - source_delta=$((source_after - source_before)) - shipped_delta=$((shipped_after - shipped_before)) - record "INFO: $description fresh deltas source=$source_delta shipped=$shipped_delta" - (( send_status == 0 )) || return 2 - if (( source_delta == 0 && shipped_delta == 0 )); then - return 0 - fi - return 1 -} -assert_isolated_probe() { - local description=$1 sender=$2 network=$3 target=$4 status - if check_isolated_probe "$description" "$sender" "$network" "$target"; then - record "PASS: $description isolation; fresh source +0 and shipped +0" - return 0 - else - status=$? - fail "$description isolation check failed (status=$status)" - fi -} -if check_isolated_probe "intentional host-loopback leak self-test" host host 127.0.0.1; then - fail "isolation leak self-test passed unexpectedly" -else - status=$? - (( status == 1 )) || fail "isolation leak self-test could not deliver its allowed host-loopback packet (status=$status)" - record "PASS: negative isolation check rejected a deliberately allowed host-loopback packet" -fi +run_isolation_leak_self_test host host 127.0.0.1 probe_targets=("$vector_ip" "$app_gateway" vector host.docker.internal 127.0.0.1) for target in "${probe_targets[@]}"; do From e0417b2a83240036a28a6d8d539eb26f8eec17b1 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 17:33:44 +0000 Subject: [PATCH 09/21] Fix Vector acceptance environment isolation --- .github/workflows/rust.yml | 3 +- scripts/tc-959-compose-env.mjs | 55 ++++++++++++++++++++++++++++ scripts/tc-959-compose-env.test.mjs | 52 ++++++++++++++++++++++++++ scripts/tc-959-vector-env-replay.mjs | 26 +++++++++++++ tc-959-node-docker-check.review.sh | 48 +++++++++++++++++++++--- 5 files changed, 178 insertions(+), 6 deletions(-) create mode 100644 scripts/tc-959-compose-env.mjs create mode 100644 scripts/tc-959-compose-env.test.mjs create mode 100644 scripts/tc-959-vector-env-replay.mjs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 09c0f971..9387b8d2 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -192,7 +192,7 @@ jobs: npm ci --ignore-scripts --no-audit --no-fund node scripts/node-vector-schema.mjs --check node scripts/node-vector-schema.mjs --check-compose - node --test scripts/node-vector-schema.test.mjs scripts/tc-959-vector-metrics.test.mjs + node --test scripts/node-vector-schema.test.mjs scripts/tc-959-vector-metrics.test.mjs scripts/tc-959-compose-env.test.mjs bash scripts/test-tc-959-isolation.sh - name: Test Vector allowlist (0.59.0, checksum pinned) run: | @@ -214,6 +214,7 @@ jobs: chmod 600 /var/lib/vector/secrets/bs_token "${vector}" validate --no-environment log-shipper/vector.yaml "${vector}" test log-shipper/vector.yaml log-shipper/vector.tests.yaml + TC_VECTOR_BIN="${vector}" TC_DRY_RUN=1 bash ./tc-959-node-docker-check.review.sh - name: Verify generated capability artifacts match capabilities.json run: node scripts/gen-capabilities.mjs --check - name: Verify deployed Policy/v3 probe contract diff --git a/scripts/tc-959-compose-env.mjs b/scripts/tc-959-compose-env.mjs new file mode 100644 index 00000000..6fba4659 --- /dev/null +++ b/scripts/tc-959-compose-env.mjs @@ -0,0 +1,55 @@ +import { fileURLToPath } from "node:url"; + +const expectedNodeKeys = [ + "ROCKET_ADDRESS", + "TINYCLOUD_ADMIN_SECRET", + "TINYCLOUD_DATABASE__WRITE_FENCE", + "TINYCLOUD_KEYS_SECRET", + "TINYCLOUD_KEYS_TYPE", + "TINYCLOUD_LOG__FORMAT", + "TINYCLOUD_QUOTA_URL", + "TINYCLOUD_SHARE_EMAIL__ENABLED", + "TINYCLOUD_STORAGE_BLOCKS_TYPE", + "TINYCLOUD_STORAGE_DATABASE", + "TINYCLOUD_TEE__MODE", +].sort(); + +function serviceEnvironment(compose, serviceName) { + const environment = compose?.services?.[serviceName]?.environment; + if (environment === null || typeof environment !== "object" || Array.isArray(environment)) { + throw new Error(`${serviceName} environment must be an object`); + } + return environment; +} + +export function assertAcceptanceEnvironment(compose) { + const node = serviceEnvironment(compose, "tinycloud"); + const nodeKeys = Object.keys(node).sort(); + if (JSON.stringify(nodeKeys) !== JSON.stringify(expectedNodeKeys) || + node.TINYCLOUD_KEYS_TYPE !== "Static" || node.TINYCLOUD_TEE__MODE !== "Off") { + throw new Error("node environment differs from the test-only override"); + } + + const ingress = serviceEnvironment(compose, "dstack-ingress"); + if (Object.keys(ingress).length !== 0) throw new Error("ingress environment contains inherited production variables"); + + const postgres = serviceEnvironment(compose, "postgres"); + const postgresKeys = Object.keys(postgres); + if (postgresKeys.length !== 1 || postgresKeys[0] !== "POSTGRES_HOST_AUTH_METHOD" || postgres.POSTGRES_HOST_AUTH_METHOD !== "trust") { + throw new Error("Postgres environment differs from the test-only override"); + } + + const vector = serviceEnvironment(compose, "vector"); + const forbiddenVectorKeys = Object.keys(vector).filter((name) => name.startsWith("VECTOR_") || name.includes("TOKEN")); + if (forbiddenVectorKeys.length || Object.keys(vector).length) { + throw new Error(`vector environment contains inherited keys${forbiddenVectorKeys.length ? `: ${forbiddenVectorKeys.join(", ")}` : ""}`); + } +} + +if (fileURLToPath(import.meta.url) === process.argv[1]) { + let input = ""; + for await (const chunk of process.stdin) input += chunk; + const compose = JSON.parse(input); + assertAcceptanceEnvironment(compose); + process.stdout.write("PASS: effective Compose environments are isolated\n"); +} diff --git a/scripts/tc-959-compose-env.test.mjs b/scripts/tc-959-compose-env.test.mjs new file mode 100644 index 00000000..6a1b06be --- /dev/null +++ b/scripts/tc-959-compose-env.test.mjs @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { assertAcceptanceEnvironment } from "./tc-959-compose-env.mjs"; + +const nodeEnvironment = { + TINYCLOUD_KEYS_TYPE: "Static", + TINYCLOUD_KEYS_SECRET: "test-only", + TINYCLOUD_TEE__MODE: "Off", + TINYCLOUD_STORAGE_DATABASE: "postgres://postgres@postgres:5432/postgres?sslmode=disable", + TINYCLOUD_STORAGE_BLOCKS_TYPE: "Local", + TINYCLOUD_LOG__FORMAT: "Json", + TINYCLOUD_DATABASE__WRITE_FENCE: "true", + TINYCLOUD_SHARE_EMAIL__ENABLED: "false", + TINYCLOUD_ADMIN_SECRET: "", + TINYCLOUD_QUOTA_URL: "http://127.0.0.1:9", + ROCKET_ADDRESS: "0.0.0.0", +}; +const safe = { + services: { + tinycloud: { environment: nodeEnvironment }, + "dstack-ingress": { environment: {} }, + postgres: { environment: { POSTGRES_HOST_AUTH_METHOD: "trust" } }, + vector: { environment: {} }, + }, +}; + +test("accepts only the test node environment and isolated helper environments", () => { + assert.doesNotThrow(() => assertAcceptanceEnvironment(safe)); +}); + +test("rejects inherited Vector config and source-token variables", () => { + for (const key of ["VECTOR_CONFIG_YAML", "BETTERSTACK_SOURCE_TOKEN"]) { + const compose = structuredClone(safe); + compose.services.vector.environment[key] = "must-not-be-exposed"; + assert.throws(() => assertAcceptanceEnvironment(compose), /vector environment/); + } +}); + +test("rejects production credentials inherited by ingress", () => { + const compose = structuredClone(safe); + compose.services["dstack-ingress"].environment.CLOUDFLARE_API_TOKEN = "must-not-be-exposed"; + assert.throws(() => assertAcceptanceEnvironment(compose), /ingress environment/); +}); + +test("rejects node production settings and TINYCLOUD variables on Postgres", () => { + const compose = structuredClone(safe); + compose.services.tinycloud.environment.TINYCLOUD_SHARE_EMAIL__TRUST_BUNDLE_BASE64 = "must-not-be-exposed"; + assert.throws(() => assertAcceptanceEnvironment(compose), /node environment/); + const postgres = structuredClone(safe); + postgres.services.postgres.environment.TINYCLOUD_ADMIN_SECRET = "must-not-be-exposed"; + assert.throws(() => assertAcceptanceEnvironment(postgres), /Postgres environment/); +}); diff --git a/scripts/tc-959-vector-env-replay.mjs b/scripts/tc-959-vector-env-replay.mjs new file mode 100644 index 00000000..37b9a7e6 --- /dev/null +++ b/scripts/tc-959-vector-env-replay.mjs @@ -0,0 +1,26 @@ +import { readFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; + +const [vectorBinary, acceptanceConfig, embeddedConfigPath] = process.argv.slice(2); +if (!vectorBinary || !acceptanceConfig || !embeddedConfigPath) { + throw new Error("usage: tc-959-vector-env-replay.mjs VECTOR ACCEPTANCE_CONFIG EMBEDDED_CONFIG"); +} +const baseEnv = { PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", HOME: "/tmp", TMPDIR: "/tmp" }; +const contaminated = spawnSync(vectorBinary, ["--config", acceptanceConfig], { + env: { ...baseEnv, VECTOR_CONFIG_YAML: readFileSync(embeddedConfigPath, "utf8") }, + encoding: "utf8", + timeout: 10000, +}); +if (contaminated.status === 0 || !contaminated.stderr?.includes("Config file not found in path")) { + throw new Error("native Vector did not reproduce the VECTOR_CONFIG_YAML-as-path failure"); +} + +const cleaned = spawnSync(vectorBinary, ["--config", acceptanceConfig], { + env: baseEnv, + encoding: "utf8", + timeout: 2000, +}); +if (cleaned.error?.code !== "ETIMEDOUT" && cleaned.status !== 0) { + throw new Error("native Vector failed to start the acceptance config with VECTOR_CONFIG_YAML removed"); +} +process.stdout.write("PASS: native Vector reproduced the environment-as-path failure and started after config env removal\n"); diff --git a/tc-959-node-docker-check.review.sh b/tc-959-node-docker-check.review.sh index 8c6c9156..b7c23b73 100755 --- a/tc-959-node-docker-check.review.sh +++ b/tc-959-node-docker-check.review.sh @@ -10,6 +10,7 @@ CONTROL="$WORK/control" mkdir -p "$CONTROL" PROJECT="tc959-gelf-$$" OVERRIDE="$WORK/compose.override.yaml" +VECTOR_REPLAY_CONFIG="$WORK/vector.acceptance.native.yaml" VECTOR_TEST_CONFIG="$WORK/vector.acceptance.yaml" NATIVE_VECTOR_CONFIG="$WORK/vector.native.yaml" NATIVE_SECRET_DIR="$WORK/native-secrets" @@ -97,7 +98,7 @@ fs.writeFileSync(targetPath, config, { mode: 0o600 }); NODE node "$ROOT/scripts/node-vector-schema.mjs" --check || fail "generated schema check failed" node "$ROOT/scripts/node-vector-schema.mjs" --check-compose || fail "embedded Compose parity failed" -node --test "$ROOT/scripts/node-vector-schema.test.mjs" "$ROOT/scripts/tc-959-vector-metrics.test.mjs" || fail "schema, packaging, and metrics tests failed" +node --test "$ROOT/scripts/node-vector-schema.test.mjs" "$ROOT/scripts/tc-959-vector-metrics.test.mjs" "$ROOT/scripts/tc-959-compose-env.test.mjs" || fail "schema, packaging, metrics, and compose environment tests failed" bash "$ROOT/scripts/test-tc-959-isolation.sh" || fail "freshness and isolation helper tests failed" "$VECTOR_BIN" validate --no-environment "$NATIVE_VECTOR_CONFIG" || fail "native Vector validation failed" "$VECTOR_BIN" test "$NATIVE_VECTOR_CONFIG" "$ROOT/log-shipper/vector.tests.yaml" || fail "native Vector tests failed" @@ -119,9 +120,9 @@ DSTACK_GATEWAY_DOMAIN=invalid.test CERTBOT_EMAIL=acceptance@example.invalid EOF -node - "$ROOT/log-shipper/vector.yaml" "$VECTOR_TEST_CONFIG" <<'NODE' +node - "$ROOT/log-shipper/vector.yaml" "$VECTOR_TEST_CONFIG" "$VECTOR_REPLAY_CONFIG" "$NATIVE_SECRET_DIR" "$NATIVE_DATA_DIR" <<'NODE' const fs = require('node:fs'); -const [sourcePath, targetPath] = process.argv.slice(2); +const [sourcePath, targetPath, replayPath, secretDir, dataDir] = process.argv.slice(2); let config = fs.readFileSync(sourcePath, 'utf8'); const sinkAt = config.lastIndexOf('\nsinks:\n'); if (sinkAt < 0) throw new Error('canonical sink block not found'); @@ -132,8 +133,13 @@ const updatedSinkAt = config.lastIndexOf('\nsinks:\n'); config = config.slice(0, updatedSinkAt) + `\nsinks:\n local_console:\n type: console\n inputs: [budget_size_guard]\n encoding: { codec: json }\n metrics_exporter:\n type: prometheus_exporter\n inputs: [internal_metrics]\n address: 0.0.0.0:9598\n`; fs.writeFileSync(targetPath, config, { mode: 0o644 }); fs.chmodSync(targetPath, 0o644); +const replayConfig = config.replaceAll('/var/lib/vector/secrets', secretDir).replaceAll('/var/lib/vector', dataDir); +fs.writeFileSync(replayPath, replayConfig, { mode: 0o644 }); +fs.chmodSync(replayPath, 0o644); NODE "$VECTOR_BIN" validate --no-environment "$VECTOR_TEST_CONFIG" || fail "acceptance Vector config validation failed" +"$VECTOR_BIN" validate --no-environment "$VECTOR_REPLAY_CONFIG" || fail "native acceptance replay config validation failed" +node "$ROOT/scripts/tc-959-vector-env-replay.mjs" "$VECTOR_BIN" "$VECTOR_REPLAY_CONFIG" "$ROOT/log-shipper/vector.yaml" || fail "native Vector environment replay failed" cat >"$SENDER" <<'NODE' import { createSocket } from 'node:dgram'; @@ -175,6 +181,7 @@ services: - "host.docker.internal:host-gateway" dstack-ingress: image: alpine:3.21 + environment: !override {} command: - /bin/sh - -ec @@ -198,8 +205,9 @@ services: ports: !override [] vector: entrypoint: !override ["/bin/sh", "-ec"] + environment: !override {} command: - - mkdir -p /var/lib/vector/secrets && printf '%s\\n' tc959-dummy-token-only > /var/lib/vector/secrets/bs_token && chmod 0644 /var/lib/vector/secrets/bs_token && exec vector --config /etc/vector/acceptance.yaml + - mkdir -p /var/lib/vector/secrets && printf '%s\\n' tc959-dummy-token-only > /var/lib/vector/secrets/bs_token && chmod 0644 /var/lib/vector/secrets/bs_token && exec env -u VECTOR_CONFIG_YAML -u BETTERSTACK_SOURCE_TOKEN -u BETTERSTACK_SOURCE_HOST vector --config /etc/vector/acceptance.yaml volumes: - "$VECTOR_TEST_CONFIG:/etc/vector/acceptance.yaml:ro" ports: @@ -231,6 +239,33 @@ const { parse } = requireFromRoot('yaml'); const [overridePath, root, harnessPath, vectorConfigPath] = process.argv.slice(2); const composeText = fs.readFileSync(overridePath, 'utf8'); const compose = parse(composeText.replaceAll('!override', '')); +const baseCompose = parse(fs.readFileSync(`${root}/docker-compose.dstack-postgres.yaml`, 'utf8')); +const baseVectorEnv = baseCompose.services.vector.environment ?? {}; +if (!Object.hasOwn(baseVectorEnv, 'VECTOR_CONFIG_YAML') || !Object.hasOwn(baseVectorEnv, 'BETTERSTACK_SOURCE_TOKEN')) { + throw new Error('base Vector environment regression fixture is missing'); +} +const vectorEnv = compose.services?.vector?.environment; +if (!vectorEnv || Object.keys(vectorEnv).length !== 0) throw new Error('acceptance Vector environment must be explicitly empty'); +const ingressEnv = compose.services?.['dstack-ingress']?.environment; +if (!ingressEnv || Object.keys(ingressEnv).length !== 0) throw new Error('acceptance ingress environment must be explicitly empty'); +const postgresEnv = compose.services?.postgres?.environment ?? {}; +if (Object.keys(postgresEnv).some((name) => name.startsWith('TINYCLOUD_'))) { + throw new Error('acceptance Postgres environment must not inherit TINYCLOUD_* variables'); +} +const nodeEnv = compose.services?.tinycloud?.environment ?? {}; +if (nodeEnv.TINYCLOUD_KEYS_TYPE !== 'Static' || nodeEnv.TINYCLOUD_TEE__MODE !== 'Off') { + throw new Error('acceptance node lost its test-only environment override'); +} +const envCheck = spawnSync(process.execPath, [`${root}/scripts/tc-959-compose-env.mjs`], { + input: JSON.stringify({ services: { + tinycloud: { environment: nodeEnv }, + 'dstack-ingress': { environment: ingressEnv }, + postgres: { environment: postgresEnv }, + vector: { environment: vectorEnv }, + } }), + encoding: 'utf8', +}); +if (envCheck.status !== 0) throw new Error(`acceptance service environments failed: ${envCheck.stderr}`); const ingressCommand = compose.services?.['dstack-ingress']?.command; if (!Array.isArray(ingressCommand) || ingressCommand.length !== 3 || ingressCommand[0] !== '/bin/sh' || ingressCommand[1] !== '-ec' || typeof ingressCommand[2] !== 'string') { @@ -242,7 +277,7 @@ if (JSON.stringify(vector?.entrypoint) !== JSON.stringify(['/bin/sh', '-ec']) || throw new Error('vector must override the production entrypoint and use a one-script command'); } const effectiveArgv = [...vector.entrypoint, ...vector.command].map((argument) => argument.replaceAll('$$', '$')); -const expectedVectorScript = String.raw`mkdir -p /var/lib/vector/secrets && printf '%s\n' tc959-dummy-token-only > /var/lib/vector/secrets/bs_token && chmod 0644 /var/lib/vector/secrets/bs_token && exec vector --config /etc/vector/acceptance.yaml`; +const expectedVectorScript = String.raw`mkdir -p /var/lib/vector/secrets && printf '%s\n' tc959-dummy-token-only > /var/lib/vector/secrets/bs_token && chmod 0644 /var/lib/vector/secrets/bs_token && exec env -u VECTOR_CONFIG_YAML -u BETTERSTACK_SOURCE_TOKEN -u BETTERSTACK_SOURCE_HOST vector --config /etc/vector/acceptance.yaml`; const expectedVectorArgv = ['/bin/sh', '-ec', expectedVectorScript]; if (JSON.stringify(effectiveArgv) !== JSON.stringify(expectedVectorArgv)) { throw new Error(`vector effective argv mismatch: ${JSON.stringify(effectiveArgv)}`); @@ -295,6 +330,9 @@ require docker require curl require ip docker compose version >/dev/null || fail "Docker Compose v2 is required" +if ! "${COMPOSE[@]}" config --format json | node "$ROOT/scripts/tc-959-compose-env.mjs"; then + fail "rendered Compose environment isolation failed" +fi record "Preflight: no DockerRootDir assumption; no host Docker-directory access." record "Metrics freshness: internal_metrics scrape interval=${METRICS_SCRAPE_INTERVAL_SECONDS}s; isolation settle=${ISOLATION_SETTLE_SECONDS}s; require two strictly newer timestamps with stable source value." From 4060cf756319b79d0940d326442f88acdadd1b5a Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 17:41:28 +0000 Subject: [PATCH 10/21] Accept omitted Compose environments in harness --- scripts/tc-959-compose-env.mjs | 12 ++++++++- scripts/tc-959-compose-env.rendered.json | 26 +++++++++++++++++++ scripts/tc-959-compose-env.test.mjs | 33 ++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 scripts/tc-959-compose-env.rendered.json diff --git a/scripts/tc-959-compose-env.mjs b/scripts/tc-959-compose-env.mjs index 6fba4659..dd6f9a04 100644 --- a/scripts/tc-959-compose-env.mjs +++ b/scripts/tc-959-compose-env.mjs @@ -15,10 +15,20 @@ const expectedNodeKeys = [ ].sort(); function serviceEnvironment(compose, serviceName) { - const environment = compose?.services?.[serviceName]?.environment; + const service = compose?.services?.[serviceName]; + if (!service || typeof service !== "object" || Array.isArray(service)) { + throw new Error(`${serviceName} service must be an object`); + } + const environment = service.environment; + if (environment === undefined) return {}; if (environment === null || typeof environment !== "object" || Array.isArray(environment)) { throw new Error(`${serviceName} environment must be an object`); } + for (const value of Object.values(environment)) { + if (value !== null && typeof value !== "string") { + throw new Error(`${serviceName} environment values must be strings or null`); + } + } return environment; } diff --git a/scripts/tc-959-compose-env.rendered.json b/scripts/tc-959-compose-env.rendered.json new file mode 100644 index 00000000..3e735e57 --- /dev/null +++ b/scripts/tc-959-compose-env.rendered.json @@ -0,0 +1,26 @@ +{ + "services": { + "tinycloud": { + "environment": { + "ROCKET_ADDRESS": "", + "TINYCLOUD_ADMIN_SECRET": "", + "TINYCLOUD_DATABASE__WRITE_FENCE": "", + "TINYCLOUD_KEYS_SECRET": "", + "TINYCLOUD_KEYS_TYPE": "Static", + "TINYCLOUD_LOG__FORMAT": "", + "TINYCLOUD_QUOTA_URL": "", + "TINYCLOUD_SHARE_EMAIL__ENABLED": "", + "TINYCLOUD_STORAGE_BLOCKS_TYPE": "", + "TINYCLOUD_STORAGE_DATABASE": "", + "TINYCLOUD_TEE__MODE": "Off" + } + }, + "dstack-ingress": {}, + "postgres": { + "environment": { + "POSTGRES_HOST_AUTH_METHOD": "trust" + } + }, + "vector": {} + } +} diff --git a/scripts/tc-959-compose-env.test.mjs b/scripts/tc-959-compose-env.test.mjs index 6a1b06be..b5716825 100644 --- a/scripts/tc-959-compose-env.test.mjs +++ b/scripts/tc-959-compose-env.test.mjs @@ -1,7 +1,10 @@ import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; import test from "node:test"; import { assertAcceptanceEnvironment } from "./tc-959-compose-env.mjs"; +const rendered = JSON.parse(readFileSync(new URL("./tc-959-compose-env.rendered.json", import.meta.url), "utf8")); + const nodeEnvironment = { TINYCLOUD_KEYS_TYPE: "Static", TINYCLOUD_KEYS_SECRET: "test-only", @@ -24,6 +27,36 @@ const safe = { }, }; +test("accepts the sanitized real Compose rendering with omitted cleared environments", () => { + assert.equal(Object.hasOwn(rendered.services["dstack-ingress"], "environment"), false); + assert.equal(Object.hasOwn(rendered.services.vector, "environment"), false); + assert.doesNotThrow(() => assertAcceptanceEnvironment(rendered)); +}); + +test("rejects forbidden environment entries added to the rendered Compose snapshot", () => { + const contaminated = structuredClone(rendered); + contaminated.services.vector.environment = { VECTOR_CONFIG_YAML: "must-not-be-exposed" }; + assert.throws(() => assertAcceptanceEnvironment(contaminated), /vector environment/); +}); + +test("rejects missing services and malformed present environments", () => { + const missingService = structuredClone(rendered); + delete missingService.services.vector; + assert.throws(() => assertAcceptanceEnvironment(missingService), /vector service/); + + for (const environment of [null, "bad", [], { VECTOR_CONFIG_YAML: 4 }, { VECTOR_CONFIG_YAML: {} }]) { + const malformed = structuredClone(rendered); + malformed.services.vector.environment = environment; + assert.throws(() => assertAcceptanceEnvironment(malformed), /vector environment/); + } +}); + +test("accepts null Compose environment values", () => { + const compose = structuredClone(safe); + compose.services.tinycloud.environment.ROCKET_ADDRESS = null; + assert.doesNotThrow(() => assertAcceptanceEnvironment(compose)); +}); + test("accepts only the test node environment and isolated helper environments", () => { assert.doesNotThrow(() => assertAcceptanceEnvironment(safe)); }); From 43a6504706c1131a5c76cea15cf27e6abf2ff743 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 19:05:42 +0000 Subject: [PATCH 11/21] Fix Vector crash recovery acceptance check --- scripts/tc-959-isolation.sh | 42 ++++++++++++++++++++++++ scripts/test-tc-959-isolation.sh | 51 ++++++++++++++++++++++++++++++ tc-959-node-docker-check.review.sh | 14 +++----- 3 files changed, 98 insertions(+), 9 deletions(-) diff --git a/scripts/tc-959-isolation.sh b/scripts/tc-959-isolation.sh index 3fa56ce5..d9c6711a 100644 --- a/scripts/tc-959-isolation.sh +++ b/scripts/tc-959-isolation.sh @@ -1,3 +1,45 @@ +check_host_ports_free() { + local port listeners containers owner + for port in 18000 9598; do + listeners=$(ss -H -ltnp "sport = :$port") || fail "could not inspect host port 127.0.0.1:$port" + [[ -n "$listeners" ]] || continue + containers=$(docker_cmd ps --filter "publish=$port" --format '{{.Names}}' 2>/dev/null || true) + owner=$listeners + [[ -z "$containers" ]] || owner="$containers; $listeners" + fail "host port 127.0.0.1:$port is already in use: $owner" + done +} + +crash_vector() { + local previous_line_count=$1 restart_before container_pid target_restart + local restart_count="" running="" deadline + restart_before=$(docker_cmd inspect -f '{{.RestartCount}}' "$VECTOR_ID") + container_pid=$(docker_cmd inspect -f '{{.State.Pid}}' "$VECTOR_ID") + [[ "$restart_before" =~ ^[0-9]+$ ]] || fail "Vector RestartCount is invalid before SIGKILL: $restart_before" + [[ "$container_pid" =~ ^[1-9][0-9]*$ ]] || fail "Vector host PID is invalid before SIGKILL: $container_pid" + target_restart=$((restart_before + 1)) + kill -KILL "$container_pid" || fail "could not SIGKILL Vector host PID $container_pid" + + deadline=$((SECONDS + 20)) + while (( SECONDS < deadline )); do + restart_count=$(docker_cmd inspect -f '{{.RestartCount}}' "$VECTOR_ID" 2>/dev/null || true) + [[ "$restart_count" =~ ^[0-9]+$ ]] || { sleep 0.2; continue; } + if (( restart_count > target_restart )); then + fail "Vector RestartCount advanced past expected +1 after SIGKILL: before=$restart_before after=$restart_count" + fi + if (( restart_count == target_restart )); then + running=$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID" 2>/dev/null || true) + [[ "$running" == true ]] && break + fi + sleep 0.2 + done + [[ "$restart_count" == "$target_restart" ]] || + fail "Vector RestartCount did not increase by exactly one after SIGKILL: before=$restart_before after=${restart_count:-unavailable}" + [[ "$running" == true ]] || fail "Vector RestartCount increased but Vector is not running after SIGKILL" + wait_vector + wait_next_line "$previous_line_count" +} + wait_vector() { local deadline=$((SECONDS + 20)) while (( SECONDS < deadline )); do diff --git a/scripts/test-tc-959-isolation.sh b/scripts/test-tc-959-isolation.sh index f09d8cdf..ffeb2b50 100755 --- a/scripts/test-tc-959-isolation.sh +++ b/scripts/test-tc-959-isolation.sh @@ -83,3 +83,54 @@ reset_fixtures run_isolation_leak_self_test host host 127.0.0.1 assert_eq "$SEND_COUNT" 1 printf '%s\n' 'PASS: isolation freshness, positive controls, and leak self-test' + +# Host-port preflight reports the owning Compose service before startup. +PORT_HOLDER_MODE=container +ss() { + if [[ "$3" == "sport = :18000" && "$PORT_HOLDER_MODE" == container ]]; then + printf '%s\n' 'LISTEN 0 4096 127.0.0.1:18000 0.0.0.0:* users:(("docker-proxy",pid=123,fd=4))' + fi +} +docker_cmd() { + if [[ "$1" == ps && "$3" == publish=18000 ]]; then + printf '%s\n' 'tc959-existing-compose' + return 0 + fi + return 1 +} +port_error=$(check_host_ports_free 2>&1) && fail "busy host port passed preflight" +[[ "$port_error" == *"127.0.0.1:18000"* && "$port_error" == *"tc959-existing-compose"* ]] || fail "port preflight omitted listener owner: $port_error" + +# A real crash sends SIGKILL to the host PID and requires exactly one restart. +VECTOR_ID=vector-test +RESTART_COUNT=4 +KILLED_PID="" +RECOVERED_LINE_COUNT="" +docker_cmd() { + if [[ "$1" != inspect ]]; then return 1; fi + case "$3" in + '{{.RestartCount}}') printf '%s\n' "$RESTART_COUNT" ;; + '{{.State.Pid}}') printf '%s\n' 4242 ;; + '{{.State.Running}}') printf '%s\n' true ;; + *) return 1 ;; + esac +} +kill() { + [[ "$1" == -KILL && "$2" == 4242 ]] || fail "crash helper did not SIGKILL the host container PID" + KILLED_PID=$2 + RESTART_COUNT=5 +} +wait_vector() { :; } +wait_next_line() { RECOVERED_LINE_COUNT=$1; } +crash_vector 17 +assert_eq "$KILLED_PID" 4242 +assert_eq "$RECOVERED_LINE_COUNT" 17 + +# A SIGKILL without a restart must fail rather than look like a successful crash. +kill() { :; } +sleep() { SECONDS=$((SECONDS + 20)); } +RESTART_COUNT=5 +SECONDS=0 +restart_error=$(crash_vector 18 2>&1) && fail "crash helper accepted a kill without a restart" +[[ "$restart_error" == *"RestartCount did not increase by exactly one"* ]] || fail "crash helper gave no explicit restart-count failure: $restart_error" +printf '%s\n' 'PASS: port ownership preflight and SIGKILL restart-count checks' diff --git a/tc-959-node-docker-check.review.sh b/tc-959-node-docker-check.review.sh index b7c23b73..8cd09430 100755 --- a/tc-959-node-docker-check.review.sh +++ b/tc-959-node-docker-check.review.sh @@ -52,7 +52,7 @@ memory_sampler() { if [[ "$running" == true ]]; then current=$(command docker stats --no-stream --format '{{.MemUsage}}' "$container" 2>/dev/null | awk '{value=$1; amount=value; sub(/[A-Za-z].*$/, "", amount); unit=substr(value, length(amount)+1); multiplier=unit=="GiB"?1073741824:unit=="MiB"?1048576:unit=="KiB"?1024:1; printf "%.0f", amount*multiplier}' || true) [[ "$current" =~ ^[0-9]+$ ]] || current=0 - (( current > peak )) && peak=$current + if (( current > peak )); then peak=$current; fi fi fi printf '%s\n' "$peak" >"$WORK/vector-peak-bytes" @@ -300,6 +300,7 @@ const declared = new Set(shellSources.flatMap((source) => [...source.matchAll(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*\(\)\s*\{/gm)].map((match) => match[1]))); const calledHelpers = [ 'record', 'docker_cmd', 'compose_cmd', 'memory_sampler', 'cleanup', 'fail', 'require', + 'check_host_ports_free', 'crash_vector', 'wait_vector', 'wait_node', 'read_metric', 'read_metric_sample', 'metric_now_ms', 'wait_fresh_stable_metric', 'fresh_sample', 'wait_metric', 'vector_shipped_count', 'vector_event_count', 'wait_next_line', 'send_from_container', 'send_probe', @@ -329,6 +330,8 @@ fi require docker require curl require ip +require ss +check_host_ports_free docker compose version >/dev/null || fail "Docker Compose v2 is required" if ! "${COMPOSE[@]}" config --format json | node "$ROOT/scripts/tc-959-compose-env.mjs"; then fail "rendered Compose environment isolation failed" @@ -382,14 +385,7 @@ for mode in restart crash recreate; do ;; crash) before=$(vector_event_count) - docker_cmd kill --signal KILL "$VECTOR_ID" >/dev/null - deadline=$((SECONDS + 20)) - while (( SECONDS < deadline )); do - if [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID" 2>/dev/null || true)" == true ]]; then break; fi - sleep 0.2 - done - wait_vector - wait_next_line "$before" + crash_vector "$before" ;; recreate) compose_cmd up -d --force-recreate --no-deps vector >/dev/null From 00062be3dfc4c6690e952a598257c943e6f9bcee Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 19:28:00 +0000 Subject: [PATCH 12/21] Use init-supervised Vector crash proof --- docker-compose.dstack-postgres.yaml | 3 +- scripts/node-vector-schema.mjs | 6 ++ scripts/node-vector-schema.test.mjs | 8 ++ scripts/tc-959-compose-env.mjs | 3 + scripts/tc-959-compose-env.rendered.json | 4 +- scripts/tc-959-compose-env.test.mjs | 13 ++- scripts/tc-959-isolation.sh | 58 ++++++++---- scripts/test-tc-959-isolation.sh | 112 ++++++++++++++++++----- tc-959-node-docker-check.review.sh | 7 +- 9 files changed, 168 insertions(+), 46 deletions(-) diff --git a/docker-compose.dstack-postgres.yaml b/docker-compose.dstack-postgres.yaml index 27f76ae9..6a8cb8d9 100644 --- a/docker-compose.dstack-postgres.yaml +++ b/docker-compose.dstack-postgres.yaml @@ -136,8 +136,9 @@ services: max-size: "10m" max-file: "3" vector: - image: timberio/vector:0.59.0-alpine@sha256:0e36bc2a290039e9b6a5e92f04bd2f133223506f72d09682c1df16c015172b7a restart: unless-stopped + image: timberio/vector:0.59.0-alpine@sha256:0e36bc2a290039e9b6a5e92f04bd2f133223506f72d09682c1df16c015172b7a + init: true user: "10001:10001" read_only: true cap_drop: [ALL] diff --git a/scripts/node-vector-schema.mjs b/scripts/node-vector-schema.mjs index e4b530b3..22d7ded1 100644 --- a/scripts/node-vector-schema.mjs +++ b/scripts/node-vector-schema.mjs @@ -289,6 +289,12 @@ export function checkComposeDeploymentPackaging(compose) { } if (!isComposeMapping(definition)) throw new Error("Compose packaging must be a YAML mapping"); + const services = definition.services; + if (isComposeMapping(services) && Object.hasOwn(services, "vector") && + (!isComposeMapping(services.vector) || services.vector.init !== true)) { + throw new Error("Compose Vector service must set init: true"); + } + for (const config of Object.values(definition.configs ?? {})) { if (isComposeMapping(config) && Object.hasOwn(config, "file")) { throw new Error("Compose packaging must not use configs.*.file"); diff --git a/scripts/node-vector-schema.test.mjs b/scripts/node-vector-schema.test.mjs index 589ee1e1..dd0fa60d 100644 --- a/scripts/node-vector-schema.test.mjs +++ b/scripts/node-vector-schema.test.mjs @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import { readFileSync } from "node:fs"; import test from "node:test"; +import { parse } from "yaml"; import { checkComposeDeploymentPackaging, checkComposeVectorConfig, checkVectorFile, renderAllowlist, renderComposeVectorConfig } from "./node-vector-schema.mjs"; const workflow = readFileSync(new URL("../.github/workflows/rust.yml", import.meta.url), "utf8"); @@ -96,11 +97,18 @@ test("CI runs native Vector validation and tests after checking the pinned relea const minimalCompose = `services: vector: + init: true environment: VECTOR_CONFIG_YAML: | sources: {} entrypoint: ["vector"] `; + +test("Vector deployment enables init and rejects a missing init guard", () => { + assert.equal(parse(compose).services.vector.init, true); + assert.doesNotThrow(() => checkComposeVectorConfig(compose, vector)); + assert.throws(() => checkComposeDeploymentPackaging(compose.replace(" init: true\n", "")), /init: true/); +}); test("rejects an appended block-style configs.file entry", () => { assert.throws( () => checkComposeVectorConfig(`${minimalCompose}\nconfigs:\n x:\n file: ./outside.yaml\n`, "sources: {}"), diff --git a/scripts/tc-959-compose-env.mjs b/scripts/tc-959-compose-env.mjs index dd6f9a04..a8d7ef4c 100644 --- a/scripts/tc-959-compose-env.mjs +++ b/scripts/tc-959-compose-env.mjs @@ -49,7 +49,10 @@ export function assertAcceptanceEnvironment(compose) { throw new Error("Postgres environment differs from the test-only override"); } + const vector = serviceEnvironment(compose, "vector"); + const vectorService = compose.services.vector; + if (vectorService.init !== true) throw new Error("Compose Vector service must set init: true"); const forbiddenVectorKeys = Object.keys(vector).filter((name) => name.startsWith("VECTOR_") || name.includes("TOKEN")); if (forbiddenVectorKeys.length || Object.keys(vector).length) { throw new Error(`vector environment contains inherited keys${forbiddenVectorKeys.length ? `: ${forbiddenVectorKeys.join(", ")}` : ""}`); diff --git a/scripts/tc-959-compose-env.rendered.json b/scripts/tc-959-compose-env.rendered.json index 3e735e57..d8ae7969 100644 --- a/scripts/tc-959-compose-env.rendered.json +++ b/scripts/tc-959-compose-env.rendered.json @@ -21,6 +21,8 @@ "POSTGRES_HOST_AUTH_METHOD": "trust" } }, - "vector": {} + "vector": { + "init": true + } } } diff --git a/scripts/tc-959-compose-env.test.mjs b/scripts/tc-959-compose-env.test.mjs index b5716825..135e2b9e 100644 --- a/scripts/tc-959-compose-env.test.mjs +++ b/scripts/tc-959-compose-env.test.mjs @@ -23,16 +23,27 @@ const safe = { tinycloud: { environment: nodeEnvironment }, "dstack-ingress": { environment: {} }, postgres: { environment: { POSTGRES_HOST_AUTH_METHOD: "trust" } }, - vector: { environment: {} }, + vector: { init: true, environment: {} }, }, }; test("accepts the sanitized real Compose rendering with omitted cleared environments", () => { assert.equal(Object.hasOwn(rendered.services["dstack-ingress"], "environment"), false); assert.equal(Object.hasOwn(rendered.services.vector, "environment"), false); + assert.equal(rendered.services.vector.init, true); assert.doesNotThrow(() => assertAcceptanceEnvironment(rendered)); }); +test("requires init on the rendered Vector service", () => { + const missing = structuredClone(safe); + delete missing.services.vector.init; + assert.throws(() => assertAcceptanceEnvironment(missing), /Vector service must set init: true/); + const disabled = structuredClone(safe); + disabled.services.vector.init = false; + assert.throws(() => assertAcceptanceEnvironment(disabled), /Vector service must set init: true/); + assert.doesNotThrow(() => assertAcceptanceEnvironment(safe)); +}); + test("rejects forbidden environment entries added to the rendered Compose snapshot", () => { const contaminated = structuredClone(rendered); contaminated.services.vector.environment = { VECTOR_CONFIG_YAML: "must-not-be-exposed" }; diff --git a/scripts/tc-959-isolation.sh b/scripts/tc-959-isolation.sh index d9c6711a..e7dc5e7a 100644 --- a/scripts/tc-959-isolation.sh +++ b/scripts/tc-959-isolation.sh @@ -11,32 +11,56 @@ check_host_ports_free() { } crash_vector() { - local previous_line_count=$1 restart_before container_pid target_restart - local restart_count="" running="" deadline - restart_before=$(docker_cmd inspect -f '{{.RestartCount}}' "$VECTOR_ID") - container_pid=$(docker_cmd inspect -f '{{.State.Pid}}' "$VECTOR_ID") - [[ "$restart_before" =~ ^[0-9]+$ ]] || fail "Vector RestartCount is invalid before SIGKILL: $restart_before" - [[ "$container_pid" =~ ^[1-9][0-9]*$ ]] || fail "Vector host PID is invalid before SIGKILL: $container_pid" + local previous_line_count=$1 snapshot snapshot_id restart_before started_before + local pids vector_pid comm target_restart current current_id current_restart current_started current_running + local deadline + local -a vector_pids=() + + snapshot=$(docker_cmd inspect -f '{{.Id}}|{{.RestartCount}}|{{.State.StartedAt}}' "$VECTOR_ID") || + fail "could not snapshot Vector container before crash" + IFS='|' read -r snapshot_id restart_before started_before <<<"$snapshot" + [[ "$snapshot_id" == "$VECTOR_ID" ]] || fail "Vector container ID changed before crash" + [[ "$restart_before" =~ ^[0-9]+$ && -n "$started_before" ]] || fail "Vector crash snapshot is invalid" + + pids=$(docker_cmd exec --user 10001:10001 "$VECTOR_ID" busybox pidof vector 2>/dev/null || true) + read -r -a vector_pids <<<"$pids" + (( ${#vector_pids[@]} == 1 )) || fail "expected exactly one Vector PID in container, found ${#vector_pids[@]}" + vector_pid=${vector_pids[0]} + [[ "$vector_pid" =~ ^[0-9]+$ ]] || fail "Vector PID is not numeric: $vector_pid" + (( vector_pid > 1 )) || fail "Vector PID must be greater than 1" + comm=$(docker_cmd exec --user 10001:10001 "$VECTOR_ID" cat "/proc/$vector_pid/comm" 2>/dev/null) || + fail "could not read Vector process comm" + [[ "$comm" == vector ]] || fail "container process PID $vector_pid comm is not vector" + target_restart=$((restart_before + 1)) - kill -KILL "$container_pid" || fail "could not SIGKILL Vector host PID $container_pid" + docker_cmd exec --user 10001:10001 "$VECTOR_ID" busybox kill -KILL "$vector_pid" >/dev/null || + fail "container-scoped SIGKILL failed; crash signal was not retried" deadline=$((SECONDS + 20)) + current="" while (( SECONDS < deadline )); do - restart_count=$(docker_cmd inspect -f '{{.RestartCount}}' "$VECTOR_ID" 2>/dev/null || true) - [[ "$restart_count" =~ ^[0-9]+$ ]] || { sleep 0.2; continue; } - if (( restart_count > target_restart )); then - fail "Vector RestartCount advanced past expected +1 after SIGKILL: before=$restart_before after=$restart_count" + current=$(docker_cmd inspect -f '{{.Id}}|{{.RestartCount}}|{{.State.StartedAt}}|{{.State.Running}}' "$VECTOR_ID" 2>/dev/null || true) + IFS='|' read -r current_id current_restart current_started current_running <<<"$current" + [[ -n "$current_id" ]] || { sleep 0.2; continue; } + [[ "$current_id" == "$snapshot_id" ]] || fail "Vector container ID changed during crash recovery" + [[ "$current_restart" =~ ^[0-9]+$ ]] || { sleep 0.2; continue; } + if (( current_restart > target_restart )); then + fail "Vector RestartCount advanced past expected +1: before=$restart_before after=$current_restart" fi - if (( restart_count == target_restart )); then - running=$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID" 2>/dev/null || true) - [[ "$running" == true ]] && break + if (( current_restart == target_restart )) && + [[ "$current_started" > "$started_before" && "$current_running" == true ]]; then + break fi sleep 0.2 done - [[ "$restart_count" == "$target_restart" ]] || - fail "Vector RestartCount did not increase by exactly one after SIGKILL: before=$restart_before after=${restart_count:-unavailable}" - [[ "$running" == true ]] || fail "Vector RestartCount increased but Vector is not running after SIGKILL" + + [[ "$current_id" == "$snapshot_id" ]] || fail "Vector container ID changed during crash recovery" + [[ "$current_restart" =~ ^[0-9]+$ ]] && (( current_restart == target_restart )) || + fail "Vector RestartCount did not increase by exactly one: before=$restart_before after=${current_restart:-unavailable}" + [[ "$current_started" > "$started_before" ]] || fail "Vector StartedAt did not advance after crash" + [[ "$current_running" == true ]] || fail "Vector was not running after crash recovery" wait_vector + [[ "$VECTOR_ID" == "$snapshot_id" ]] || fail "Vector container ID changed while waiting for output" wait_next_line "$previous_line_count" } diff --git a/scripts/test-tc-959-isolation.sh b/scripts/test-tc-959-isolation.sh index ffeb2b50..1fc513f0 100755 --- a/scripts/test-tc-959-isolation.sh +++ b/scripts/test-tc-959-isolation.sh @@ -101,36 +101,98 @@ docker_cmd() { port_error=$(check_host_ports_free 2>&1) && fail "busy host port passed preflight" [[ "$port_error" == *"127.0.0.1:18000"* && "$port_error" == *"tc959-existing-compose"* ]] || fail "port preflight omitted listener owner: $port_error" -# A real crash sends SIGKILL to the host PID and requires exactly one restart. -VECTOR_ID=vector-test -RESTART_COUNT=4 -KILLED_PID="" +# Container-scoped crash proofs use an exact container ID and never host PIDs. +VECTOR_ID=0123456789abcdef-vector +SNAPSHOT_ID=$VECTOR_ID +SNAPSHOT_RESTARTS=4 +SNAPSHOT_STARTED=2026-10-14T10:00:00.000000000Z +CURRENT_ID=$VECTOR_ID +CURRENT_RESTARTS=5 +CURRENT_STARTED=2026-10-14T10:01:00.000000000Z +CURRENT_RUNNING=true +VECTOR_PIDS=42 +VECTOR_COMM=vector +SIGNALS=0 RECOVERED_LINE_COUNT="" docker_cmd() { - if [[ "$1" != inspect ]]; then return 1; fi - case "$3" in - '{{.RestartCount}}') printf '%s\n' "$RESTART_COUNT" ;; - '{{.State.Pid}}') printf '%s\n' 4242 ;; - '{{.State.Running}}') printf '%s\n' true ;; - *) return 1 ;; - esac -} -kill() { - [[ "$1" == -KILL && "$2" == 4242 ]] || fail "crash helper did not SIGKILL the host container PID" - KILLED_PID=$2 - RESTART_COUNT=5 + if [[ "$1" == inspect ]]; then + case "$3" in + '{{.Id}}|{{.RestartCount}}|{{.State.StartedAt}}') + printf '%s|%s|%s\n' "$SNAPSHOT_ID" "$SNAPSHOT_RESTARTS" "$SNAPSHOT_STARTED" + ;; + '{{.Id}}|{{.RestartCount}}|{{.State.StartedAt}}|{{.State.Running}}') + printf '%s|%s|%s|%s\n' "$CURRENT_ID" "$CURRENT_RESTARTS" "$CURRENT_STARTED" "$CURRENT_RUNNING" + ;; + *) return 1 ;; + esac + return 0 + fi + if [[ "$1" == exec && "$2" == --user && "$3" == 10001:10001 && "$4" == "$VECTOR_ID" ]]; then + case "$5" in + busybox) + case "$6" in + pidof) + [[ "$7" == vector && -n "$VECTOR_PIDS" ]] || return 1 + printf '%s\n' "$VECTOR_PIDS" + ;; + kill) + [[ "$7" == -KILL && "$8" == "${VECTOR_PIDS%% *}" ]] || return 1 + SIGNALS=$((SIGNALS + 1)) + printf '%s\n' SIGNAL_SENT >&2 + ;; + *) return 1 ;; + esac + ;; + cat) + [[ "$6" == "/proc/${VECTOR_PIDS%% *}/comm" ]] || return 1 + printf '%s\n' "$VECTOR_COMM" + ;; + *) return 1 ;; + esac + return 0 + fi + return 1 } wait_vector() { :; } wait_next_line() { RECOVERED_LINE_COUNT=$1; } +sleep() { SECONDS=$((SECONDS + 20)); } +record() { :; } +expect_crash_failure() { + local expected=$1 signal_expected=${2:-no} output + SECONDS=0 + output=$(crash_vector 17 2>&1) && fail "crash helper accepted $expected" + [[ "$output" == *"$expected"* ]] || fail "crash helper failure did not explain $expected: $output" + if [[ "$signal_expected" == no ]]; then + [[ "$output" != *SIGNAL_SENT* ]] || fail "crash helper signalled before rejecting $expected" + else + [[ "$output" == *SIGNAL_SENT* ]] || fail "crash helper did not signal before checking $expected" + fi +} crash_vector 17 -assert_eq "$KILLED_PID" 4242 +assert_eq "$SIGNALS" 1 assert_eq "$RECOVERED_LINE_COUNT" 17 -# A SIGKILL without a restart must fail rather than look like a successful crash. -kill() { :; } -sleep() { SECONDS=$((SECONDS + 20)); } -RESTART_COUNT=5 -SECONDS=0 -restart_error=$(crash_vector 18 2>&1) && fail "crash helper accepted a kill without a restart" -[[ "$restart_error" == *"RestartCount did not increase by exactly one"* ]] || fail "crash helper gave no explicit restart-count failure: $restart_error" -printf '%s\n' 'PASS: port ownership preflight and SIGKILL restart-count checks' +VECTOR_PIDS="" +expect_crash_failure "exactly one Vector PID" +VECTOR_PIDS="42 43" +expect_crash_failure "exactly one Vector PID" +VECTOR_PIDS=1 +expect_crash_failure "PID must be greater than 1" +VECTOR_PIDS=42 +VECTOR_COMM=nginx +expect_crash_failure "comm is not vector" +VECTOR_COMM=vector +SNAPSHOT_ID=another-container-id +expect_crash_failure "container ID changed before crash" +SNAPSHOT_ID=$VECTOR_ID +CURRENT_ID=another-container-id +expect_crash_failure "container ID changed" after-signal +CURRENT_ID=$VECTOR_ID +CURRENT_RESTARTS=4 +expect_crash_failure "RestartCount did not increase by exactly one" after-signal +CURRENT_RESTARTS=6 +expect_crash_failure "RestartCount advanced past expected +1" after-signal +CURRENT_RESTARTS=5 +CURRENT_STARTED=$SNAPSHOT_STARTED +expect_crash_failure "StartedAt did not advance" after-signal +printf '%s\n' 'PASS: exact-container crash identity and restart proofs' diff --git a/tc-959-node-docker-check.review.sh b/tc-959-node-docker-check.review.sh index 8cd09430..0f88f94f 100755 --- a/tc-959-node-docker-check.review.sh +++ b/tc-959-node-docker-check.review.sh @@ -204,6 +204,7 @@ services: - "$CONTROL:/control" ports: !override [] vector: + init: true entrypoint: !override ["/bin/sh", "-ec"] environment: !override {} command: @@ -244,6 +245,9 @@ const baseVectorEnv = baseCompose.services.vector.environment ?? {}; if (!Object.hasOwn(baseVectorEnv, 'VECTOR_CONFIG_YAML') || !Object.hasOwn(baseVectorEnv, 'BETTERSTACK_SOURCE_TOKEN')) { throw new Error('base Vector environment regression fixture is missing'); } +if (baseCompose.services?.vector?.init !== true) { + throw new Error('production Vector service must set init: true for restart supervision'); +} const vectorEnv = compose.services?.vector?.environment; if (!vectorEnv || Object.keys(vectorEnv).length !== 0) throw new Error('acceptance Vector environment must be explicitly empty'); const ingressEnv = compose.services?.['dstack-ingress']?.environment; @@ -261,7 +265,7 @@ const envCheck = spawnSync(process.execPath, [`${root}/scripts/tc-959-compose-en tinycloud: { environment: nodeEnv }, 'dstack-ingress': { environment: ingressEnv }, postgres: { environment: postgresEnv }, - vector: { environment: vectorEnv }, + vector: { init: true, environment: vectorEnv }, } }), encoding: 'utf8', }); @@ -272,6 +276,7 @@ if (!Array.isArray(ingressCommand) || ingressCommand.length !== 3 || throw new Error('dstack-ingress effective Compose argv must be [/bin/sh, -ec, script]'); } const vector = compose.services?.vector; +if (vector?.init !== true) throw new Error('acceptance Vector service must set init: true'); if (JSON.stringify(vector?.entrypoint) !== JSON.stringify(['/bin/sh', '-ec']) || !Array.isArray(vector.command) || vector.command.length !== 1 || typeof vector.command[0] !== 'string') { throw new Error('vector must override the production entrypoint and use a one-script command'); From 64080af791a3c631dfdc6b4c9148a195783c44b7 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 19:35:59 +0000 Subject: [PATCH 13/21] Compare Vector restart timestamps as nanoseconds --- scripts/tc-959-isolation.sh | 34 ++++++++++++++++++++++++++++++-- scripts/test-tc-959-isolation.sh | 30 ++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/scripts/tc-959-isolation.sh b/scripts/tc-959-isolation.sh index e7dc5e7a..88a09034 100644 --- a/scripts/tc-959-isolation.sh +++ b/scripts/tc-959-isolation.sh @@ -10,6 +10,35 @@ check_host_ports_free() { done } +started_at_is_newer() { + node -e ' +const parseNanoseconds = (value) => { + const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:\d{2})$/.exec(value); + if (!match) throw new Error("invalid RFC3339Nano timestamp"); + const [, year, month, day, hour, minute, second, fraction = "", zone] = match; + if (Number(hour) > 23 || Number(minute) > 59 || Number(second) > 59) throw new Error("invalid timestamp time"); + const offsetHour = zone === "Z" ? 0 : Number(zone.slice(1, 3)); + const offsetMinute = zone === "Z" ? 0 : Number(zone.slice(4, 6)); + if (offsetHour > 23 || offsetMinute > 59) throw new Error("invalid timestamp offset"); + const instant = Date.parse(`${year}-${month}-${day}T${hour}:${minute}:${second}${zone}`); + if (!Number.isSafeInteger(instant)) throw new Error("timestamp is out of range"); + const offsetSign = zone === "Z" ? 0 : (zone[0] === "+" ? 1 : -1); + const wallTime = new Date(instant + offsetSign * (offsetHour * 60 + offsetMinute) * 60000).toISOString().slice(0, 19); + if (wallTime !== `${year}-${month}-${day}T${hour}:${minute}:${second}`) throw new Error("invalid timestamp date"); + const nanoseconds = BigInt(instant / 1000) * 1000000000n + BigInt(fraction.padEnd(9, "0") || "0"); + if (nanoseconds === -62135596800000000000n) throw new Error("Docker StartedAt is the zero time"); + return nanoseconds; +}; +try { + const [current, previous] = process.argv.slice(1); + process.exit(parseNanoseconds(current) > parseNanoseconds(previous) ? 0 : 1); +} catch { + process.exit(1); +} +' "$1" "$2" +} + + crash_vector() { local previous_line_count=$1 snapshot snapshot_id restart_before started_before local pids vector_pid comm target_restart current current_id current_restart current_started current_running @@ -48,7 +77,8 @@ crash_vector() { fail "Vector RestartCount advanced past expected +1: before=$restart_before after=$current_restart" fi if (( current_restart == target_restart )) && - [[ "$current_started" > "$started_before" && "$current_running" == true ]]; then + [[ "$current_running" == true ]] && + started_at_is_newer "$current_started" "$started_before"; then break fi sleep 0.2 @@ -57,7 +87,7 @@ crash_vector() { [[ "$current_id" == "$snapshot_id" ]] || fail "Vector container ID changed during crash recovery" [[ "$current_restart" =~ ^[0-9]+$ ]] && (( current_restart == target_restart )) || fail "Vector RestartCount did not increase by exactly one: before=$restart_before after=${current_restart:-unavailable}" - [[ "$current_started" > "$started_before" ]] || fail "Vector StartedAt did not advance after crash" + started_at_is_newer "$current_started" "$started_before" || fail "Vector StartedAt did not advance after crash" [[ "$current_running" == true ]] || fail "Vector was not running after crash recovery" wait_vector [[ "$VECTOR_ID" == "$snapshot_id" ]] || fail "Vector container ID changed while waiting for output" diff --git a/scripts/test-tc-959-isolation.sh b/scripts/test-tc-959-isolation.sh index 1fc513f0..99048750 100755 --- a/scripts/test-tc-959-isolation.sh +++ b/scripts/test-tc-959-isolation.sh @@ -195,4 +195,34 @@ expect_crash_failure "RestartCount advanced past expected +1" after-signal CURRENT_RESTARTS=5 CURRENT_STARTED=$SNAPSHOT_STARTED expect_crash_failure "StartedAt did not advance" after-signal +SNAPSHOT_STARTED=2026-10-14T10:00:00.11Z +CURRENT_STARTED=2026-10-14T10:00:00.1Z +expect_crash_failure "StartedAt did not advance" after-signal + +assert_crash_recovered() { + SECONDS=0 + SIGNALS=0 + RECOVERED_LINE_COUNT="" + crash_vector 17 + assert_eq "$SIGNALS" 1 + assert_eq "$RECOVERED_LINE_COUNT" 17 +} +SNAPSHOT_STARTED=2026-10-14T10:00:00.1Z +CURRENT_STARTED=2026-10-14T10:00:00.11Z +assert_crash_recovered +SNAPSHOT_STARTED=2026-10-14T10:00:00Z +CURRENT_STARTED=2026-10-14T10:00:00Z +expect_crash_failure "StartedAt did not advance" after-signal +SNAPSHOT_STARTED=2026-10-14T10:00:00Z +CURRENT_STARTED=2026-10-14T10:00:00.000000001Z +assert_crash_recovered +SNAPSHOT_STARTED=2026-10-14T10:00:00.123456788Z +CURRENT_STARTED=2026-10-14T10:00:00.123456789Z +assert_crash_recovered +SNAPSHOT_STARTED=2026-10-14T10:00:00Z +CURRENT_STARTED=0001-01-01T00:00:00Z +expect_crash_failure "StartedAt did not advance" after-signal +SNAPSHOT_STARTED=2026-10-14T10:00:00.999999999+01:30 +CURRENT_STARTED=2026-10-14T08:30:01Z +assert_crash_recovered printf '%s\n' 'PASS: exact-container crash identity and restart proofs' From f39a759e8a131267a6e27244406e21264cb372f0 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 20:05:25 +0000 Subject: [PATCH 14/21] Tighten TC-959 acceptance and clear clippy --- README.md | 2 +- docker-compose.dstack-postgres.yaml | 4 ++++ .../tc-959-betterstack-log-shipping.md | 2 +- scripts/tc-959-isolation.sh | 8 ++++++++ scripts/test-tc-959-isolation.sh | 9 +++++++++ tc-959-node-docker-check.review.sh | 18 +++++++++++++++--- tinycloud-node-server/src/tracing.rs | 2 +- 7 files changed, 39 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 762b1a19..c00329ae 100644 --- a/README.md +++ b/README.md @@ -312,7 +312,7 @@ The panic, app ERROR, app WARN, gate-summary, and ingress lanes are capped at 2, On the 2-vCPU/4-GiB/20-GB tdx.medium CVM, Vector is capped at 64 MiB and 0.25 CPU. Its data directory and token file live on an 8 MiB tmpfs; the Vector and ssh-gateway `json-file` logs are capped at `10m × 3` each (60 MB total). The review-only Docker acceptance script measures Vector peak memory and transport loss; neither has been measured by this change. -The production token is GitHub `production` environment secret `PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN`, written to the sealed environment as `BETTERSTACK_SOURCE_TOKEN`. The entrypoint copies it to a mode-0600 `SECRET[]` file on Vector's tmpfs and unsets the variable before executing Vector. Docker daemon/API principals can still inspect container configuration; restrict access. At go-live, create the `node_fatal_panic` alert and a per-service source-silence alert. See [`docs/runbooks/tc-959-betterstack-log-shipping.md`](docs/runbooks/tc-959-betterstack-log-shipping.md) for redaction tests, privilege boundaries, retention, and release checks. +The production token is GitHub `production` environment secret `PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN`, written to the sealed environment as `BETTERSTACK_SOURCE_TOKEN`. The entrypoint copies it to a mode-0600 file on Vector's tmpfs, then unsets the token and executes Vector. Production uses `unset …; exec vector`; `exec env -u` exists only in the acceptance override. The token is not in Vector's process environment. It remains in the container's configured environment (visible to `docker inspect`) and in the init process's initial environment, which only principals that can already read the tmpfs token file can read. Restrict access to both; see [`docs/runbooks/tc-959-betterstack-log-shipping.md`](docs/runbooks/tc-959-betterstack-log-shipping.md) for details. ## Usage diff --git a/docker-compose.dstack-postgres.yaml b/docker-compose.dstack-postgres.yaml index 6a8cb8d9..23e1ef5e 100644 --- a/docker-compose.dstack-postgres.yaml +++ b/docker-compose.dstack-postgres.yaml @@ -679,6 +679,10 @@ services: auth: { strategy: bearer, token: "SECRET[bs.bs_token]" } batch: { max_bytes: 4000000, timeout_secs: 5 } buffer: { type: memory, max_events: 1000, when_full: drop_newest } + # Production uses unset; exec vector, removing the token from Vector's + # process environment only. tini (PID 1, UID 10001) retains the initial + # environment; same-UID readers can already read the tmpfs token file. + # Docker daemon/API principals can inspect the configured environment. entrypoint: - /bin/sh - -ec diff --git a/docs/runbooks/tc-959-betterstack-log-shipping.md b/docs/runbooks/tc-959-betterstack-log-shipping.md index 8bf34cd7..7e9752e9 100644 --- a/docs/runbooks/tc-959-betterstack-log-shipping.md +++ b/docs/runbooks/tc-959-betterstack-log-shipping.md @@ -28,7 +28,7 @@ The CVM is tdx.medium: 2 vCPU, 4 GiB RAM, 20 GB disk. Vector runs as UID 10001 w ## Secrets and local-log exceptions -The ingest host is configured once as `BETTERSTACK_SOURCE_HOST` and substituted into the sink URL. The source token is GitHub `production` environment secret `PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN`, written to the sealed environment as `BETTERSTACK_SOURCE_TOKEN`. The entrypoint writes `SECRET[bs.bs_token]` into `/var/lib/vector/secrets/bs_token` on Vector's tmpfs, with a restrictive umask, then unsets the token and config environment variables before executing Vector. The token is not committed or logged. Docker daemon/API principals can inspect the container's configured environment; restrict daemon/API access. +The ingest host is configured once as `BETTERSTACK_SOURCE_HOST` and substituted into the sink URL. The source token is GitHub `production` environment secret `PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN`, written to the sealed environment as `BETTERSTACK_SOURCE_TOKEN`. The entrypoint writes `SECRET[bs.bs_token]` into `/var/lib/vector/secrets/bs_token` on Vector's tmpfs, with a restrictive umask, then unsets the token and config variables and executes Vector. Production uses `unset …; exec vector`; `exec env -u` is used only in the acceptance override. The token is not in Vector's process environment. It remains in the container's configured environment (visible to `docker inspect`) and in the init process's initial environment, which only principals that can already read the tmpfs token file can read. Restrict access to the tmpfs and Docker daemon/API; the token is not committed or logged. Only node and ingress logs carry `tc.logship`. Vector and ssh-gateway retain capped, unlabelled `json-file` logs and are not shipped, preventing self-ingestion. Postgres is not mounted into Vector. diff --git a/scripts/tc-959-isolation.sh b/scripts/tc-959-isolation.sh index 88a09034..1313495d 100644 --- a/scripts/tc-959-isolation.sh +++ b/scripts/tc-959-isolation.sh @@ -8,6 +8,14 @@ check_host_ports_free() { [[ -z "$containers" ]] || owner="$containers; $listeners" fail "host port 127.0.0.1:$port is already in use: $owner" done + + listeners=$(ss -H -lunp 'sport = :12201') || fail "could not inspect host UDP port 127.0.0.1:12201" + if [[ -n "$listeners" ]]; then + containers=$(docker_cmd ps --filter "publish=12201/udp" --format '{{.Names}}' 2>/dev/null || true) + owner=$listeners + [[ -z "$containers" ]] || owner="$containers; $listeners" + fail "host port 127.0.0.1:12201/udp is already in use: $owner" + fi } started_at_is_newer() { diff --git a/scripts/test-tc-959-isolation.sh b/scripts/test-tc-959-isolation.sh index 99048750..4bf60fca 100755 --- a/scripts/test-tc-959-isolation.sh +++ b/scripts/test-tc-959-isolation.sh @@ -89,6 +89,8 @@ PORT_HOLDER_MODE=container ss() { if [[ "$3" == "sport = :18000" && "$PORT_HOLDER_MODE" == container ]]; then printf '%s\n' 'LISTEN 0 4096 127.0.0.1:18000 0.0.0.0:* users:(("docker-proxy",pid=123,fd=4))' + elif [[ "$3" == "sport = :12201" && "$PORT_HOLDER_MODE" == udp ]]; then + printf '%s\n' 'UNCONN 0 0 0.0.0.0:12201 0.0.0.0:* users:(("docker-proxy",pid=456,fd=7))' fi } docker_cmd() { @@ -96,10 +98,17 @@ docker_cmd() { printf '%s\n' 'tc959-existing-compose' return 0 fi + if [[ "$1" == ps && "$3" == publish=12201/udp ]]; then + printf '%s\n' 'tc959-udp-owner' + return 0 + fi return 1 } port_error=$(check_host_ports_free 2>&1) && fail "busy host port passed preflight" [[ "$port_error" == *"127.0.0.1:18000"* && "$port_error" == *"tc959-existing-compose"* ]] || fail "port preflight omitted listener owner: $port_error" +PORT_HOLDER_MODE=udp +port_error=$(check_host_ports_free 2>&1) && fail "busy host UDP port passed preflight" +[[ "$port_error" == *"127.0.0.1:12201/udp"* && "$port_error" == *"tc959-udp-owner"* ]] || fail "UDP port preflight omitted listener owner: $port_error" # Container-scoped crash proofs use an exact container ID and never host PIDs. VECTOR_ID=0123456789abcdef-vector diff --git a/tc-959-node-docker-check.review.sh b/tc-959-node-docker-check.review.sh index 0f88f94f..f632c1ed 100755 --- a/tc-959-node-docker-check.review.sh +++ b/tc-959-node-docker-check.review.sh @@ -34,6 +34,12 @@ docker_cmd() { record "DRY-RUN docker $*" return 0 fi + if [[ "${1:-}" == timeout ]]; then + local seconds=$2 + shift 2 + timeout "$seconds" docker "$@" + return + fi command docker "$@" } compose_cmd() { @@ -61,12 +67,17 @@ memory_sampler() { } cleanup() { - local status=$? + local status=$? probe_id + local -a probe_ids=() if [[ -n "$SAMPLER_PID" ]]; then rm -f "$SAMPLER_STOP" wait "$SAMPLER_PID" || true fi if [[ "$DRY_RUN" != 1 && "$STARTED" == 1 ]]; then + mapfile -t probe_ids < <(docker_cmd ps -aq --filter "label=com.docker.compose.project=$PROJECT" --filter "label=tc959.probe=true" 2>/dev/null || true) + for probe_id in "${probe_ids[@]}"; do + [[ -n "$probe_id" ]] && docker_cmd rm -f "$probe_id" >/dev/null 2>&1 || true + done compose_cmd down --remove-orphans --volumes >/dev/null 2>&1 || true fi rm -rf "$WORK" @@ -180,7 +191,7 @@ services: extra_hosts: - "host.docker.internal:host-gateway" dstack-ingress: - image: alpine:3.21 + image: alpine:3.21@sha256:ce64758a109eb420d874a118f87920e625e12d3634e03b4a5573fd9f6e5d3507 environment: !override {} command: - /bin/sh @@ -333,6 +344,7 @@ if [[ "$DRY_RUN" == 1 ]]; then fi require docker +require timeout require curl require ip require ss @@ -448,7 +460,7 @@ INGRESS_ID=$(compose_cmd ps -q dstack-ingress) send_from_container() { local network=$1 target=$2 - docker_cmd run --rm --network "$network" --mount "type=bind,src=$SENDER,dst=/sender.mjs,readonly" node:20-alpine node /sender.mjs "$target" 12201 "$payload" + docker_cmd timeout 60 run --rm --label "com.docker.compose.project=$PROJECT" --label "tc959.probe=true" --network "$network" --mount "type=bind,src=$SENDER,dst=/sender.mjs,readonly" node:20-alpine@sha256:fb4cd12c85ee03686f6af5362a0b0d56d50c58a04632e6c0fb8363f609372293 node /sender.mjs "$target" 12201 "$payload" } send_probe() { local sender=$1 network=$2 target=$3 diff --git a/tinycloud-node-server/src/tracing.rs b/tinycloud-node-server/src/tracing.rs index cccf8dcb..0cd3f397 100644 --- a/tinycloud-node-server/src/tracing.rs +++ b/tinycloud-node-server/src/tracing.rs @@ -515,7 +515,7 @@ pub fn tracing_try_init( config::LoggingFormat::Text => subscriber.boxed(), config::LoggingFormat::Json => subscriber .json() - .with_writer(CappedJsonMakeWriter(|| std::io::stderr())) + .with_writer(CappedJsonMakeWriter(std::io::stderr)) .boxed(), }; let telemetry = if config.tracing.enabled { From 43eafdb2a476c7f593ef2e91df31b2d405145d2b Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 22:21:17 +0000 Subject: [PATCH 15/21] fix(TC-959): count probe delivery precisely --- scripts/tc-959-isolation.sh | 22 ++++-- scripts/tc-959-vector-metrics.mjs | 36 +++++++++- scripts/tc-959-vector-metrics.test.mjs | 48 ++++++++++++- scripts/test-tc-959-isolation.sh | 14 +++- tc-959-node-docker-check.review.sh | 98 ++++++++++++++++++-------- 5 files changed, 177 insertions(+), 41 deletions(-) diff --git a/scripts/tc-959-isolation.sh b/scripts/tc-959-isolation.sh index 1313495d..51cef4ef 100644 --- a/scripts/tc-959-isolation.sh +++ b/scripts/tc-959-isolation.sh @@ -181,29 +181,34 @@ fresh_sample() { } assert_positive_delivery() { - local source_before=$1 shipped_before=$2 sent_at=$3 description=$4 - local source_after source_delta deadline=$((SECONDS + 20)) shipped_after shipped_delta + local source_before=$1 shipped_before=$2 sent_at=$3 description=$4 shipped_names_before=$5 + local source_after source_delta deadline=$((SECONDS + 20)) shipped_after shipped_delta shipped_names_after source_after=$(wait_fresh_stable_metric "$sent_at") source_delta=$((source_after - source_before)) (( source_delta == 1 )) || fail "$description positive control source delta was $source_delta, expected +1" while (( SECONDS < deadline )); do shipped_after=$(vector_shipped_count) shipped_delta=$((shipped_after - shipped_before)) + if (( shipped_delta != 0 )); then + shipped_names_after=$(vector_shipped_name_counts) + report_shipped_event_delta "$shipped_names_before" "$shipped_names_after" + fi if (( shipped_delta == 1 )); then - record "PASS: $description positive control; fresh source +1 and shipped +1" + record "PASS: $description positive control; fresh source +1 and shipped probe +1" return 0 fi - (( shipped_delta <= 1 )) || fail "$description produced duplicate shipped events: +$shipped_delta" + (( shipped_delta <= 1 )) || fail "$description produced duplicate probe events: +$shipped_delta" sleep 0.2 done - fail "$description positive control shipped delta was $shipped_delta, expected +1" + fail "$description positive control shipped probe delta was $shipped_delta, expected +1" } check_isolated_probe() { local description=$1 sender=$2 network=$3 target=$4 - local source_before shipped_before sent_at source_after shipped_after source_delta shipped_delta send_status=0 + local source_before shipped_before shipped_names_before sent_at source_after shipped_after shipped_names_after source_delta shipped_delta send_status=0 source_before=$(fresh_sample) shipped_before=$(vector_shipped_count) + shipped_names_before=$(vector_shipped_name_counts) sent_at=$(metric_now_ms) if send_probe "$sender" "$network" "$target"; then record "INFO: $description sender completed" @@ -213,14 +218,17 @@ check_isolated_probe() { fi source_after=$(wait_fresh_stable_metric "$sent_at") shipped_after=$(vector_shipped_count) + shipped_names_after=$(vector_shipped_name_counts) source_delta=$((source_after - source_before)) shipped_delta=$((shipped_after - shipped_before)) - record "INFO: $description fresh deltas source=$source_delta shipped=$shipped_delta" + if (( shipped_delta != 0 )); then report_shipped_event_delta "$shipped_names_before" "$shipped_names_after"; fi + record "INFO: $description fresh deltas source=$source_delta shipped_probe=$shipped_delta" (( send_status == 0 )) || return 2 if (( source_delta == 0 && shipped_delta == 0 )); then return 0; fi return 1 } + assert_isolated_probe() { local description=$1 sender=$2 network=$3 target=$4 status if check_isolated_probe "$description" "$sender" "$network" "$target"; then diff --git a/scripts/tc-959-vector-metrics.mjs b/scripts/tc-959-vector-metrics.mjs index 2451c9b9..a6da0b65 100644 --- a/scripts/tc-959-vector-metrics.mjs +++ b/scripts/tc-959-vector-metrics.mjs @@ -57,6 +57,31 @@ export function countShippedEvents(logs) { export function countShippedEvent(logs, event) { return parsedEvents(logs).filter((record) => record.event === event).length; } +export function countShippedProbeEvents(logs) { + return parsedEvents(logs).filter((record) => record.event === "node_warning" && record.event_code === "invocation_replay_insert_failed").length; +} + +export function shippedEventNameCounts(logs) { + const counts = new Map(); + for (const record of parsedEvents(logs)) { + if (typeof record.event !== "string" || typeof record.event_code !== "string") continue; + if (!/^[a-z][a-z0-9_]*$/.test(record.event) || !/^[a-z][a-z0-9_]*$/.test(record.event_code)) continue; + const key = `${record.event}\t${record.event_code}`; + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return [...counts].map(([key, count]) => { + const [event, event_code] = key.split("\t"); + return { event, event_code, count }; + }); +} + +export function shippedEventNameDelta(before, after) { + const prior = new Map(before.map(({ event, event_code, count }) => [`${event}\t${event_code}`, count])); + return after.flatMap(({ event, event_code, count }) => { + const delta = count - (prior.get(`${event}\t${event_code}`) ?? 0); + return delta > 0 ? [{ event, event_code, count: delta }] : []; + }); +} if (fileURLToPath(import.meta.url) === process.argv[1]) { const [operation, event] = process.argv.slice(2); @@ -72,7 +97,16 @@ if (fileURLToPath(import.meta.url) === process.argv[1]) { process.stdout.write(String(countShippedEvents(input))); } else if (operation === "event-count" && event) { process.stdout.write(String(countShippedEvent(input, event))); + } else if (operation === "probe-count") { + process.stdout.write(String(countShippedProbeEvents(input))); + } else if (operation === "event-name-counts") { + process.stdout.write(JSON.stringify(shippedEventNameCounts(input))); + } else if (operation === "event-name-delta") { + const [before, after] = input.trimEnd().split(/\r?\n/).map((line) => JSON.parse(line)); + for (const { event, event_code, count } of shippedEventNameDelta(before, after)) { + process.stdout.write(`event=${event} event_code=${event_code} count=${count}\n`); + } } else { - throw new Error("usage: tc-959-vector-metrics.mjs received-counter|received-sample|shipped-count|event-count EVENT"); + throw new Error("usage: tc-959-vector-metrics.mjs received-counter|received-sample|shipped-count|event-count EVENT|probe-count|event-name-counts|event-name-delta"); } } diff --git a/scripts/tc-959-vector-metrics.test.mjs b/scripts/tc-959-vector-metrics.test.mjs index d92eccbb..f70cf485 100644 --- a/scripts/tc-959-vector-metrics.test.mjs +++ b/scripts/tc-959-vector-metrics.test.mjs @@ -2,7 +2,7 @@ import { spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; import assert from "node:assert/strict"; import test from "node:test"; -import { parseGelfReceivedSample } from "./tc-959-vector-metrics.mjs"; +import { parseGelfReceivedSample, shippedEventNameCounts, shippedEventNameDelta } from "./tc-959-vector-metrics.mjs"; test("parses the counter value and exporter sample timestamp separately", () => { const sample = parseGelfReceivedSample(`# HELP vector_component_received_events_total Events received\nvector_component_received_events_total{component_id="gelf_in",component_kind="source",component_type="socket",host="vector",mode="udp"} 1 1791992950070\n`); @@ -21,3 +21,49 @@ test("received-sample CLI prints value and timestamp for a captured exporter sam assert.equal(result.status, 0, result.stderr); assert.equal(result.stdout, "1 1791992950070"); }); +test("probe-count CLI counts only the exact probe event and code pair", () => { + const input = [ + JSON.stringify({ event: "node_warning", event_code: "invocation_replay_insert_failed" }), + JSON.stringify({ event: "node_warning", event_code: "different_warning" }), + JSON.stringify({ event: "logship_gate_summary_total", event_code: "logship_gate_summary_total" }), + JSON.stringify({ event: "ingress_request_slow", event_code: "ingress_request_slow" }), + JSON.stringify({ event: "different_event", event_code: "invocation_replay_insert_failed" }), + ].join("\n"); + const result = spawnSync(process.execPath, [fileURLToPath(new URL("./tc-959-vector-metrics.mjs", import.meta.url)), "probe-count"], { + input, + encoding: "utf8", + }); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, "1"); +}); + +test("event-name delta exposes new fixed enums without console content", () => { + const before = shippedEventNameCounts([ + JSON.stringify({ event: "node_warning", event_code: "invocation_replay_insert_failed", message: "before" }), + JSON.stringify({ event: "logship_gate_summary_total", event_code: "logship_gate_summary_total" }), + ].join("\n")); + const after = shippedEventNameCounts([ + JSON.stringify({ event: "node_warning", event_code: "invocation_replay_insert_failed", message: "before" }), + JSON.stringify({ event: "logship_gate_summary_total", event_code: "logship_gate_summary_total" }), + JSON.stringify({ event: "ingress_request_slow", event_code: "ingress_request_slow" }), + JSON.stringify({ event: "node_error", event_code: "safe_fixed_code", message: "secret-like content" }), + ].join("\n")); + assert.deepEqual(shippedEventNameDelta(before, after), [ + { event: "ingress_request_slow", event_code: "ingress_request_slow", count: 1 }, + { event: "node_error", event_code: "safe_fixed_code", count: 1 }, + ]); +}); + +test("event-name-delta CLI reports only newly observed enum pairs", () => { + const before = [{ event: "logship_gate_summary_total", event_code: "logship_gate_summary_total", count: 1 }]; + const after = [ + ...before, + { event: "node_warning", event_code: "invocation_replay_insert_failed", count: 1 }, + ]; + const result = spawnSync(process.execPath, [fileURLToPath(new URL("./tc-959-vector-metrics.mjs", import.meta.url)), "event-name-delta"], { + input: `${JSON.stringify(before)}\n${JSON.stringify(after)}\n`, + encoding: "utf8", + }); + assert.equal(result.status, 0, result.stderr); + assert.equal(result.stdout, "event=node_warning event_code=invocation_replay_insert_failed count=1\n"); +}); diff --git a/scripts/test-tc-959-isolation.sh b/scripts/test-tc-959-isolation.sh index 4bf60fca..df65e021 100755 --- a/scripts/test-tc-959-isolation.sh +++ b/scripts/test-tc-959-isolation.sh @@ -8,7 +8,7 @@ METRICS_SCRAPE_INTERVAL_SECONDS=1 ISOLATION_SETTLE_SECONDS=2 TEST_NOW_MS=3000 METRIC_SAMPLES=() -SHIPPED_SAMPLES=() +SHIPPED_NAME_SAMPLES=() SEND_COUNT=0 read_metric_sample() { @@ -22,6 +22,12 @@ vector_shipped_count() { IFS= read -r sample <&4 || { printf 'no captured shipped count\n' >&2; return 1; } printf '%s\n' "$sample" } +vector_shipped_name_counts() { + local sample + IFS= read -r sample <&6 || { printf 'no captured event names\n' >&2; return 1; } + printf '%s\n' "$sample" +} +report_shipped_event_delta() { :; } compose_cmd() { printf '%s\n' test-vector; } docker_cmd() { [[ "$1" == inspect ]] && printf '%s\n' true; } curl() { return 0; } @@ -43,6 +49,7 @@ reset_fixtures() { SEND_COUNT=0 exec 3< <(printf '%s\n' "${METRIC_SAMPLES[@]}") exec 4< <(printf '%s\n' "${SHIPPED_SAMPLES[@]}") + exec 6< <(printf '%s\n' "${SHIPPED_NAME_SAMPLES[@]}") } wait_vector wait_node @@ -62,14 +69,16 @@ TEST_NOW_MS=6000 ISOLATION_SETTLE_SECONDS=0 METRIC_SAMPLES=('6 5000' '6 6000') SHIPPED_SAMPLES=(5) +SHIPPED_NAME_SAMPLES=('[]') reset_fixtures -assert_positive_delivery 5 4 4000 'captured positive control' +assert_positive_delivery 5 4 4000 'captured positive control' '[]' # A negative probe remains isolated when fresh source and sink counts do not move. reset_fixtures TEST_NOW_MS=7000 METRIC_SAMPLES=('10 8000' '10 9000' '10 10000' '10 11000') SHIPPED_SAMPLES=(5 5) +SHIPPED_NAME_SAMPLES=('[]' '[]') reset_fixtures assert_isolated_probe 'captured isolated probe' container test-network 127.0.0.1 assert_eq "$SEND_COUNT" 1 @@ -79,6 +88,7 @@ reset_fixtures TEST_NOW_MS=12000 METRIC_SAMPLES=('10 13000' '10 14000' '11 15000' '11 16000') SHIPPED_SAMPLES=(5 6) +SHIPPED_NAME_SAMPLES=('[]' '[{"event":"node_warning","event_code":"invocation_replay_insert_failed","count":1}]') reset_fixtures run_isolation_leak_self_test host host 127.0.0.1 assert_eq "$SEND_COUNT" 1 diff --git a/tc-959-node-docker-check.review.sh b/tc-959-node-docker-check.review.sh index f632c1ed..024f59c1 100755 --- a/tc-959-node-docker-check.review.sh +++ b/tc-959-node-docker-check.review.sh @@ -1,11 +1,12 @@ #!/usr/bin/env bash -set -Eeuo pipefail +umask 077 ROOT=$(git rev-parse --show-toplevel) BASE_COMPOSE="$ROOT/docker-compose.dstack-postgres.yaml" VECTOR_BIN=${TC_VECTOR_BIN:-/tmp/tc959-vector-0.59.0/vector-x86_64-unknown-linux-musl/bin/vector} DRY_RUN=${TC_DRY_RUN:-0} WORK=$(mktemp -d "${TMPDIR:-/tmp}/tc959-gelf-check.XXXXXX") +CPU_OVERRIDE="$WORK/compose.cpu-override.yaml" CONTROL="$WORK/control" mkdir -p "$CONTROL" PROJECT="tc959-gelf-$$" @@ -18,6 +19,8 @@ NATIVE_DATA_DIR="$WORK/native-data" SENDER="$WORK/send-gelf.mjs" ENV_FILE="$WORK/acceptance.env" REPORT=${TC_REPORT:-"/tmp/tc959-node-gelf-check-$$.txt"} +touch "$REPORT" +chmod 0600 "$REPORT" COMPOSE=(docker compose --env-file "$ENV_FILE" --project-directory "$ROOT" -p "$PROJECT" -f "$BASE_COMPOSE" -f "$OVERRIDE") VECTOR_ID="" NODE_ID="" @@ -49,6 +52,13 @@ compose_cmd() { fi "${COMPOSE[@]}" "$@" } +compose_half_cpu_cmd() { + if [[ "$DRY_RUN" == 1 ]]; then + record "DRY-RUN compose -f $OVERRIDE -f $CPU_OVERRIDE $*" + return 0 + fi + "${COMPOSE[@]}" -f "$CPU_OVERRIDE" "$@" +} memory_sampler() { local peak=0 container running current while [[ -f "$SAMPLER_STOP" ]]; do @@ -242,15 +252,22 @@ services: max-size: "10m" max-file: "3" EOF -node - "$OVERRIDE" "$ROOT" "$ROOT/tc-959-node-docker-check.review.sh" "$VECTOR_TEST_CONFIG" <<'NODE' +cat >"$CPU_OVERRIDE" <<'EOF' +services: + vector: + cpus: 0.5 +EOF +node - "$OVERRIDE" "$CPU_OVERRIDE" "$ROOT" "$ROOT/tc-959-node-docker-check.review.sh" "$VECTOR_TEST_CONFIG" <<'NODE' const fs = require('node:fs'); const { spawnSync } = require('node:child_process'); const { createRequire } = require('node:module'); -const requireFromRoot = createRequire(`${process.argv[3]}/package.json`); +const requireFromRoot = createRequire(`${process.argv[4]}/package.json`); const { parse } = requireFromRoot('yaml'); -const [overridePath, root, harnessPath, vectorConfigPath] = process.argv.slice(2); +const [overridePath, cpuOverridePath, root, harnessPath, vectorConfigPath] = process.argv.slice(2); const composeText = fs.readFileSync(overridePath, 'utf8'); +const cpuOverride = parse(fs.readFileSync(cpuOverridePath, 'utf8')); const compose = parse(composeText.replaceAll('!override', '')); +if (cpuOverride.services?.vector?.cpus !== 0.5) throw new Error('half-CPU override must set only Vector cpus: 0.5'); const baseCompose = parse(fs.readFileSync(`${root}/docker-compose.dstack-postgres.yaml`, 'utf8')); const baseVectorEnv = baseCompose.services.vector.environment ?? {}; if (!Object.hasOwn(baseVectorEnv, 'VECTOR_CONFIG_YAML') || !Object.hasOwn(baseVectorEnv, 'BETTERSTACK_SOURCE_TOKEN')) { @@ -259,6 +276,7 @@ if (!Object.hasOwn(baseVectorEnv, 'VECTOR_CONFIG_YAML') || !Object.hasOwn(baseVe if (baseCompose.services?.vector?.init !== true) { throw new Error('production Vector service must set init: true for restart supervision'); } +if (baseCompose.services?.vector?.cpus !== 0.25) throw new Error('production Vector CPU setting changed unexpectedly'); const vectorEnv = compose.services?.vector?.environment; if (!vectorEnv || Object.keys(vectorEnv).length !== 0) throw new Error('acceptance Vector environment must be explicitly empty'); const ingressEnv = compose.services?.['dstack-ingress']?.environment; @@ -317,9 +335,10 @@ const declared = new Set(shellSources.flatMap((source) => const calledHelpers = [ 'record', 'docker_cmd', 'compose_cmd', 'memory_sampler', 'cleanup', 'fail', 'require', 'check_host_ports_free', 'crash_vector', - 'wait_vector', 'wait_node', 'read_metric', 'read_metric_sample', 'metric_now_ms', 'wait_fresh_stable_metric', 'fresh_sample', 'wait_metric', 'vector_shipped_count', - 'vector_event_count', 'wait_next_line', 'send_from_container', 'send_probe', + 'vector_shipped_name_counts', 'report_shipped_event_delta', 'vector_event_count', + 'wait_next_line', 'send_from_container', 'send_probe', 'compose_half_cpu_cmd', + 'measure_burst', 'assert_positive_delivery', 'check_isolated_probe', 'assert_isolated_probe', 'run_isolation_leak_self_test', ]; @@ -335,10 +354,10 @@ if [[ "$DRY_RUN" == 1 ]]; then docker_cmd version compose_cmd config record "DRY-RUN: acceptance self-test expects an allowed host-loopback packet to make the negative check fail" - record "DRY-RUN: positive host-loopback and logship-control-container probes require fresh source +1 and shipped +1" - record "DRY-RUN: node/ingress/Postgres sibling and host external-interface probes require two newer stable source samples and shipped +0" - record "DRY-RUN: metric freshness internal_metrics=${METRICS_SCRAPE_INTERVAL_SECONDS}s; isolation settle=${ISOLATION_SETTLE_SECONDS}s" - record "DRY-RUN: sample Vector peak RSS; 5,000-line delivery-loss percentage is informational" + record "DRY-RUN: positive controls require source +1 and exact event=node_warning/event_code=invocation_replay_insert_failed shipped +1" + record "DRY-RUN: negative probes require source +0 and exact probe shipped +0; unrelated event deltas print event/event_code enums" + record "DRY-RUN: 5,000-line bursts measured with fresh source samples at production cpus=0.25 and temporary cpus=0.5" + record "DRY-RUN: half-CPU Vector is recreated only for the second burst and production cpus=0.25 restored before isolation" record "DRY-RUN: actual Docker, network and memory checks NOT run" exit 0 fi @@ -372,12 +391,37 @@ read_metric_sample() { metric_now_ms() { node -p 'Date.now()'; } vector_shipped_count() { [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] || fail "refusing to read events from a stopped Vector container" - docker_cmd logs "$VECTOR_ID" 2>&1 | node "$ROOT/scripts/tc-959-vector-metrics.mjs" shipped-count + docker_cmd logs "$VECTOR_ID" 2>&1 | node "$ROOT/scripts/tc-959-vector-metrics.mjs" probe-count +} +vector_shipped_name_counts() { + [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] || fail "refusing to read events from a stopped Vector container" + docker_cmd logs "$VECTOR_ID" 2>&1 | node "$ROOT/scripts/tc-959-vector-metrics.mjs" event-name-counts +} +report_shipped_event_delta() { + printf '%s\n%s\n' "$1" "$2" | node "$ROOT/scripts/tc-959-vector-metrics.mjs" event-name-delta | + while IFS= read -r event; do record "INFO: newly shipped console $event"; done } vector_event_count() { [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] || fail "refusing to read events from a stopped Vector container" docker_cmd logs "$VECTOR_ID" 2>&1 | node "$ROOT/scripts/tc-959-vector-metrics.mjs" event-count ingress_request_slow } +measure_burst() { + local cpus=$1 before sent_at after received deadline loss + before=$(fresh_sample) + sent_at=$(metric_now_ms) + rm -f "$CONTROL/burst-done" + printf '%s\n' 5000 >"$CONTROL/burst" + deadline=$((SECONDS + 30)) + while [[ ! -f "$CONTROL/burst-done" ]] && (( SECONDS < deadline )); do sleep 0.1; done + [[ -f "$CONTROL/burst-done" ]] || fail "5,000-line producer burst did not complete at cpus=$cpus" + rm -f "$CONTROL/burst-done" + after=$(wait_fresh_stable_metric "$sent_at") + received=$((after - before)) + (( received >= 0 )) || fail "GELF receive counter regressed during cpus=$cpus burst" + loss=$(( (5000 - received) * 100 / 5000 )) + (( loss >= 0 )) || loss=0 + record "INFO: 5,000-line burst cpus=$cpus received=$received loss=${loss}%" +} compose_cmd ps -q tinycloud >/dev/null NODE_ID=$(compose_cmd ps -q tinycloud) @@ -432,23 +476,15 @@ node_restarts_after=$(docker_cmd inspect -f '{{.RestartCount}}' "$NODE_ID") [[ "$node_restarts" == "$node_restarts_after" ]] || fail "node restarted during Vector recovery matrix" record "PASS: app restart count unchanged across all Vector recovery tests" -before_metric=$(wait_metric 0) -printf '%s\n' 5000 >"$CONTROL/burst" -deadline=$((SECONDS + 30)) -while [[ ! -f "$CONTROL/burst-done" ]] && (( SECONDS < deadline )); do sleep 0.1; done -[[ -f "$CONTROL/burst-done" ]] || fail "5,000-line producer burst did not complete" -rm -f "$CONTROL/burst-done" -deadline=$((SECONDS + 20)) -after_metric=$before_metric -while (( SECONDS < deadline )); do - after_metric=$(read_metric) - [[ "$after_metric" =~ ^[0-9]+$ ]] || after_metric=$before_metric - (( after_metric >= before_metric + 5000 )) && break - sleep 0.5 -done -loss=100 -if (( after_metric >= before_metric )); then loss=$(( (5000 - (after_metric - before_metric)) * 100 / 5000 )); (( loss < 0 )) && loss=0; fi -record "INFO: 5,000-line receive count delta=$((after_metric - before_metric)); estimated transport loss=${loss}% (informational)" +measure_burst 0.25 +compose_half_cpu_cmd up -d --force-recreate --no-deps vector >/dev/null +wait_vector +wait_next_line 0 +measure_burst 0.5 +compose_cmd up -d --force-recreate --no-deps vector >/dev/null +wait_vector +wait_next_line 0 +record "INFO: restored production Vector cpus=0.25 before isolation, node-latency, and memory checks" # Keep the idle synthetic ingress up so its network namespace is available for sibling probes. sleep 1 @@ -474,15 +510,17 @@ send_probe() { # Positive controls precede the leak self-test and every forbidden-source assertion. source_before=$(fresh_sample) shipped_before=$(vector_shipped_count) +shipped_names_before=$(vector_shipped_name_counts) sent_at=$(metric_now_ms) node "$SENDER" 127.0.0.1 12201 "$payload" -assert_positive_delivery "$source_before" "$shipped_before" "$sent_at" "host loopback" +assert_positive_delivery "$source_before" "$shipped_before" "$sent_at" "host loopback" "$shipped_names_before" source_before=$(fresh_sample) shipped_before=$(vector_shipped_count) +shipped_names_before=$(vector_shipped_name_counts) sent_at=$(metric_now_ms) send_from_container "${PROJECT}_logship" "$vector_ip" -assert_positive_delivery "$source_before" "$shipped_before" "$sent_at" "logship-network control container" +assert_positive_delivery "$source_before" "$shipped_before" "$sent_at" "logship-network control container" "$shipped_names_before" run_isolation_leak_self_test host host 127.0.0.1 From 708355f400f518811f7d9c45d4855be3bb852cf8 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 22:28:07 +0000 Subject: [PATCH 16/21] test(TC-959): scan raw console for probe canary --- scripts/tc-959-isolation.sh | 6 ++++++ scripts/test-tc-959-isolation.sh | 18 +++++++++++++++++- tc-959-node-docker-check.review.sh | 12 ++++++------ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/scripts/tc-959-isolation.sh b/scripts/tc-959-isolation.sh index 51cef4ef..b8a9f5cb 100644 --- a/scripts/tc-959-isolation.sh +++ b/scripts/tc-959-isolation.sh @@ -229,6 +229,12 @@ check_isolated_probe() { } +assert_no_console_marker() { + local marker=$1 console_output match_count + console_output=$(docker_cmd logs "$VECTOR_ID" 2>&1) || return 2 + match_count=$(printf '%s\n' "$console_output" | grep -cF -- "$marker" || true) + [[ "$match_count" == 0 ]] +} assert_isolated_probe() { local description=$1 sender=$2 network=$3 target=$4 status if check_isolated_probe "$description" "$sender" "$network" "$target"; then diff --git a/scripts/test-tc-959-isolation.sh b/scripts/test-tc-959-isolation.sh index df65e021..46ceb076 100755 --- a/scripts/test-tc-959-isolation.sh +++ b/scripts/test-tc-959-isolation.sh @@ -29,7 +29,10 @@ vector_shipped_name_counts() { } report_shipped_event_delta() { :; } compose_cmd() { printf '%s\n' test-vector; } -docker_cmd() { [[ "$1" == inspect ]] && printf '%s\n' true; } +docker_cmd() { + if [[ "$1" == logs ]]; then printf '%s\n' "$RAW_CONSOLE_OUTPUT"; return 0; fi + [[ "$1" == inspect ]] && printf '%s\n' true +} curl() { return 0; } read_metric() { printf '%s\n' 3; } vector_event_count() { @@ -94,6 +97,19 @@ run_isolation_leak_self_test host host 127.0.0.1 assert_eq "$SEND_COUNT" 1 printf '%s\n' 'PASS: isolation freshness, positive controls, and leak self-test' +# The raw console scan catches a marker hidden in an unexpected field even when the probe filter returns zero. +VECTOR_ID=test-vector +RAW_CONSOLE_OUTPUT='{"event":"node_warning","event_code":"other_warning","unexpected_field":"TC959_FORGED_PROBE_CANARY_UNEXPECTED_FIELD"}' +filtered_probe_count=$(printf '%s\n' "$RAW_CONSOLE_OUTPUT" | node "$ROOT/scripts/tc-959-vector-metrics.mjs" probe-count) +assert_eq "$filtered_probe_count" 0 +if assert_no_console_marker TC959_FORGED_PROBE_CANARY_UNEXPECTED_FIELD; then + fail "raw console scan missed a marker in an unexpected field" +else + assert_eq "$?" 1 +fi +RAW_CONSOLE_OUTPUT='{"event":"node_warning","event_code":"other_warning"}' +assert_no_console_marker TC959_FORGED_PROBE_CANARY_UNEXPECTED_FIELD || fail "raw console scan falsely matched absent marker" + # Host-port preflight reports the owning Compose service before startup. PORT_HOLDER_MODE=container ss() { diff --git a/tc-959-node-docker-check.review.sh b/tc-959-node-docker-check.review.sh index 024f59c1..a8fe5fd2 100755 --- a/tc-959-node-docker-check.review.sh +++ b/tc-959-node-docker-check.review.sh @@ -337,10 +337,8 @@ const calledHelpers = [ 'check_host_ports_free', 'crash_vector', 'wait_fresh_stable_metric', 'fresh_sample', 'wait_metric', 'vector_shipped_count', 'vector_shipped_name_counts', 'report_shipped_event_delta', 'vector_event_count', - 'wait_next_line', 'send_from_container', 'send_probe', 'compose_half_cpu_cmd', - 'measure_burst', - 'assert_positive_delivery', 'check_isolated_probe', 'assert_isolated_probe', - 'run_isolation_leak_self_test', + 'assert_no_console_marker', 'wait_next_line', 'send_from_container', 'send_probe', + 'compose_half_cpu_cmd', 'measure_burst', 'run_isolation_leak_self_test', ]; const missing = calledHelpers.filter((name) => !declared.has(name)); if (missing.length) throw new Error(`harness calls undefined shell functions: ${missing.join(', ')}`); @@ -487,8 +485,8 @@ wait_next_line 0 record "INFO: restored production Vector cpus=0.25 before isolation, node-latency, and memory checks" # Keep the idle synthetic ingress up so its network namespace is available for sibling probes. -sleep 1 -payload='{"version":"1.1","host":"tc959-isolation-probe","short_message":"{\"timestamp\":\"2026-10-14T10:00:00.000Z\",\"level\":\"WARN\",\"fields\":{\"message\":\"invocation replay insert failed\"}}","timestamp":1760436000,"level":4,"_tc.logship":"tinycloud-node"}' +PROBE_CANARY=TC959_FORGED_PROBE_CANARY_UNEXPECTED_FIELD +payload='{"version":"1.1","host":"tc959-isolation-probe","short_message":"{\"timestamp\":\"2026-10-14T10:00:00.000Z\",\"level\":\"WARN\",\"fields\":{\"message\":\"invocation replay insert failed\"}}","timestamp":1760436000,"level":4,"_forged_probe_canary":"'"$PROBE_CANARY"'","_tc.logship":"tinycloud-node"}' vector_ip=10.255.253.2 app_gateway=$(docker_cmd inspect -f '{{range .NetworkSettings.Networks}}{{.Gateway}}{{end}}' "$NODE_ID") INGRESS_ID=$(compose_cmd ps -q dstack-ingress) @@ -538,6 +536,8 @@ while read -r host_ip; do [[ -n "$host_ip" ]] || continue assert_isolated_probe "host external interface $host_ip" host host "$host_ip" done <<<"$host_ips" +assert_no_console_marker "$PROBE_CANARY" || fail "forged probe canary appeared anywhere in Vector console output" +record "PASS: raw Vector console output contains no forged-probe canary after the isolation matrix" compose_cmd stop vector >/dev/null max_ms=0 From e88aefefd52ca90811e88a53a45ea76f955fb959 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 22:37:41 +0000 Subject: [PATCH 17/21] fix(TC-959): fail closed on log reader errors --- scripts/tc-959-isolation.sh | 27 ++++++++++++++++++++++++--- scripts/test-tc-959-isolation.sh | 17 ++++++++++++++++- tc-959-node-docker-check.review.sh | 23 +++++++++-------------- 3 files changed, 49 insertions(+), 18 deletions(-) diff --git a/scripts/tc-959-isolation.sh b/scripts/tc-959-isolation.sh index b8a9f5cb..e1f466c9 100644 --- a/scripts/tc-959-isolation.sh +++ b/scripts/tc-959-isolation.sh @@ -229,10 +229,31 @@ check_isolated_probe() { } +vector_shipped_count() { + [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] || fail "refusing to read events from a stopped Vector container" + local logs + logs=$(docker_cmd logs "$VECTOR_ID" 2>&1) || fail "docker logs failed" + printf '%s\n' "$logs" | node "$ROOT/scripts/tc-959-vector-metrics.mjs" probe-count +} + +vector_shipped_name_counts() { + [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] || fail "refusing to read events from a stopped Vector container" + local logs + logs=$(docker_cmd logs "$VECTOR_ID" 2>&1) || fail "docker logs failed" + printf '%s\n' "$logs" | node "$ROOT/scripts/tc-959-vector-metrics.mjs" event-name-counts +} + +vector_event_count() { + [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] || fail "refusing to read events from a stopped Vector container" + local logs + logs=$(docker_cmd logs "$VECTOR_ID" 2>&1) || fail "docker logs failed" + printf '%s\n' "$logs" | node "$ROOT/scripts/tc-959-vector-metrics.mjs" event-count ingress_request_slow +} + assert_no_console_marker() { - local marker=$1 console_output match_count - console_output=$(docker_cmd logs "$VECTOR_ID" 2>&1) || return 2 - match_count=$(printf '%s\n' "$console_output" | grep -cF -- "$marker" || true) + local marker=$1 logs match_count + logs=$(docker_cmd logs "$VECTOR_ID" 2>&1) || fail "docker logs failed" + match_count=$(printf '%s\n' "$logs" | grep -cF -- "$marker" || true) [[ "$match_count" == 0 ]] } assert_isolated_probe() { diff --git a/scripts/test-tc-959-isolation.sh b/scripts/test-tc-959-isolation.sh index 46ceb076..240421e7 100755 --- a/scripts/test-tc-959-isolation.sh +++ b/scripts/test-tc-959-isolation.sh @@ -29,8 +29,13 @@ vector_shipped_name_counts() { } report_shipped_event_delta() { :; } compose_cmd() { printf '%s\n' test-vector; } +LOGS_SHOULD_FAIL=0 docker_cmd() { - if [[ "$1" == logs ]]; then printf '%s\n' "$RAW_CONSOLE_OUTPUT"; return 0; fi + if [[ "$1" == logs ]]; then + if (( LOGS_SHOULD_FAIL )); then printf '%s\n' 'stub docker logs failure' >&2; return 1; fi + printf '%s\n' "$RAW_CONSOLE_OUTPUT" + return 0 + fi [[ "$1" == inspect ]] && printf '%s\n' true } curl() { return 0; } @@ -109,6 +114,16 @@ else fi RAW_CONSOLE_OUTPUT='{"event":"node_warning","event_code":"other_warning"}' assert_no_console_marker TC959_FORGED_PROBE_CANARY_UNEXPECTED_FIELD || fail "raw console scan falsely matched absent marker" +unset -f vector_shipped_count vector_shipped_name_counts vector_event_count assert_no_console_marker +source "$ROOT/scripts/tc-959-isolation.sh" +LOGS_SHOULD_FAIL=1 +for reader in vector_shipped_count vector_shipped_name_counts vector_event_count assert_no_console_marker; do + if output=$("$reader" TC959_FORGED_PROBE_CANARY_UNEXPECTED_FIELD 2>&1); then + fail "$reader passed after docker logs failed" + fi + [[ "$output" == *"docker logs failed"* ]] || fail "$reader did not fail closed: $output" +done +LOGS_SHOULD_FAIL=0 # Host-port preflight reports the owning Compose service before startup. PORT_HOLDER_MODE=container diff --git a/tc-959-node-docker-check.review.sh b/tc-959-node-docker-check.review.sh index a8fe5fd2..bce310a6 100755 --- a/tc-959-node-docker-check.review.sh +++ b/tc-959-node-docker-check.review.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash umask 077 +set -Eeuo pipefail ROOT=$(git rev-parse --show-toplevel) BASE_COMPOSE="$ROOT/docker-compose.dstack-postgres.yaml" @@ -329,7 +330,11 @@ for (const fragment of ['count=$$(cat /control/burst)', 'while [ "$$i" -lt "$$co if (!ingress.includes(fragment)) throw new Error(`rendered producer command lost Compose escaping: ${fragment}`); } const libraryPath = `${root}/scripts/tc-959-isolation.sh`; -const shellSources = [fs.readFileSync(harnessPath, 'utf8'), fs.readFileSync(libraryPath, 'utf8')]; +const harnessSource = fs.readFileSync(harnessPath, 'utf8'); +if (!harnessSource.split(/\r?\n/).slice(1, 3).includes('set -Eeuo pipefail')) { + throw new Error('harness must keep set -Eeuo pipefail on line 2 or 3'); +} +const shellSources = [harnessSource, fs.readFileSync(libraryPath, 'utf8')]; const declared = new Set(shellSources.flatMap((source) => [...source.matchAll(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*\(\)\s*\{/gm)].map((match) => match[1]))); const calledHelpers = [ @@ -387,22 +392,10 @@ read_metric_sample() { curl --fail --silent http://127.0.0.1:9598/metrics | node "$ROOT/scripts/tc-959-vector-metrics.mjs" received-sample } metric_now_ms() { node -p 'Date.now()'; } -vector_shipped_count() { - [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] || fail "refusing to read events from a stopped Vector container" - docker_cmd logs "$VECTOR_ID" 2>&1 | node "$ROOT/scripts/tc-959-vector-metrics.mjs" probe-count -} -vector_shipped_name_counts() { - [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] || fail "refusing to read events from a stopped Vector container" - docker_cmd logs "$VECTOR_ID" 2>&1 | node "$ROOT/scripts/tc-959-vector-metrics.mjs" event-name-counts -} report_shipped_event_delta() { printf '%s\n%s\n' "$1" "$2" | node "$ROOT/scripts/tc-959-vector-metrics.mjs" event-name-delta | while IFS= read -r event; do record "INFO: newly shipped console $event"; done } -vector_event_count() { - [[ "$(docker_cmd inspect -f '{{.State.Running}}' "$VECTOR_ID")" == true ]] || fail "refusing to read events from a stopped Vector container" - docker_cmd logs "$VECTOR_ID" 2>&1 | node "$ROOT/scripts/tc-959-vector-metrics.mjs" event-count ingress_request_slow -} measure_burst() { local cpus=$1 before sent_at after received deadline loss before=$(fresh_sample) @@ -482,7 +475,9 @@ measure_burst 0.5 compose_cmd up -d --force-recreate --no-deps vector >/dev/null wait_vector wait_next_line 0 -record "INFO: restored production Vector cpus=0.25 before isolation, node-latency, and memory checks" +restored_nano_cpus=$(docker_cmd inspect -f '{{.HostConfig.NanoCpus}}' "$VECTOR_ID") +[[ "$restored_nano_cpus" == 250000000 ]] || fail "Vector CPU limit after restore is $restored_nano_cpus, expected 250000000" +record "PASS: restored production Vector NanoCpus=250000000 before isolation, node-latency, and memory checks" # Keep the idle synthetic ingress up so its network namespace is available for sibling probes. PROBE_CANARY=TC959_FORGED_PROBE_CANARY_UNEXPECTED_FIELD From da999ed753b841b606d8b1defda3c1d7e4b3cfc7 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Wed, 14 Oct 2026 23:14:57 +0000 Subject: [PATCH 18/21] fix(TC-959): harden GELF isolation probes --- CHANGELOG.md | 2 +- README.md | 2 +- docker-compose.dstack-postgres.yaml | 2 +- .../tc-959-betterstack-log-shipping.md | 2 +- scripts/tc-959-isolation.sh | 90 +++++++++++-- scripts/test-tc-959-isolation.sh | 122 +++++++++++++++++- tc-959-node-docker-check.review.sh | 27 ++-- 7 files changed, 218 insertions(+), 29 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea1ce6e7..52fe0c0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog ## [Unreleased] -- TC-959: ship only labelled node/ingress logs through GELF/UDP to a fixed-address, isolated Vector socket; cap complete node JSON lines at 15,000 bytes including newline, with the panic hook on the same path; gate labels before metadata stripping, JSON parsing, and the existing allowlist. Gate reasons are summarized without content on an independent 1/min lane; ingress is capped at 26/min to preserve the 39/min, 0.473 GB/30d budget while app WARN remains 5/min. Vector publishes UDP on host loopback only, has no Docker-directory mount or socket, and runs with capped resources. Unlabelled services remain on capped `json-file`. +- TC-959: ship only labelled node/ingress logs through GELF/UDP to a fixed-address, isolated Vector socket; cap complete node JSON lines at 15,000 bytes including newline, with the panic hook on the same path; gate labels before metadata stripping, JSON parsing, and the existing allowlist. Gate reasons are summarized without content on an independent 1/min lane; ingress is capped at 26/min to preserve the 39/min, 0.473 GB/30d budget while app WARN remains 5/min. Vector publishes UDP on host loopback only, has no Docker-directory mount or socket, and is capped at 64 MiB and 0.5 CPU on the 2-vCPU CVM. Unlabelled services remain on capped `json-file`. - Security (TC-1132, TC-1080 Parts C–D): webhook registration and every delivery attempt share an egress policy covering special-use IPv4/IPv6, embedded IPv4 (mapped, compatible, NAT64 and 6to4), Teredo and site-local IPv6. Ambiguous local-use NAT64 is refused if any valid IPv4 interpretation is forbidden. Corrected public controls such as `192.2.1.1` and `198.51.0.1` are no longer over-blocked. Mixed DNS answers retain only permitted addresses. - Webhooks deny the resolved addresses of the configured network database, quota URL, space allowlist and explicit S3 endpoint on every port, including aliases and embedded IPv4 forms. The service deny-set refreshes every ten minutes; validation waits for an in-progress refresh, and failed/stale resolution closes egress until recovery. Temporary DNS/policy failures retry on the existing bounded schedule; invalid or forbidden callbacks dead-letter. Existing subscriptions whose destinations become forbidden dead-letter on the next attempt. diff --git a/README.md b/README.md index c00329ae..3dcf6065 100644 --- a/README.md +++ b/README.md @@ -310,7 +310,7 @@ The node and ingress use the Docker GELF driver with exact `tc.logship` labels a The panic, app ERROR, app WARN, gate-summary, and ingress lanes are capped at 2, 5, 5, 1, and 26 events/minute. The one-per-minute gate summary carries four reason counts and no content; it cannot consume a real WARN slot. With the 281-byte final event limit, the monthly ceiling is `(2 + 5 + 5 + 1 + 26) × 60 × 24 × 30 × 281 = 473,428,800 bytes` (0.473 GB/30d), below the node's 0.48 GB share of the 3 GB free quota. Unknown routes collapse to ``. -On the 2-vCPU/4-GiB/20-GB tdx.medium CVM, Vector is capped at 64 MiB and 0.25 CPU. Its data directory and token file live on an 8 MiB tmpfs; the Vector and ssh-gateway `json-file` logs are capped at `10m × 3` each (60 MB total). The review-only Docker acceptance script measures Vector peak memory and transport loss; neither has been measured by this change. +On the 2-vCPU/4-GiB/20-GB tdx.medium CVM, Vector is capped at 64 MiB and ≤0.5 CPU (25% of the CVM's 2 vCPU). Its data directory and token file live on an 8 MiB tmpfs; the Vector and ssh-gateway `json-file` logs are capped at `10m × 3` each (60 MB total). The review-only Docker acceptance script measures Vector peak memory and transport loss; neither has been measured by this change. The production token is GitHub `production` environment secret `PROD_TINYCLOUD_BETTERSTACK_SOURCE_TOKEN`, written to the sealed environment as `BETTERSTACK_SOURCE_TOKEN`. The entrypoint copies it to a mode-0600 file on Vector's tmpfs, then unsets the token and executes Vector. Production uses `unset …; exec vector`; `exec env -u` exists only in the acceptance override. The token is not in Vector's process environment. It remains in the container's configured environment (visible to `docker inspect`) and in the init process's initial environment, which only principals that can already read the tmpfs token file can read. Restrict access to both; see [`docs/runbooks/tc-959-betterstack-log-shipping.md`](docs/runbooks/tc-959-betterstack-log-shipping.md) for details. diff --git a/docker-compose.dstack-postgres.yaml b/docker-compose.dstack-postgres.yaml index 23e1ef5e..33209024 100644 --- a/docker-compose.dstack-postgres.yaml +++ b/docker-compose.dstack-postgres.yaml @@ -145,7 +145,7 @@ services: security_opt: ["no-new-privileges:true"] mem_limit: 64m memswap_limit: 64m - cpus: 0.25 + cpus: 0.5 ports: - "127.0.0.1:12201:12201/udp" networks: diff --git a/docs/runbooks/tc-959-betterstack-log-shipping.md b/docs/runbooks/tc-959-betterstack-log-shipping.md index 7e9752e9..dff1a858 100644 --- a/docs/runbooks/tc-959-betterstack-log-shipping.md +++ b/docs/runbooks/tc-959-betterstack-log-shipping.md @@ -24,7 +24,7 @@ At the 281-byte final event-size limit, the worst case is `(2 + 5 + 5 + 1 + 26) ## CVM resource and disk budget -The CVM is tdx.medium: 2 vCPU, 4 GiB RAM, 20 GB disk. Vector runs as UID 10001 with 64 MiB memory/swap and 0.25 CPU limits, read-only root, all capabilities dropped, and `no-new-privileges`. Its `data_dir` is an 8 MiB tmpfs; the token file is mode 0600 under that same tmpfs. There is no volume initializer, Docker log-tree mount, or persistent checkpoint volume. Vector and ssh-gateway each have capped `json-file` logs (`10m × 3`), a 60 MB combined retention ceiling. Node and ingress logs travel through GELF rather than a local log file. The review-only acceptance harness measures Vector peak memory and reports 5,000-line burst loss; neither value has been measured by this change. +The CVM is tdx.medium: 2 vCPU, 4 GiB RAM, 20 GB disk. Vector runs as UID 10001 with 64 MiB memory/swap and a CPU limit of ≤0.5 CPU (25% of the 2-vCPU CVM), read-only root, all capabilities dropped, and `no-new-privileges`. Its `data_dir` is an 8 MiB tmpfs; the token file is mode 0600 under that same tmpfs. There is no volume initializer, Docker log-tree mount, or persistent checkpoint volume. Vector and ssh-gateway each have capped `json-file` logs (`10m × 3`), a 60 MB combined retention ceiling. Node and ingress logs travel through GELF rather than a local log file. The review-only acceptance harness measures Vector peak memory and reports 5,000-line burst loss; neither value has been measured by this change. ## Secrets and local-log exceptions diff --git a/scripts/tc-959-isolation.sh b/scripts/tc-959-isolation.sh index e1f466c9..8616a701 100644 --- a/scripts/tc-959-isolation.sh +++ b/scripts/tc-959-isolation.sh @@ -203,19 +203,81 @@ assert_positive_delivery() { fail "$description positive control shipped probe delta was $shipped_delta, expected +1" } +run_probe_sender() { + local sender=$1 network=$2 target=$3 + if send_probe "$sender" "$network" "$target" >"$CONTROL/probe-sender.out" 2>&1; then + LAST_PROBE_SEND_STATUS=0 + else + LAST_PROBE_SEND_STATUS=$? + fi + LAST_PROBE_SEND_OUTPUT=$(<"$CONTROL/probe-sender.out") + (( LAST_PROBE_SEND_STATUS == 0 )) +} + +send_positive_probe() { + local description=$1 sender=$2 network=$3 target=$4 error_code + if run_probe_sender "$sender" "$network" "$target"; then + return 0 + fi + if error_code=$(sender_error_code "$LAST_PROBE_SEND_STATUS" "$LAST_PROBE_SEND_OUTPUT"); then + record "INFO: $description sender error code=$error_code exit=$LAST_PROBE_SEND_STATUS" + fi + fail "$description positive control sender failed (exit=$LAST_PROBE_SEND_STATUS)" +} + +sender_error_code() { + local exit_status=$1 output=$2 + (( exit_status == 1 )) || return 1 + [[ "$output" =~ ^GELF_SEND_ERROR\ code=(ENOTFOUND|EAI_AGAIN|ENETUNREACH|EHOSTUNREACH|ECONNREFUSED|other)$ ]] || return 1 + printf '%s\n' "${BASH_REMATCH[1]}" +} + +probe_routing_error_matches_target() { + local code=$1 target=$2 + case "$code" in + ENOTFOUND) [[ "$target" == vector || "$target" == host.docker.internal ]] ;; + ENETUNREACH|EHOSTUNREACH) [[ "$target" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] ;; + *) return 1 ;; + esac +} + check_isolated_probe() { local description=$1 sender=$2 network=$3 target=$4 - local source_before shipped_before shipped_names_before sent_at source_after shipped_after shipped_names_after source_delta shipped_delta send_status=0 + local sender_errors=${5:-allow-routing-errors} + local source_before shipped_before shipped_names_before sent_at source_after shipped_after shipped_names_after + local source_delta shipped_delta send_status=0 sender_code="" routing_code="" eai_attempts=0 source_before=$(fresh_sample) shipped_before=$(vector_shipped_count) shipped_names_before=$(vector_shipped_name_counts) sent_at=$(metric_now_ms) - if send_probe "$sender" "$network" "$target"; then - record "INFO: $description sender completed" - else - send_status=$? - record "INFO: $description sender reported transport error exit=$send_status" - fi + while true; do + if run_probe_sender "$sender" "$network" "$target"; then + record "INFO: $description sender completed" + break + fi + send_status=$LAST_PROBE_SEND_STATUS + if ! sender_code=$(sender_error_code "$send_status" "$LAST_PROBE_SEND_OUTPUT"); then + record "INFO: $description sender error is unclassified (exit=$send_status)" + return 2 + fi + record "INFO: $description sender error code=$sender_code exit=$send_status" + if [[ "$sender_code" == EAI_AGAIN ]]; then + eai_attempts=$((eai_attempts + 1)) + if (( eai_attempts >= 3 )); then + record "INFO: $description EAI_AGAIN remained after 3 attempts" + return 2 + fi + record "INFO: $description retrying EAI_AGAIN attempt $((eai_attempts + 1))/3" + continue + fi + if [[ "$sender_errors" == allow-routing-errors ]] && probe_routing_error_matches_target "$sender_code" "$target"; then + routing_code=$sender_code + record "INFO: $description not routable ($routing_code)" + break + fi + record "INFO: $description sender error code=$sender_code is not acceptable for this target/context" + return 2 + done source_after=$(wait_fresh_stable_metric "$sent_at") shipped_after=$(vector_shipped_count) shipped_names_after=$(vector_shipped_name_counts) @@ -223,8 +285,13 @@ check_isolated_probe() { shipped_delta=$((shipped_after - shipped_before)) if (( shipped_delta != 0 )); then report_shipped_event_delta "$shipped_names_before" "$shipped_names_after"; fi record "INFO: $description fresh deltas source=$source_delta shipped_probe=$shipped_delta" - (( send_status == 0 )) || return 2 - if (( source_delta == 0 && shipped_delta == 0 )); then return 0; fi + if (( source_delta == 0 && shipped_delta == 0 )); then + if [[ -n "$routing_code" ]]; then + record "PASS: $description isolated: not routable ($routing_code); fresh source +0 and shipped +0" + return 3 + fi + return 0 + fi return 1 } @@ -263,13 +330,14 @@ assert_isolated_probe() { return 0 else status=$? - fail "$description isolation check failed (status=$status)" fi + if (( status == 3 )); then return 0; fi + fail "$description isolation check failed (status=$status)" } run_isolation_leak_self_test() { local sender=$1 network=$2 target=$3 status - if check_isolated_probe "intentional host-loopback leak self-test" "$sender" "$network" "$target"; then + if check_isolated_probe "intentional host-loopback leak self-test" "$sender" "$network" "$target" must-deliver; then fail "isolation leak self-test passed unexpectedly" else status=$? diff --git a/scripts/test-tc-959-isolation.sh b/scripts/test-tc-959-isolation.sh index 240421e7..bc89f62b 100755 --- a/scripts/test-tc-959-isolation.sh +++ b/scripts/test-tc-959-isolation.sh @@ -45,14 +45,40 @@ vector_event_count() { IFS= read -r sample <&5 || return 1 printf '%s\n' "$sample" } +SENDER_EXIT=0 +SENDER_OUTPUT="" +SEND_COUNT_FILE="" +RECORD_LINES=() sleep() { :; } CONTROL=$(mktemp -d) trap 'rm -rf "$CONTROL"' EXIT -send_probe() { SEND_COUNT=$((SEND_COUNT + 1)); } -record() { :; } +send_probe() { + SEND_COUNT=$((SEND_COUNT + 1)) + [[ -n "$SEND_COUNT_FILE" ]] && printf x >>"$SEND_COUNT_FILE" + printf '%s' "$SENDER_OUTPUT" + return "$SENDER_EXIT" +} +record() { RECORD_LINES+=("$*"); } fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } assert_eq() { [[ "$1" == "$2" ]] || fail "expected '$2', got '$1'"; } +expect_probe_failure() { + local expected=$1 target=$2 status=$3 output_text=$4 actual + SENDER_EXIT=$status + SENDER_OUTPUT=$output_text + if actual=$(assert_isolated_probe "stub failure" container test-network "$target" 2>&1); then + fail "probe failure was accepted: $expected" + fi + [[ "$actual" == *"$expected"* ]] || fail "probe failure did not explain $expected: $actual" +} + +prepare_sender_error_probe() { + TEST_NOW_MS=50000 + METRIC_SAMPLES=('10 51000' '10 52000') + SHIPPED_SAMPLES=(5) + SHIPPED_NAME_SAMPLES=('[]') + reset_fixtures +} reset_fixtures() { SEND_COUNT=0 exec 3< <(printf '%s\n' "${METRIC_SAMPLES[@]}") @@ -100,6 +126,98 @@ SHIPPED_NAME_SAMPLES=('[]' '[{"event":"node_warning","event_code":"invocation_re reset_fixtures run_isolation_leak_self_test host host 127.0.0.1 assert_eq "$SEND_COUNT" 1 +# Not-routable sender errors pass only after fresh +0/+0 samples for a matching target type. +TEST_NOW_MS=20000 +METRIC_SAMPLES=('10 21000' '10 22000' '10 23000' '10 24000') +SHIPPED_SAMPLES=(5 5) +SHIPPED_NAME_SAMPLES=('[]' '[]') +SENDER_EXIT=1 +SENDER_OUTPUT='GELF_SEND_ERROR code=ENOTFOUND' +reset_fixtures +RECORD_LINES=() +assert_isolated_probe 'name target' container test-network vector +[[ "${RECORD_LINES[*]}" == *"name target not routable (ENOTFOUND)"* ]] || fail "name target ENOTFOUND was not reported as not routable" +[[ "${RECORD_LINES[*]}" == *"source=0 shipped_probe=0"* ]] || fail "name target did not require zero deltas" + +# ENOTFOUND is only valid for names; reachability errors are only valid for IP literals. +prepare_sender_error_probe +expect_probe_failure 'isolation check failed' 10.255.253.2 1 'GELF_SEND_ERROR code=ENOTFOUND' + +TEST_NOW_MS=30000 +METRIC_SAMPLES=('10 31000' '10 32000' '10 33000' '10 34000') +SHIPPED_SAMPLES=(5 5) +SHIPPED_NAME_SAMPLES=('[]' '[]') +SENDER_OUTPUT='GELF_SEND_ERROR code=EHOSTUNREACH' +reset_fixtures +RECORD_LINES=() +assert_isolated_probe 'IP target' container test-network 10.255.253.2 +[[ "${RECORD_LINES[*]}" == *"IP target not routable (EHOSTUNREACH)"* ]] || fail "IP EHOSTUNREACH was not reported as not routable" + +TEST_NOW_MS=35000 +METRIC_SAMPLES=('10 36000' '10 37000' '10 38000' '10 39000') +SHIPPED_SAMPLES=(5 5) +SHIPPED_NAME_SAMPLES=('[]' '[]') +SENDER_OUTPUT='GELF_SEND_ERROR code=ENETUNREACH' +reset_fixtures +RECORD_LINES=() +assert_isolated_probe 'unreachable IP target' container test-network 10.255.253.2 +[[ "${RECORD_LINES[*]}" == *"unreachable IP target not routable (ENETUNREACH)"* ]] || fail "IP ENETUNREACH was not reported as not routable" +SENDER_OUTPUT='GELF_SEND_ERROR code=EHOSTUNREACH' +prepare_sender_error_probe +expect_probe_failure 'status=2' vector 1 "$SENDER_OUTPUT" + +# Transient DNS failures retry exactly three times, then fail closed. +SENDER_OUTPUT='GELF_SEND_ERROR code=EAI_AGAIN' +SEND_COUNT=0 +RECORD_LINES=() +SEND_COUNT_FILE="$CONTROL/sender-attempts" +: >"$SEND_COUNT_FILE" +prepare_sender_error_probe +if check_isolated_probe 'EAI_AGAIN target' container test-network vector; then + fail "EAI_AGAIN passed after retries" +else + assert_eq "$?" 2 +fi +assert_eq "$SEND_COUNT" 3 +assert_eq "$(wc -c <"$SEND_COUNT_FILE")" 3 +[[ "${RECORD_LINES[*]}" == *"EAI_AGAIN remained after 3 attempts"* ]] || fail "EAI_AGAIN retry exhaustion was not reported" +SEND_COUNT_FILE="" + +prepare_sender_error_probe +expect_probe_failure 'status=2' vector 1 'GELF_SEND_ERROR code=ETIMEDOUT' +prepare_sender_error_probe +expect_probe_failure 'status=2' vector 124 'GELF_SEND_ERROR code=ETIMEDOUT' +prepare_sender_error_probe +expect_probe_failure 'status=2' vector 125 'docker run failed' +prepare_sender_error_probe +expect_probe_failure 'status=2' vector 126 'docker run failed' +prepare_sender_error_probe +expect_probe_failure 'status=2' vector 1 'usage: send-gelf HOST PORT JSON' + +# A not-routable error contradicts a received probe and therefore cannot pass. +TEST_NOW_MS=40000 +METRIC_SAMPLES=('10 41000' '10 42000' '11 43000' '11 44000') +SHIPPED_SAMPLES=(5 6) +SHIPPED_NAME_SAMPLES=('[]' '[{"event":"node_warning","event_code":"invocation_replay_insert_failed","count":1}]') +SENDER_OUTPUT='GELF_SEND_ERROR code=ENOTFOUND' +reset_fixtures +expect_probe_failure 'isolation check failed' vector 1 "$SENDER_OUTPUT" + +# Positive controls and the isolation leak self-test never accept sender errors. +SENDER_OUTPUT='GELF_SEND_ERROR code=ENOTFOUND' +if output=$(send_positive_probe 'host loopback' host host 127.0.0.1 2>&1); then + fail "positive-control sender error passed" +fi +[[ "$output" == *"positive control sender failed"* ]] || fail "positive-control sender error was not fatal: $output" +SENDER_OUTPUT='GELF_SEND_ERROR code=EHOSTUNREACH' +prepare_sender_error_probe +if output=$(run_isolation_leak_self_test host host 127.0.0.1 2>&1); then + fail "leak self-test accepted a sender error" +fi +[[ "$output" == *"could not deliver its allowed packet"* ]] || fail "leak self-test sender error was not fatal: $output" +SENDER_EXIT=0 +SENDER_OUTPUT="" +printf '%s\n' 'PASS: sender failures are classified by exact code, target type, and context' printf '%s\n' 'PASS: isolation freshness, positive controls, and leak self-test' # The raw console scan catches a marker hidden in an unexpected field even when the probe filter returns zero. diff --git a/tc-959-node-docker-check.review.sh b/tc-959-node-docker-check.review.sh index bce310a6..72b19c19 100755 --- a/tc-959-node-docker-check.review.sh +++ b/tc-959-node-docker-check.review.sh @@ -167,11 +167,13 @@ cat >"$SENDER" <<'NODE' import { createSocket } from 'node:dgram'; const [host, portText, payload] = process.argv.slice(2); if (!host || !portText || !payload) throw new Error('usage: send-gelf HOST PORT JSON'); +const recognizedCodes = new Set(['ENOTFOUND', 'EAI_AGAIN', 'ENETUNREACH', 'EHOSTUNREACH', 'ECONNREFUSED']); const socket = createSocket('udp4'); socket.send(Buffer.from(payload), Number(portText), host, (error) => { socket.close(); if (error) { - console.error(`GELF UDP send failed to ${host}:${portText}: ${error.code ?? error.message}`); + const code = recognizedCodes.has(error.code) ? error.code : 'other'; + console.error(`GELF_SEND_ERROR code=${code}`); process.exitCode = 1; } }); @@ -256,7 +258,7 @@ EOF cat >"$CPU_OVERRIDE" <<'EOF' services: vector: - cpus: 0.5 + cpus: 0.25 EOF node - "$OVERRIDE" "$CPU_OVERRIDE" "$ROOT" "$ROOT/tc-959-node-docker-check.review.sh" "$VECTOR_TEST_CONFIG" <<'NODE' const fs = require('node:fs'); @@ -268,7 +270,7 @@ const [overridePath, cpuOverridePath, root, harnessPath, vectorConfigPath] = pro const composeText = fs.readFileSync(overridePath, 'utf8'); const cpuOverride = parse(fs.readFileSync(cpuOverridePath, 'utf8')); const compose = parse(composeText.replaceAll('!override', '')); -if (cpuOverride.services?.vector?.cpus !== 0.5) throw new Error('half-CPU override must set only Vector cpus: 0.5'); +if (cpuOverride.services?.vector?.cpus !== 0.25) throw new Error('half-CPU override must set only Vector cpus: 0.25'); const baseCompose = parse(fs.readFileSync(`${root}/docker-compose.dstack-postgres.yaml`, 'utf8')); const baseVectorEnv = baseCompose.services.vector.environment ?? {}; if (!Object.hasOwn(baseVectorEnv, 'VECTOR_CONFIG_YAML') || !Object.hasOwn(baseVectorEnv, 'BETTERSTACK_SOURCE_TOKEN')) { @@ -277,7 +279,7 @@ if (!Object.hasOwn(baseVectorEnv, 'VECTOR_CONFIG_YAML') || !Object.hasOwn(baseVe if (baseCompose.services?.vector?.init !== true) { throw new Error('production Vector service must set init: true for restart supervision'); } -if (baseCompose.services?.vector?.cpus !== 0.25) throw new Error('production Vector CPU setting changed unexpectedly'); +if (baseCompose.services?.vector?.cpus !== 0.5) throw new Error('production Vector CPU setting changed unexpectedly'); const vectorEnv = compose.services?.vector?.environment; if (!vectorEnv || Object.keys(vectorEnv).length !== 0) throw new Error('acceptance Vector environment must be explicitly empty'); const ingressEnv = compose.services?.['dstack-ingress']?.environment; @@ -343,6 +345,7 @@ const calledHelpers = [ 'wait_fresh_stable_metric', 'fresh_sample', 'wait_metric', 'vector_shipped_count', 'vector_shipped_name_counts', 'report_shipped_event_delta', 'vector_event_count', 'assert_no_console_marker', 'wait_next_line', 'send_from_container', 'send_probe', + 'send_positive_probe', 'run_probe_sender', 'sender_error_code', 'probe_routing_error_matches_target', 'compose_half_cpu_cmd', 'measure_burst', 'run_isolation_leak_self_test', ]; const missing = calledHelpers.filter((name) => !declared.has(name)); @@ -359,8 +362,8 @@ if [[ "$DRY_RUN" == 1 ]]; then record "DRY-RUN: acceptance self-test expects an allowed host-loopback packet to make the negative check fail" record "DRY-RUN: positive controls require source +1 and exact event=node_warning/event_code=invocation_replay_insert_failed shipped +1" record "DRY-RUN: negative probes require source +0 and exact probe shipped +0; unrelated event deltas print event/event_code enums" - record "DRY-RUN: 5,000-line bursts measured with fresh source samples at production cpus=0.25 and temporary cpus=0.5" - record "DRY-RUN: half-CPU Vector is recreated only for the second burst and production cpus=0.25 restored before isolation" + record "DRY-RUN: 5,000-line bursts measured with fresh source samples at production cpus=0.5 and temporary cpus=0.25" + record "DRY-RUN: half-CPU Vector is recreated only for the second burst and production cpus=0.5 restored before isolation" record "DRY-RUN: actual Docker, network and memory checks NOT run" exit 0 fi @@ -467,17 +470,17 @@ node_restarts_after=$(docker_cmd inspect -f '{{.RestartCount}}' "$NODE_ID") [[ "$node_restarts" == "$node_restarts_after" ]] || fail "node restarted during Vector recovery matrix" record "PASS: app restart count unchanged across all Vector recovery tests" -measure_burst 0.25 +measure_burst 0.5 compose_half_cpu_cmd up -d --force-recreate --no-deps vector >/dev/null wait_vector wait_next_line 0 -measure_burst 0.5 +measure_burst 0.25 compose_cmd up -d --force-recreate --no-deps vector >/dev/null wait_vector wait_next_line 0 restored_nano_cpus=$(docker_cmd inspect -f '{{.HostConfig.NanoCpus}}' "$VECTOR_ID") -[[ "$restored_nano_cpus" == 250000000 ]] || fail "Vector CPU limit after restore is $restored_nano_cpus, expected 250000000" -record "PASS: restored production Vector NanoCpus=250000000 before isolation, node-latency, and memory checks" +[[ "$restored_nano_cpus" == 500000000 ]] || fail "Vector CPU limit after restore is $restored_nano_cpus, expected 500000000" +record "PASS: restored production Vector NanoCpus=500000000 before isolation, node-latency, and memory checks" # Keep the idle synthetic ingress up so its network namespace is available for sibling probes. PROBE_CANARY=TC959_FORGED_PROBE_CANARY_UNEXPECTED_FIELD @@ -505,14 +508,14 @@ source_before=$(fresh_sample) shipped_before=$(vector_shipped_count) shipped_names_before=$(vector_shipped_name_counts) sent_at=$(metric_now_ms) -node "$SENDER" 127.0.0.1 12201 "$payload" +send_positive_probe "host loopback" host host 127.0.0.1 assert_positive_delivery "$source_before" "$shipped_before" "$sent_at" "host loopback" "$shipped_names_before" source_before=$(fresh_sample) shipped_before=$(vector_shipped_count) shipped_names_before=$(vector_shipped_name_counts) sent_at=$(metric_now_ms) -send_from_container "${PROJECT}_logship" "$vector_ip" +send_positive_probe "logship-network control container" container "${PROJECT}_logship" "$vector_ip" assert_positive_delivery "$source_before" "$shipped_before" "$sent_at" "logship-network control container" "$shipped_names_before" From d0160f2d6a80c3ae55452c42c8d4a813078d0950 Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Thu, 15 Oct 2026 00:02:39 +0000 Subject: [PATCH 19/21] fix(tc-959): prove name-target isolation --- scripts/tc-959-isolation.sh | 212 +++++++++++++++++++++++++-- scripts/test-tc-959-isolation.sh | 222 ++++++++++++++++++++++++++--- tc-959-node-docker-check.review.sh | 9 +- 3 files changed, 412 insertions(+), 31 deletions(-) diff --git a/scripts/tc-959-isolation.sh b/scripts/tc-959-isolation.sh index 8616a701..be9a890c 100644 --- a/scripts/tc-959-isolation.sh +++ b/scripts/tc-959-isolation.sh @@ -240,10 +240,189 @@ probe_routing_error_matches_target() { *) return 1 ;; esac } +probe_target_is_name() { + [[ "$1" == vector || "$1" == host.docker.internal ]] +} + +name_network_names_from_json() { + node -e ' + const fs = require("node:fs"); + let networks; + try { networks = JSON.parse(fs.readFileSync(0, "utf8")); } catch { process.exit(2); } + if (!networks || typeof networks !== "object" || Array.isArray(networks)) process.exit(2); + const names = Object.keys(networks); + if (names.length === 0) process.exit(3); + if (names.some((name) => !/^[A-Za-z0-9_.-]+$/.test(name))) process.exit(2); + process.stdout.write(`${names.sort().join(",")}\n`); + ' +} + +name_network_container_ids_from_json() { + node -e ' + const fs = require("node:fs"); + let containers; + try { containers = JSON.parse(fs.readFileSync(0, "utf8")); } catch { process.exit(2); } + if (containers === null) containers = {}; + if (typeof containers !== "object" || Array.isArray(containers)) process.exit(2); + const ids = Object.keys(containers); + if (ids.some((id) => !/^[A-Za-z0-9_.-]+$/.test(id))) process.exit(2); + if (ids.length) process.stdout.write(`${ids.join("\n")}\n`); + ' +} + +name_peer_matches_target() { + local network=$1 target=$2 peer_json=$3 + printf '%s\n' "$peer_json" | node -e ' + const fs = require("node:fs"); + const [network, target] = process.argv.slice(1); + let peer; + try { peer = JSON.parse(fs.readFileSync(0, "utf8")); } catch { process.exit(2); } + if (!peer || typeof peer !== "object" || Array.isArray(peer) || + typeof peer.name !== "string" || typeof peer.hostname !== "string" || + (peer.service !== null && typeof peer.service !== "string") || + !peer.networks || typeof peer.networks !== "object" || !Object.hasOwn(peer.networks, network)) { + process.exit(2); + } + const endpoint = peer.networks[network]; + if (!endpoint || typeof endpoint !== "object" || Array.isArray(endpoint)) process.exit(2); + const aliases = endpoint.Aliases ?? []; + const dnsNames = endpoint.DNSNames ?? []; + if (!Array.isArray(aliases) || !Array.isArray(dnsNames) || + aliases.some((name) => typeof name !== "string") || + dnsNames.some((name) => typeof name !== "string")) process.exit(2); + const names = [peer.name.replace(/^\/+/, ""), peer.hostname, peer.service, ...aliases, ...dnsNames]; + process.exit(names.includes(target) ? 0 : 1); + ' "$network" "$target" +} + +hosts_file_has_name() { + local hosts_file=$1 target=$2 + awk -v target="$target" ' + /^[[:space:]]*#/ { next } + { + for (i = 2; i <= NF; i++) { + if ($i ~ /^#/) break; + if ($i == target) found = 1; + } + } + END { exit !found } + ' <<<"$hosts_file" +} + +name_resolver_positive_control() { + local target_container=$1 result project=${PROJECT:-tc959-isolation-test} + result=$(docker_cmd timeout 60 run --rm \ + --label "com.docker.compose.project=$project" \ + --label "tc959.probe=true" \ + --network "container:$target_container" \ + node:20-alpine@sha256:fb4cd12c85ee03686f6af5362a0b0d56d50c58a04632e6c0fb8363f609372293 \ + node -e 'require("node:dns").lookup("tinycloud", (error) => { + if (error) { + process.stderr.write(`GELF_RESOLVER_ERROR code=${error.code === "EAI_AGAIN" ? "EAI_AGAIN" : "other"}\n`); + process.exitCode = 1; + } else process.stdout.write("GELF_RESOLVER_OK\n"); + })' 2>&1) || return 1 + [[ "$result" == GELF_RESOLVER_OK ]] +} + +prove_name_target_structure() { + local description=$1 target_container=$2 target=$3 error_code=$4 + local hosts_file sender_networks_json vector_networks_json sender_networks vector_networks + local sender_parse_status vector_parse_status shared_networks=0 alias_matches=0 + local network containers_json peer_ids peer_id peer_metadata peer_match_status + local sender_member_found peer_format + local -a sender_network_names=() vector_network_names=() + if [[ -z "$target_container" ]]; then + record "INFO: $description name proof failed; sender_networks=unavailable vector_networks=unavailable shared_networks=unknown alias_matches=unknown hosts_entry=unknown resolver_control=not-run error_code=$error_code reason=missing_container_id" + return 2 + fi + if ! hosts_file=$(docker_cmd exec "$target_container" cat /etc/hosts 2>/dev/null); then + record "INFO: $description name proof failed; sender_networks=unavailable vector_networks=unavailable shared_networks=unknown alias_matches=unknown hosts_entry=unreadable resolver_control=not-run error_code=$error_code" + return 2 + fi + if hosts_file_has_name "$hosts_file" "$target"; then + record "INFO: $description name target is in hosts file; sender_networks=not-checked vector_networks=not-checked shared_networks=not-applicable alias_matches=not-applicable hosts_entry=1 resolver_control=not-needed error_code=$error_code" + return 2 + fi + if ! sender_networks_json=$(docker_cmd inspect -f '{{json .NetworkSettings.Networks}}' "$target_container" 2>/dev/null); then + record "INFO: $description name proof failed; sender_networks=unavailable vector_networks=unavailable shared_networks=unknown alias_matches=unknown hosts_entry=0 resolver_control=not-run error_code=$error_code inspect_error=1" + return 2 + fi + if ! vector_networks_json=$(docker_cmd inspect -f '{{json .NetworkSettings.Networks}}' "$VECTOR_ID" 2>/dev/null); then + record "INFO: $description name proof failed; sender_networks=read vector_networks=unavailable shared_networks=unknown alias_matches=unknown hosts_entry=0 resolver_control=not-run error_code=$error_code inspect_error=1" + return 2 + fi + if sender_networks=$(name_network_names_from_json <<<"$sender_networks_json"); then + : + else + sender_parse_status=$? + if (( sender_parse_status == 3 )); then sender_networks=empty; else sender_networks=invalid; fi + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=read shared_networks=unknown alias_matches=unknown hosts_entry=0 resolver_control=not-run error_code=$error_code inspect_error=1" + return 2 + fi + if vector_networks=$(name_network_names_from_json <<<"$vector_networks_json"); then + : + else + vector_parse_status=$? + if (( vector_parse_status == 3 )); then vector_networks=empty; else vector_networks=invalid; fi + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=unknown alias_matches=unknown hosts_entry=0 resolver_control=not-run error_code=$error_code inspect_error=1" + return 2 + fi + IFS=, read -r -a sender_network_names <<<"$sender_networks" + IFS=, read -r -a vector_network_names <<<"$vector_networks" + for network in "${sender_network_names[@]}"; do + for peer_id in "${vector_network_names[@]}"; do + if [[ "$network" == "$peer_id" ]]; then shared_networks=$((shared_networks + 1)); fi + done + done + peer_format='{{printf "{\"name\":%s,\"hostname\":%s,\"service\":%s,\"networks\":%s}" (json .Name) (json .Config.Hostname) (json (index .Config.Labels "com.docker.compose.service")) (json .NetworkSettings.Networks)}}' + for network in "${sender_network_names[@]}"; do + if ! containers_json=$(docker_cmd network inspect -f '{{json .Containers}}' "$network" 2>/dev/null); then + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=$alias_matches hosts_entry=0 resolver_control=not-run error_code=$error_code network_inspect_error=1" + return 2 + fi + if ! peer_ids=$(name_network_container_ids_from_json <<<"$containers_json"); then + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=$alias_matches hosts_entry=0 resolver_control=not-run error_code=$error_code network_inspect_error=1" + return 2 + fi + sender_member_found=0 + while IFS= read -r peer_id; do + [[ -n "$peer_id" ]] || continue + [[ "$peer_id" == "$target_container" ]] && sender_member_found=1 + if ! peer_metadata=$(docker_cmd inspect -f "$peer_format" "$peer_id" 2>/dev/null); then + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=$alias_matches hosts_entry=0 resolver_control=not-run error_code=$error_code inspect_error=1" + return 2 + fi + if name_peer_matches_target "$network" "$target" "$peer_metadata"; then + alias_matches=1 + else + peer_match_status=$? + if (( peer_match_status != 1 )); then + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=unknown hosts_entry=0 resolver_control=not-run error_code=$error_code inspect_error=1" + return 2 + fi + fi + done <<<"$peer_ids" + if (( sender_member_found == 0 )); then + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=$alias_matches hosts_entry=0 resolver_control=not-run error_code=$error_code network_sender_match=0" + return 2 + fi + done + if (( shared_networks != 0 || alias_matches != 0 )); then + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=$alias_matches hosts_entry=0 resolver_control=not-run error_code=$error_code" + return 2 + fi + if ! name_resolver_positive_control "$target_container"; then + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=0 alias_matches=0 hosts_entry=0 resolver_control=failed error_code=$error_code" + return 2 + fi + PROBE_NAME_EVIDENCE="sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=0 alias_matches=0 hosts_entry=0 resolver_control=ok error_code=$error_code" + record "INFO: $description name unresolvable ($error_code); structural: $PROBE_NAME_EVIDENCE" +} check_isolated_probe() { local description=$1 sender=$2 network=$3 target=$4 - local sender_errors=${5:-allow-routing-errors} + local target_container=${5:-} sender_errors=${6:-allow-routing-errors} local source_before shipped_before shipped_names_before sent_at source_after shipped_after shipped_names_after local source_delta shipped_delta send_status=0 sender_code="" routing_code="" eai_attempts=0 source_before=$(fresh_sample) @@ -262,17 +441,29 @@ check_isolated_probe() { fi record "INFO: $description sender error code=$sender_code exit=$send_status" if [[ "$sender_code" == EAI_AGAIN ]]; then + probe_target_is_name "$target" && [[ "$sender_errors" == allow-routing-errors ]] || { + record "INFO: $description EAI_AGAIN is not acceptable for this target/context" + return 2 + } eai_attempts=$((eai_attempts + 1)) if (( eai_attempts >= 3 )); then - record "INFO: $description EAI_AGAIN remained after 3 attempts" - return 2 + if ! prove_name_target_structure "$description" "$target_container" "$target" "$sender_code"; then + return 2 + fi + routing_code=$sender_code + break fi record "INFO: $description retrying EAI_AGAIN attempt $((eai_attempts + 1))/3" continue fi if [[ "$sender_errors" == allow-routing-errors ]] && probe_routing_error_matches_target "$sender_code" "$target"; then + if [[ "$sender_code" == ENOTFOUND ]]; then + if ! prove_name_target_structure "$description" "$target_container" "$target" "$sender_code"; then + return 2 + fi + fi routing_code=$sender_code - record "INFO: $description not routable ($routing_code)" + if [[ "$sender_code" != ENOTFOUND ]]; then record "INFO: $description not routable ($routing_code)"; fi break fi record "INFO: $description sender error code=$sender_code is not acceptable for this target/context" @@ -287,7 +478,11 @@ check_isolated_probe() { record "INFO: $description fresh deltas source=$source_delta shipped_probe=$shipped_delta" if (( source_delta == 0 && shipped_delta == 0 )); then if [[ -n "$routing_code" ]]; then - record "PASS: $description isolated: not routable ($routing_code); fresh source +0 and shipped +0" + if [[ "$routing_code" == ENOTFOUND || "$routing_code" == EAI_AGAIN ]]; then + record "PASS: $description isolated: name unresolvable ($routing_code); fresh source +0 and shipped +0" + else + record "PASS: $description isolated: not routable ($routing_code); fresh source +0 and shipped +0" + fi return 3 fi return 0 @@ -324,8 +519,9 @@ assert_no_console_marker() { [[ "$match_count" == 0 ]] } assert_isolated_probe() { - local description=$1 sender=$2 network=$3 target=$4 status - if check_isolated_probe "$description" "$sender" "$network" "$target"; then + local description=$1 sender=$2 network=$3 target=$4 + local target_container=${5:-} status + if check_isolated_probe "$description" "$sender" "$network" "$target" "$target_container"; then record "PASS: $description isolation; fresh source +0 and shipped +0" return 0 else @@ -337,7 +533,7 @@ assert_isolated_probe() { run_isolation_leak_self_test() { local sender=$1 network=$2 target=$3 status - if check_isolated_probe "intentional host-loopback leak self-test" "$sender" "$network" "$target" must-deliver; then + if check_isolated_probe "intentional host-loopback leak self-test" "$sender" "$network" "$target" "" must-deliver; then fail "isolation leak self-test passed unexpectedly" else status=$? diff --git a/scripts/test-tc-959-isolation.sh b/scripts/test-tc-959-isolation.sh index bc89f62b..588d1194 100755 --- a/scripts/test-tc-959-isolation.sh +++ b/scripts/test-tc-959-isolation.sh @@ -4,6 +4,7 @@ set -Eeuo pipefail ROOT=$(cd "$(dirname "$0")/.." && pwd) source "$ROOT/scripts/tc-959-isolation.sh" +LOGS_SHOULD_FAIL=0 METRICS_SCRAPE_INTERVAL_SECONDS=1 ISOLATION_SETTLE_SECONDS=2 TEST_NOW_MS=3000 @@ -29,13 +30,69 @@ vector_shipped_name_counts() { } report_shipped_event_delta() { :; } compose_cmd() { printf '%s\n' test-vector; } -LOGS_SHOULD_FAIL=0 +NAME_STUBS=0 +NAME_SENDER_ID="" +NAME_SENDER_NETWORKS='{"default":{}}' +NAME_VECTOR_NETWORKS='{"logship":{}}' +NAME_NETWORK_CONTAINERS='{"sender-id":{"Name":"tinycloud"},"peer-id":{"Name":"other"}}' +NAME_SENDER_METADATA='{"name":"/tinycloud-container","hostname":"tinycloud-host","service":"tinycloud","networks":{"default":{"Aliases":["tinycloud"],"DNSNames":["tinycloud-container"]}}}' +NAME_PEER_METADATA='{"name":"/other","hostname":"other-host","service":"other","networks":{"default":{"Aliases":["other"],"DNSNames":["other"]}}}' +NAME_HOSTS='127.0.0.1 localhost' +NAME_INSPECT_FAIL=0 +NAME_NETWORK_INSPECT_FAIL=0 +NAME_HOSTS_FAIL=0 +NAME_RESOLVER_MODE=ok +NAME_RESOLVER_NETWORK_FILE="" docker_cmd() { if [[ "$1" == logs ]]; then if (( LOGS_SHOULD_FAIL )); then printf '%s\n' 'stub docker logs failure' >&2; return 1; fi printf '%s\n' "$RAW_CONSOLE_OUTPUT" return 0 fi + if (( NAME_STUBS )); then + case "$1" in + inspect) + if (( NAME_INSPECT_FAIL )); then return 1; fi + if [[ "${3:-}" == '{{json .NetworkSettings.Networks}}' ]]; then + if [[ "${4:-}" == "$NAME_SENDER_ID" ]]; then printf '%s\n' "$NAME_SENDER_NETWORKS" + elif [[ "${4:-}" == "$VECTOR_ID" ]]; then printf '%s\n' "$NAME_VECTOR_NETWORKS" + else printf '%s\n' "$NAME_PEER_METADATA" + fi + return 0 + fi + if [[ "${3:-}" == *networks* ]]; then + if [[ "${4:-}" == "$NAME_SENDER_ID" ]]; then printf '%s\n' "$NAME_SENDER_METADATA" + else printf '%s\n' "$NAME_PEER_METADATA" + fi + return 0 + fi + ;; + network) + if (( NAME_NETWORK_INSPECT_FAIL )); then return 1; fi + printf '%s\n' "$NAME_NETWORK_CONTAINERS" + return 0 + ;; + exec) + if (( NAME_HOSTS_FAIL )); then return 1; fi + [[ "${2:-}" == "$NAME_SENDER_ID" && "${3:-}" == cat && "${4:-}" == /etc/hosts ]] || return 1 + printf '%s\n' "$NAME_HOSTS" + return 0 + ;; + timeout) + if [[ -n "$NAME_RESOLVER_NETWORK_FILE" ]]; then + for arg in "$@"; do + if [[ "$arg" == "container:$NAME_SENDER_ID" ]]; then printf '%s\n' "$arg" >"$NAME_RESOLVER_NETWORK_FILE"; fi + done + + fi + case "$NAME_RESOLVER_MODE" in + ok) printf '%s\n' GELF_RESOLVER_OK; return 0 ;; + eai) printf '%s\n' 'GELF_RESOLVER_ERROR code=EAI_AGAIN'; return 1 ;; + *) return 1 ;; + esac + ;; + esac + fi [[ "$1" == inspect ]] && printf '%s\n' true } curl() { return 0; } @@ -74,6 +131,7 @@ expect_probe_failure() { prepare_sender_error_probe() { TEST_NOW_MS=50000 + SENDER_EXIT=1 METRIC_SAMPLES=('10 51000' '10 52000') SHIPPED_SAMPLES=(5) SHIPPED_NAME_SAMPLES=('[]') @@ -85,6 +143,51 @@ reset_fixtures() { exec 4< <(printf '%s\n' "${SHIPPED_SAMPLES[@]}") exec 6< <(printf '%s\n' "${SHIPPED_NAME_SAMPLES[@]}") } + +prepare_name_probe() { + NAME_STUBS=1 + NAME_SENDER_ID=sender-id + VECTOR_ID=vector-id + NAME_SENDER_NETWORKS='{"default":{}}' + NAME_VECTOR_NETWORKS='{"logship":{}}' + NAME_NETWORK_CONTAINERS='{"sender-id":{"Name":"tinycloud"},"peer-id":{"Name":"other"}}' + NAME_SENDER_METADATA='{"name":"/tinycloud-container","hostname":"tinycloud-host","service":"tinycloud","networks":{"default":{"Aliases":["tinycloud"],"DNSNames":["tinycloud-container"]}}}' + NAME_PEER_METADATA='{"name":"/other","hostname":"other-host","service":"other","networks":{"default":{"Aliases":["other"],"DNSNames":["other"]}}}' + NAME_HOSTS='127.0.0.1 localhost' + NAME_INSPECT_FAIL=0 + NAME_NETWORK_INSPECT_FAIL=0 + NAME_HOSTS_FAIL=0 + NAME_RESOLVER_MODE=ok + NAME_RESOLVER_NETWORK_FILE="$CONTROL/resolver-network" + SENDER_EXIT=1 + SENDER_OUTPUT='GELF_SEND_ERROR code=EAI_AGAIN' + TEST_NOW_MS=60000 + METRIC_SAMPLES=('10 61000' '10 62000' '10 63000' '10 64000') + SHIPPED_SAMPLES=(5 5) + SHIPPED_NAME_SAMPLES=('[]' '[]') + RECORD_LINES=() + : >"$NAME_RESOLVER_NETWORK_FILE" + reset_fixtures +} + +expect_name_proof_failure() { + local expected=$1 target=${2:-vector} status + if prove_name_target_structure 'stub-name-proof' "$NAME_SENDER_ID" "$target" EAI_AGAIN; then + fail "name target proof accepted $expected" + else + status=$? + fi + assert_eq "$status" 2 + [[ "${RECORD_LINES[*]}" == *"$expected"* ]] || fail "name target proof did not report $expected: ${RECORD_LINES[*]}" +} + +expect_name_isolation_failure() { + local expected=$1 target=$2 output + if output=$(assert_isolated_probe "stub-name->$target" container "container:$NAME_SENDER_ID" "$target" "$NAME_SENDER_ID" 2>&1); then + fail "name isolation check accepted $expected" + fi + [[ "$output" == *"$expected"* ]] || fail "name isolation check did not explain $expected: $output" +} wait_vector wait_node assert_eq "$(wait_metric 3)" 3 @@ -126,18 +229,6 @@ SHIPPED_NAME_SAMPLES=('[]' '[{"event":"node_warning","event_code":"invocation_re reset_fixtures run_isolation_leak_self_test host host 127.0.0.1 assert_eq "$SEND_COUNT" 1 -# Not-routable sender errors pass only after fresh +0/+0 samples for a matching target type. -TEST_NOW_MS=20000 -METRIC_SAMPLES=('10 21000' '10 22000' '10 23000' '10 24000') -SHIPPED_SAMPLES=(5 5) -SHIPPED_NAME_SAMPLES=('[]' '[]') -SENDER_EXIT=1 -SENDER_OUTPUT='GELF_SEND_ERROR code=ENOTFOUND' -reset_fixtures -RECORD_LINES=() -assert_isolated_probe 'name target' container test-network vector -[[ "${RECORD_LINES[*]}" == *"name target not routable (ENOTFOUND)"* ]] || fail "name target ENOTFOUND was not reported as not routable" -[[ "${RECORD_LINES[*]}" == *"source=0 shipped_probe=0"* ]] || fail "name target did not require zero deltas" # ENOTFOUND is only valid for names; reachability errors are only valid for IP literals. prepare_sender_error_probe @@ -180,7 +271,7 @@ else fi assert_eq "$SEND_COUNT" 3 assert_eq "$(wc -c <"$SEND_COUNT_FILE")" 3 -[[ "${RECORD_LINES[*]}" == *"EAI_AGAIN remained after 3 attempts"* ]] || fail "EAI_AGAIN retry exhaustion was not reported" +[[ "${RECORD_LINES[*]}" == *"missing_container_id"* ]] || fail "EAI_AGAIN did not require structural proof" SEND_COUNT_FILE="" prepare_sender_error_probe @@ -194,14 +285,13 @@ expect_probe_failure 'status=2' vector 126 'docker run failed' prepare_sender_error_probe expect_probe_failure 'status=2' vector 1 'usage: send-gelf HOST PORT JSON' -# A not-routable error contradicts a received probe and therefore cannot pass. -TEST_NOW_MS=40000 -METRIC_SAMPLES=('10 41000' '10 42000' '11 43000' '11 44000') +prepare_name_probe +SENDER_OUTPUT='GELF_SEND_ERROR code=ENOTFOUND' +METRIC_SAMPLES=('10 61000' '10 62000' '11 63000' '11 64000') SHIPPED_SAMPLES=(5 6) SHIPPED_NAME_SAMPLES=('[]' '[{"event":"node_warning","event_code":"invocation_replay_insert_failed","count":1}]') -SENDER_OUTPUT='GELF_SEND_ERROR code=ENOTFOUND' reset_fixtures -expect_probe_failure 'isolation check failed' vector 1 "$SENDER_OUTPUT" +expect_name_isolation_failure 'status=1' vector # Positive controls and the isolation leak self-test never accept sender errors. SENDER_OUTPUT='GELF_SEND_ERROR code=ENOTFOUND' @@ -215,6 +305,97 @@ if output=$(run_isolation_leak_self_test host host 127.0.0.1 2>&1); then fail "leak self-test accepted a sender error" fi [[ "$output" == *"could not deliver its allowed packet"* ]] || fail "leak self-test sender error was not fatal: $output" + +# Name errors require independent structure and a successful same-network resolver control. +prepare_name_probe +assert_isolated_probe 'vector name with transient DNS errors' container "container:$NAME_SENDER_ID" vector "$NAME_SENDER_ID" +assert_eq "$SEND_COUNT" 3 +assert_eq "$(<"$NAME_RESOLVER_NETWORK_FILE")" "container:$NAME_SENDER_ID" +[[ "${RECORD_LINES[*]}" == *"name unresolvable (EAI_AGAIN); structural: sender_networks=default vector_networks=logship shared_networks=0 alias_matches=0 hosts_entry=0 resolver_control=ok error_code=EAI_AGAIN"* ]] || fail "name proof did not report safe structural evidence: ${RECORD_LINES[*]}" +[[ "${RECORD_LINES[*]}" == *"fresh source +0 and shipped +0"* ]] || fail "name proof passed without zero traffic deltas" + +prepare_name_probe +SENDER_OUTPUT='GELF_SEND_ERROR code=ENOTFOUND' +assert_isolated_probe 'vector name with ENOTFOUND' container "container:$NAME_SENDER_ID" vector "$NAME_SENDER_ID" +assert_eq "$SEND_COUNT" 1 + +prepare_name_probe +NAME_VECTOR_NETWORKS='{"default":{}}' +expect_name_proof_failure 'shared_networks=1' + +prepare_name_probe +NAME_PEER_METADATA='{"name":"/peer","hostname":"peer-host","service":"peer","networks":{"default":{"Aliases":["vector"],"DNSNames":["peer"]}}}' +expect_name_proof_failure 'alias_matches=1' + +prepare_name_probe +NAME_PEER_METADATA='{"name":"/peer","hostname":"vector","service":"peer","networks":{"default":{"Aliases":["peer"],"DNSNames":["peer"]}}}' +expect_name_proof_failure 'alias_matches=1' + +prepare_name_probe +NAME_PEER_METADATA='{"name":"/vector","hostname":"peer-host","service":"peer","networks":{"default":{"Aliases":["peer"],"DNSNames":["peer"]}}}' +expect_name_proof_failure 'alias_matches=1' + +prepare_name_probe +NAME_PEER_METADATA='{"name":"/peer","hostname":"peer-host","service":"vector","networks":{"default":{"Aliases":["peer"],"DNSNames":["peer"]}}}' +expect_name_proof_failure 'alias_matches=1' + +prepare_name_probe +NAME_PEER_METADATA='{"name":"/peer","hostname":"peer-host","service":"peer","networks":{"default":{"Aliases":["peer"],"DNSNames":["vector"]}}}' +expect_name_proof_failure 'alias_matches=1' + +prepare_name_probe +NAME_HOSTS_FAIL=1 +expect_name_proof_failure 'hosts_entry=unreadable' + +prepare_name_probe +NAME_HOSTS='127.0.0.1 localhost host.docker.internal' +expect_name_proof_failure 'hosts_entry=1' host.docker.internal + +prepare_name_probe +NAME_HOSTS='127.0.0.1 localhost host.docker.internal' +SENDER_EXIT=0 +SENDER_OUTPUT="" +METRIC_SAMPLES=('10 61000' '10 62000' '10 63000' '10 64000') +SHIPPED_SAMPLES=(5 5) +SHIPPED_NAME_SAMPLES=('[]' '[]') +reset_fixtures +assert_isolated_probe 'hosts-mapped name is a real send' container "container:$NAME_SENDER_ID" host.docker.internal "$NAME_SENDER_ID" +assert_eq "$SEND_COUNT" 1 + +prepare_name_probe +NAME_INSPECT_FAIL=1 +expect_name_proof_failure 'inspect_error=1' + +prepare_name_probe +NAME_NETWORK_INSPECT_FAIL=1 +expect_name_proof_failure 'network_inspect_error=1' + +prepare_name_probe +NAME_SENDER_NETWORKS='{}' +expect_name_proof_failure 'sender_networks=empty' + +prepare_name_probe +NAME_VECTOR_NETWORKS='{}' +expect_name_proof_failure 'vector_networks=empty' + +prepare_name_probe +NAME_RESOLVER_MODE=eai +expect_name_proof_failure 'resolver_control=failed' + +prepare_name_probe +NAME_RESOLVER_MODE=fail +expect_name_proof_failure 'resolver_control=failed' + +prepare_name_probe +METRIC_SAMPLES=('10 61000' '10 62000' '11 63000' '11 64000') +reset_fixtures +expect_name_isolation_failure 'status=1' vector + +prepare_name_probe +SENDER_OUTPUT='GELF_SEND_ERROR code=EAI_AGAIN' +expect_probe_failure 'status=2' 10.255.253.2 1 "$SENDER_OUTPUT" + +NAME_STUBS=0 SENDER_EXIT=0 SENDER_OUTPUT="" printf '%s\n' 'PASS: sender failures are classified by exact code, target type, and context' @@ -236,12 +417,13 @@ unset -f vector_shipped_count vector_shipped_name_counts vector_event_count asse source "$ROOT/scripts/tc-959-isolation.sh" LOGS_SHOULD_FAIL=1 for reader in vector_shipped_count vector_shipped_name_counts vector_event_count assert_no_console_marker; do - if output=$("$reader" TC959_FORGED_PROBE_CANARY_UNEXPECTED_FIELD 2>&1); then + if output=$( ( "$reader" TC959_FORGED_PROBE_CANARY_UNEXPECTED_FIELD ) 2>&1); then fail "$reader passed after docker logs failed" fi [[ "$output" == *"docker logs failed"* ]] || fail "$reader did not fail closed: $output" done LOGS_SHOULD_FAIL=0 +printf '%s\n' 'PASS: console readers fail closed on docker logs errors' # Host-port preflight reports the owning Compose service before startup. PORT_HOLDER_MODE=container diff --git a/tc-959-node-docker-check.review.sh b/tc-959-node-docker-check.review.sh index 72b19c19..e7980a7a 100755 --- a/tc-959-node-docker-check.review.sh +++ b/tc-959-node-docker-check.review.sh @@ -346,6 +346,9 @@ const calledHelpers = [ 'vector_shipped_name_counts', 'report_shipped_event_delta', 'vector_event_count', 'assert_no_console_marker', 'wait_next_line', 'send_from_container', 'send_probe', 'send_positive_probe', 'run_probe_sender', 'sender_error_code', 'probe_routing_error_matches_target', + 'probe_target_is_name', 'name_network_names_from_json', 'name_network_container_ids_from_json', + 'name_peer_matches_target', 'hosts_file_has_name', 'name_resolver_positive_control', + 'prove_name_target_structure', 'compose_half_cpu_cmd', 'measure_burst', 'run_isolation_leak_self_test', ]; const missing = calledHelpers.filter((name) => !declared.has(name)); @@ -523,9 +526,9 @@ run_isolation_leak_self_test host host 127.0.0.1 probe_targets=("$vector_ip" "$app_gateway" vector host.docker.internal 127.0.0.1) for target in "${probe_targets[@]}"; do - assert_isolated_probe "node->$target" container "container:$NODE_ID" "$target" - assert_isolated_probe "ingress->$target" container "container:$INGRESS_ID" "$target" - assert_isolated_probe "postgres->$target" container "container:$PG_ID" "$target" + assert_isolated_probe "node->$target" container "container:$NODE_ID" "$target" "$NODE_ID" + assert_isolated_probe "ingress->$target" container "container:$INGRESS_ID" "$target" "$INGRESS_ID" + assert_isolated_probe "postgres->$target" container "container:$PG_ID" "$target" "$PG_ID" done host_ips=$(ip -o -4 addr show scope global | awk '{split($4,a,"/"); print a[1]}') From cd14439102a0e5771a128c4a710801799a2bf10d Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Thu, 15 Oct 2026 00:33:36 +0000 Subject: [PATCH 20/21] fix(tc-959): read probe hosts through namespace helper --- scripts/tc-959-isolation.sh | 52 ++++++++++++++++++++---------- scripts/test-tc-959-isolation.sh | 50 ++++++++++++++++++++-------- tc-959-node-docker-check.review.sh | 2 +- 3 files changed, 73 insertions(+), 31 deletions(-) diff --git a/scripts/tc-959-isolation.sh b/scripts/tc-959-isolation.sh index be9a890c..e3ae65a0 100644 --- a/scripts/tc-959-isolation.sh +++ b/scripts/tc-959-isolation.sh @@ -325,31 +325,49 @@ name_resolver_positive_control() { [[ "$result" == GELF_RESOLVER_OK ]] } +read_target_hosts_file() { + local target_container=$1 project=${PROJECT:-tc959-isolation-test} + docker_cmd timeout 60 run --rm \ + --label "com.docker.compose.project=$project" \ + --label "tc959.probe=true" \ + --network "container:$target_container" \ + node:20-alpine@sha256:fb4cd12c85ee03686f6af5362a0b0d56d50c58a04632e6c0fb8363f609372293 \ + cat /etc/hosts +} + prove_name_target_structure() { local description=$1 target_container=$2 target=$3 error_code=$4 - local hosts_file sender_networks_json vector_networks_json sender_networks vector_networks + local target_hostname hosts_file sender_networks_json vector_networks_json sender_networks vector_networks local sender_parse_status vector_parse_status shared_networks=0 alias_matches=0 local network containers_json peer_ids peer_id peer_metadata peer_match_status local sender_member_found peer_format local -a sender_network_names=() vector_network_names=() if [[ -z "$target_container" ]]; then - record "INFO: $description name proof failed; sender_networks=unavailable vector_networks=unavailable shared_networks=unknown alias_matches=unknown hosts_entry=unknown resolver_control=not-run error_code=$error_code reason=missing_container_id" + record "INFO: $description name proof failed; sender_networks=unavailable vector_networks=unavailable shared_networks=unknown alias_matches=unknown hosts_entry=unknown target_hostname=unavailable resolver_control=not-run error_code=$error_code reason=missing_container_id" + return 2 + fi + if ! target_hostname=$(docker_cmd inspect -f '{{.Config.Hostname}}' "$target_container" 2>/dev/null) || [[ -z "$target_hostname" ]]; then + record "INFO: $description name proof failed; sender_networks=unavailable vector_networks=unavailable shared_networks=unknown alias_matches=unknown hosts_entry=unknown target_hostname=unavailable resolver_control=not-run error_code=$error_code inspect_error=1" + return 2 + fi + if ! hosts_file=$(read_target_hosts_file "$target_container"); then + record "INFO: $description name proof failed; sender_networks=unavailable vector_networks=unavailable shared_networks=unknown alias_matches=unknown hosts_entry=unreadable target_hostname=unverified resolver_control=not-run error_code=$error_code" return 2 fi - if ! hosts_file=$(docker_cmd exec "$target_container" cat /etc/hosts 2>/dev/null); then - record "INFO: $description name proof failed; sender_networks=unavailable vector_networks=unavailable shared_networks=unknown alias_matches=unknown hosts_entry=unreadable resolver_control=not-run error_code=$error_code" + if ! hosts_file_has_name "$hosts_file" "$target_hostname"; then + record "INFO: $description name proof failed; sender_networks=unavailable vector_networks=unavailable shared_networks=unknown alias_matches=unknown hosts_entry=0 target_hostname=missing resolver_control=not-run error_code=$error_code" return 2 fi if hosts_file_has_name "$hosts_file" "$target"; then - record "INFO: $description name target is in hosts file; sender_networks=not-checked vector_networks=not-checked shared_networks=not-applicable alias_matches=not-applicable hosts_entry=1 resolver_control=not-needed error_code=$error_code" + record "INFO: $description name target is in hosts file; sender_networks=not-checked vector_networks=not-checked shared_networks=not-applicable alias_matches=not-applicable hosts_entry=1 target_hostname=present resolver_control=not-needed error_code=$error_code" return 2 fi if ! sender_networks_json=$(docker_cmd inspect -f '{{json .NetworkSettings.Networks}}' "$target_container" 2>/dev/null); then - record "INFO: $description name proof failed; sender_networks=unavailable vector_networks=unavailable shared_networks=unknown alias_matches=unknown hosts_entry=0 resolver_control=not-run error_code=$error_code inspect_error=1" + record "INFO: $description name proof failed; sender_networks=unavailable vector_networks=unavailable shared_networks=unknown alias_matches=unknown hosts_entry=0 target_hostname=present resolver_control=not-run error_code=$error_code inspect_error=1" return 2 fi if ! vector_networks_json=$(docker_cmd inspect -f '{{json .NetworkSettings.Networks}}' "$VECTOR_ID" 2>/dev/null); then - record "INFO: $description name proof failed; sender_networks=read vector_networks=unavailable shared_networks=unknown alias_matches=unknown hosts_entry=0 resolver_control=not-run error_code=$error_code inspect_error=1" + record "INFO: $description name proof failed; sender_networks=read vector_networks=unavailable shared_networks=unknown alias_matches=unknown hosts_entry=0 target_hostname=present resolver_control=not-run error_code=$error_code inspect_error=1" return 2 fi if sender_networks=$(name_network_names_from_json <<<"$sender_networks_json"); then @@ -357,7 +375,7 @@ prove_name_target_structure() { else sender_parse_status=$? if (( sender_parse_status == 3 )); then sender_networks=empty; else sender_networks=invalid; fi - record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=read shared_networks=unknown alias_matches=unknown hosts_entry=0 resolver_control=not-run error_code=$error_code inspect_error=1" + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=read shared_networks=unknown alias_matches=unknown hosts_entry=0 target_hostname=present resolver_control=not-run error_code=$error_code inspect_error=1" return 2 fi if vector_networks=$(name_network_names_from_json <<<"$vector_networks_json"); then @@ -365,7 +383,7 @@ prove_name_target_structure() { else vector_parse_status=$? if (( vector_parse_status == 3 )); then vector_networks=empty; else vector_networks=invalid; fi - record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=unknown alias_matches=unknown hosts_entry=0 resolver_control=not-run error_code=$error_code inspect_error=1" + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=unknown alias_matches=unknown hosts_entry=0 target_hostname=present resolver_control=not-run error_code=$error_code inspect_error=1" return 2 fi IFS=, read -r -a sender_network_names <<<"$sender_networks" @@ -378,11 +396,11 @@ prove_name_target_structure() { peer_format='{{printf "{\"name\":%s,\"hostname\":%s,\"service\":%s,\"networks\":%s}" (json .Name) (json .Config.Hostname) (json (index .Config.Labels "com.docker.compose.service")) (json .NetworkSettings.Networks)}}' for network in "${sender_network_names[@]}"; do if ! containers_json=$(docker_cmd network inspect -f '{{json .Containers}}' "$network" 2>/dev/null); then - record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=$alias_matches hosts_entry=0 resolver_control=not-run error_code=$error_code network_inspect_error=1" + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=$alias_matches hosts_entry=0 target_hostname=present resolver_control=not-run error_code=$error_code network_inspect_error=1" return 2 fi if ! peer_ids=$(name_network_container_ids_from_json <<<"$containers_json"); then - record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=$alias_matches hosts_entry=0 resolver_control=not-run error_code=$error_code network_inspect_error=1" + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=$alias_matches hosts_entry=0 target_hostname=present resolver_control=not-run error_code=$error_code network_inspect_error=1" return 2 fi sender_member_found=0 @@ -390,7 +408,7 @@ prove_name_target_structure() { [[ -n "$peer_id" ]] || continue [[ "$peer_id" == "$target_container" ]] && sender_member_found=1 if ! peer_metadata=$(docker_cmd inspect -f "$peer_format" "$peer_id" 2>/dev/null); then - record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=$alias_matches hosts_entry=0 resolver_control=not-run error_code=$error_code inspect_error=1" + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=$alias_matches hosts_entry=0 target_hostname=present resolver_control=not-run error_code=$error_code inspect_error=1" return 2 fi if name_peer_matches_target "$network" "$target" "$peer_metadata"; then @@ -398,25 +416,25 @@ prove_name_target_structure() { else peer_match_status=$? if (( peer_match_status != 1 )); then - record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=unknown hosts_entry=0 resolver_control=not-run error_code=$error_code inspect_error=1" + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=unknown hosts_entry=0 target_hostname=present resolver_control=not-run error_code=$error_code inspect_error=1" return 2 fi fi done <<<"$peer_ids" if (( sender_member_found == 0 )); then - record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=$alias_matches hosts_entry=0 resolver_control=not-run error_code=$error_code network_sender_match=0" + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=$alias_matches hosts_entry=0 target_hostname=present resolver_control=not-run error_code=$error_code network_sender_match=0" return 2 fi done if (( shared_networks != 0 || alias_matches != 0 )); then - record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=$alias_matches hosts_entry=0 resolver_control=not-run error_code=$error_code" + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=$shared_networks alias_matches=$alias_matches hosts_entry=0 target_hostname=present resolver_control=not-run error_code=$error_code" return 2 fi if ! name_resolver_positive_control "$target_container"; then - record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=0 alias_matches=0 hosts_entry=0 resolver_control=failed error_code=$error_code" + record "INFO: $description name proof failed; sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=0 alias_matches=0 hosts_entry=0 target_hostname=present resolver_control=failed error_code=$error_code" return 2 fi - PROBE_NAME_EVIDENCE="sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=0 alias_matches=0 hosts_entry=0 resolver_control=ok error_code=$error_code" + PROBE_NAME_EVIDENCE="sender_networks=$sender_networks vector_networks=$vector_networks shared_networks=0 alias_matches=0 hosts_entry=0 target_hostname=present resolver_control=ok error_code=$error_code" record "INFO: $description name unresolvable ($error_code); structural: $PROBE_NAME_EVIDENCE" } diff --git a/scripts/test-tc-959-isolation.sh b/scripts/test-tc-959-isolation.sh index 588d1194..8c5f2c1b 100755 --- a/scripts/test-tc-959-isolation.sh +++ b/scripts/test-tc-959-isolation.sh @@ -37,10 +37,12 @@ NAME_VECTOR_NETWORKS='{"logship":{}}' NAME_NETWORK_CONTAINERS='{"sender-id":{"Name":"tinycloud"},"peer-id":{"Name":"other"}}' NAME_SENDER_METADATA='{"name":"/tinycloud-container","hostname":"tinycloud-host","service":"tinycloud","networks":{"default":{"Aliases":["tinycloud"],"DNSNames":["tinycloud-container"]}}}' NAME_PEER_METADATA='{"name":"/other","hostname":"other-host","service":"other","networks":{"default":{"Aliases":["other"],"DNSNames":["other"]}}}' -NAME_HOSTS='127.0.0.1 localhost' +NAME_TARGET_HOSTNAME=tinycloud-container +NAME_HOSTS='127.0.0.1 localhost tinycloud-container' NAME_INSPECT_FAIL=0 NAME_NETWORK_INSPECT_FAIL=0 -NAME_HOSTS_FAIL=0 +NAME_HOSTS_RUN_FAIL=0 +NAME_HOST_READ_TRACE_FILE="" NAME_RESOLVER_MODE=ok NAME_RESOLVER_NETWORK_FILE="" docker_cmd() { @@ -53,6 +55,10 @@ docker_cmd() { case "$1" in inspect) if (( NAME_INSPECT_FAIL )); then return 1; fi + if [[ "${3:-}" == '{{.Config.Hostname}}' ]]; then + printf '%s\n' "$NAME_TARGET_HOSTNAME" + return 0 + fi if [[ "${3:-}" == '{{json .NetworkSettings.Networks}}' ]]; then if [[ "${4:-}" == "$NAME_SENDER_ID" ]]; then printf '%s\n' "$NAME_SENDER_NETWORKS" elif [[ "${4:-}" == "$VECTOR_ID" ]]; then printf '%s\n' "$NAME_VECTOR_NETWORKS" @@ -73,17 +79,27 @@ docker_cmd() { return 0 ;; exec) - if (( NAME_HOSTS_FAIL )); then return 1; fi - [[ "${2:-}" == "$NAME_SENDER_ID" && "${3:-}" == cat && "${4:-}" == /etc/hosts ]] || return 1 - printf '%s\n' "$NAME_HOSTS" - return 0 + printf '%s\n' exec >>"$NAME_HOST_READ_TRACE_FILE" + return 1 ;; timeout) + if [[ "${12:-}" == cat && "${13:-}" == /etc/hosts ]]; then + printf 'run:%s\n' "${10#container:}" >>"$NAME_HOST_READ_TRACE_FILE" + if [[ "$1" != timeout || "$2" != 60 || "$3" != run || "$4" != --rm || + "$5" != --label || "$6" != "com.docker.compose.project=${PROJECT:-tc959-isolation-test}" || + "$7" != --label || "$8" != tc959.probe=true || "$9" != --network || + "${10:-}" != "container:$NAME_SENDER_ID" || + "${11:-}" != node:20-alpine@sha256:fb4cd12c85ee03686f6af5362a0b0d56d50c58a04632e6c0fb8363f609372293 ]]; then + return 125 + fi + (( NAME_HOSTS_RUN_FAIL == 0 )) || return 1 + printf '%s\n' "$NAME_HOSTS" + return 0 + fi if [[ -n "$NAME_RESOLVER_NETWORK_FILE" ]]; then for arg in "$@"; do if [[ "$arg" == "container:$NAME_SENDER_ID" ]]; then printf '%s\n' "$arg" >"$NAME_RESOLVER_NETWORK_FILE"; fi done - fi case "$NAME_RESOLVER_MODE" in ok) printf '%s\n' GELF_RESOLVER_OK; return 0 ;; @@ -153,10 +169,12 @@ prepare_name_probe() { NAME_NETWORK_CONTAINERS='{"sender-id":{"Name":"tinycloud"},"peer-id":{"Name":"other"}}' NAME_SENDER_METADATA='{"name":"/tinycloud-container","hostname":"tinycloud-host","service":"tinycloud","networks":{"default":{"Aliases":["tinycloud"],"DNSNames":["tinycloud-container"]}}}' NAME_PEER_METADATA='{"name":"/other","hostname":"other-host","service":"other","networks":{"default":{"Aliases":["other"],"DNSNames":["other"]}}}' - NAME_HOSTS='127.0.0.1 localhost' + NAME_TARGET_HOSTNAME=tinycloud-container + NAME_HOSTS='127.0.0.1 localhost tinycloud-container' NAME_INSPECT_FAIL=0 NAME_NETWORK_INSPECT_FAIL=0 - NAME_HOSTS_FAIL=0 + NAME_HOSTS_RUN_FAIL=0 + NAME_HOST_READ_TRACE_FILE="$CONTROL/host-read-trace" NAME_RESOLVER_MODE=ok NAME_RESOLVER_NETWORK_FILE="$CONTROL/resolver-network" SENDER_EXIT=1 @@ -167,6 +185,7 @@ prepare_name_probe() { SHIPPED_NAME_SAMPLES=('[]' '[]') RECORD_LINES=() : >"$NAME_RESOLVER_NETWORK_FILE" + : >"$NAME_HOST_READ_TRACE_FILE" reset_fixtures } @@ -311,8 +330,9 @@ prepare_name_probe assert_isolated_probe 'vector name with transient DNS errors' container "container:$NAME_SENDER_ID" vector "$NAME_SENDER_ID" assert_eq "$SEND_COUNT" 3 assert_eq "$(<"$NAME_RESOLVER_NETWORK_FILE")" "container:$NAME_SENDER_ID" -[[ "${RECORD_LINES[*]}" == *"name unresolvable (EAI_AGAIN); structural: sender_networks=default vector_networks=logship shared_networks=0 alias_matches=0 hosts_entry=0 resolver_control=ok error_code=EAI_AGAIN"* ]] || fail "name proof did not report safe structural evidence: ${RECORD_LINES[*]}" +[[ "${RECORD_LINES[*]}" == *"name unresolvable (EAI_AGAIN); structural: sender_networks=default vector_networks=logship shared_networks=0 alias_matches=0 hosts_entry=0 target_hostname=present resolver_control=ok error_code=EAI_AGAIN"* ]] || fail "name proof did not report safe structural evidence: ${RECORD_LINES[*]}" [[ "${RECORD_LINES[*]}" == *"fresh source +0 and shipped +0"* ]] || fail "name proof passed without zero traffic deltas" +assert_eq "$(<"$NAME_HOST_READ_TRACE_FILE")" "run:$NAME_SENDER_ID" prepare_name_probe SENDER_OUTPUT='GELF_SEND_ERROR code=ENOTFOUND' @@ -344,15 +364,19 @@ NAME_PEER_METADATA='{"name":"/peer","hostname":"peer-host","service":"peer","net expect_name_proof_failure 'alias_matches=1' prepare_name_probe -NAME_HOSTS_FAIL=1 +NAME_HOSTS_RUN_FAIL=1 expect_name_proof_failure 'hosts_entry=unreadable' prepare_name_probe -NAME_HOSTS='127.0.0.1 localhost host.docker.internal' +NAME_HOSTS='127.0.0.1 localhost' +expect_name_proof_failure 'target_hostname=missing' + +prepare_name_probe +NAME_HOSTS='127.0.0.1 localhost tinycloud-container host.docker.internal' expect_name_proof_failure 'hosts_entry=1' host.docker.internal prepare_name_probe -NAME_HOSTS='127.0.0.1 localhost host.docker.internal' +NAME_HOSTS='127.0.0.1 localhost tinycloud-container host.docker.internal' SENDER_EXIT=0 SENDER_OUTPUT="" METRIC_SAMPLES=('10 61000' '10 62000' '10 63000' '10 64000') diff --git a/tc-959-node-docker-check.review.sh b/tc-959-node-docker-check.review.sh index e7980a7a..8487feb4 100755 --- a/tc-959-node-docker-check.review.sh +++ b/tc-959-node-docker-check.review.sh @@ -348,7 +348,7 @@ const calledHelpers = [ 'send_positive_probe', 'run_probe_sender', 'sender_error_code', 'probe_routing_error_matches_target', 'probe_target_is_name', 'name_network_names_from_json', 'name_network_container_ids_from_json', 'name_peer_matches_target', 'hosts_file_has_name', 'name_resolver_positive_control', - 'prove_name_target_structure', + 'read_target_hosts_file', 'prove_name_target_structure', 'compose_half_cpu_cmd', 'measure_burst', 'run_isolation_leak_self_test', ]; const missing = calledHelpers.filter((name) => !declared.has(name)); From 8a2b2b4c8b6947fba2efadbe4a232782d8016cfa Mon Sep 17 00:00:00 2001 From: Sam Gbafa Date: Thu, 15 Oct 2026 02:47:07 +0000 Subject: [PATCH 21/21] fix(tc-959): mask host interface probe addresses --- scripts/tc-959-isolation.sh | 9 +++++++++ scripts/test-tc-959-isolation.sh | 12 ++++++++++++ tc-959-node-docker-check.review.sh | 8 ++------ 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/scripts/tc-959-isolation.sh b/scripts/tc-959-isolation.sh index e3ae65a0..63bab447 100644 --- a/scripts/tc-959-isolation.sh +++ b/scripts/tc-959-isolation.sh @@ -559,3 +559,12 @@ run_isolation_leak_self_test() { record "PASS: negative isolation check rejected a deliberately allowed host-loopback packet" fi } +probe_host_external_interfaces() { + local host_ips=$1 host_ip interface_index=0 + [[ -n "$host_ips" ]] || fail "no non-loopback IPv4 interface available for external-interface isolation probes" + while read -r host_ip; do + [[ -n "$host_ip" ]] || continue + interface_index=$((interface_index + 1)) + assert_isolated_probe "host external interface iface#$interface_index" host host "$host_ip" + done <<<"$host_ips" +} diff --git a/scripts/test-tc-959-isolation.sh b/scripts/test-tc-959-isolation.sh index 8c5f2c1b..6fa23322 100755 --- a/scripts/test-tc-959-isolation.sh +++ b/scripts/test-tc-959-isolation.sh @@ -600,3 +600,15 @@ SNAPSHOT_STARTED=2026-10-14T10:00:00.999999999+01:30 CURRENT_STARTED=2026-10-14T08:30:01Z assert_crash_recovered printf '%s\n' 'PASS: exact-container crash identity and restart proofs' +HOST_PROBE_REPORT="" +HOST_PROBE_TARGETS=() +record() { HOST_PROBE_REPORT+="$*"$'\n'; } +assert_isolated_probe() { + record "PASS: $1 isolation; fresh source +0 and shipped +0" + HOST_PROBE_TARGETS+=("$4") +} +probe_host_external_interfaces $'198.51.100.22\n203.0.113.17' +assert_eq "${HOST_PROBE_TARGETS[*]}" "198.51.100.22 203.0.113.17" +[[ "$HOST_PROBE_REPORT" == *"iface#1"* && "$HOST_PROBE_REPORT" == *"iface#2"* ]] || fail "host interface report omitted stable labels" +[[ "$HOST_PROBE_REPORT" != *"198.51.100.22"* && "$HOST_PROBE_REPORT" != *"203.0.113.17"* ]] || fail "host interface report exposed a full IPv4 address" +printf '%s\n' 'PASS: host interface reports use iface labels without full IPv4 addresses' diff --git a/tc-959-node-docker-check.review.sh b/tc-959-node-docker-check.review.sh index 8487feb4..68e2dbe4 100755 --- a/tc-959-node-docker-check.review.sh +++ b/tc-959-node-docker-check.review.sh @@ -349,7 +349,7 @@ const calledHelpers = [ 'probe_target_is_name', 'name_network_names_from_json', 'name_network_container_ids_from_json', 'name_peer_matches_target', 'hosts_file_has_name', 'name_resolver_positive_control', 'read_target_hosts_file', 'prove_name_target_structure', - 'compose_half_cpu_cmd', 'measure_burst', 'run_isolation_leak_self_test', + 'compose_half_cpu_cmd', 'measure_burst', 'run_isolation_leak_self_test', 'probe_host_external_interfaces', ]; const missing = calledHelpers.filter((name) => !declared.has(name)); if (missing.length) throw new Error(`harness calls undefined shell functions: ${missing.join(', ')}`); @@ -532,11 +532,7 @@ for target in "${probe_targets[@]}"; do done host_ips=$(ip -o -4 addr show scope global | awk '{split($4,a,"/"); print a[1]}') -[[ -n "$host_ips" ]] || fail "no non-loopback IPv4 interface available for external-interface isolation probes" -while read -r host_ip; do - [[ -n "$host_ip" ]] || continue - assert_isolated_probe "host external interface $host_ip" host host "$host_ip" -done <<<"$host_ips" +probe_host_external_interfaces "$host_ips" assert_no_console_marker "$PROBE_CANARY" || fail "forged probe canary appeared anywhere in Vector console output" record "PASS: raw Vector console output contains no forged-probe canary after the isolation matrix"