From a1fe9ce0609a85303d75c8280e2c0dfcebc29e93 Mon Sep 17 00:00:00 2001 From: Afnan Date: Mon, 21 Sep 2026 21:31:25 +0530 Subject: [PATCH 1/3] Add eval harness (SLA, failover, citation guardrail, vision template); rewrite README Extract SLA tier logic into server/services/sla.js (behaviour unchanged). Co-Authored-By: Claude Sonnet 5 --- eval/README.md | 62 +++++++++++++ eval/fixtures/citations.json | 47 ++++++++++ eval/lib/citations.js | 56 ++++++++++++ eval/lib/metrics.js | 21 +++++ eval/lib/results.js | 11 +++ eval/package.json | 9 ++ eval/results/app_guardrail_passthrough.json | 6 ++ eval/results/citation_validator.json | 10 +++ eval/results/failover.json | 67 ++++++++++++++ eval/results/sla_tiers.json | 5 ++ eval/tests/citations.test.js | 24 +++++ eval/tests/failover.test.js | 99 +++++++++++++++++++++ eval/tests/guardrail_passthrough.test.js | 52 +++++++++++ eval/tests/metrics.test.js | 18 ++++ eval/tests/sla.test.js | 42 +++++++++ eval/vision/labels.template.csv | 7 ++ eval/vision/run.js | 61 +++++++++++++ server/routes/violations.js | 7 +- server/services/sla.js | 14 +++ 19 files changed, 614 insertions(+), 4 deletions(-) create mode 100644 eval/README.md create mode 100644 eval/fixtures/citations.json create mode 100644 eval/lib/citations.js create mode 100644 eval/lib/metrics.js create mode 100644 eval/lib/results.js create mode 100644 eval/package.json create mode 100644 eval/results/app_guardrail_passthrough.json create mode 100644 eval/results/citation_validator.json create mode 100644 eval/results/failover.json create mode 100644 eval/results/sla_tiers.json create mode 100644 eval/tests/citations.test.js create mode 100644 eval/tests/failover.test.js create mode 100644 eval/tests/guardrail_passthrough.test.js create mode 100644 eval/tests/metrics.test.js create mode 100644 eval/tests/sla.test.js create mode 100644 eval/vision/labels.template.csv create mode 100644 eval/vision/run.js create mode 100644 server/services/sla.js diff --git a/eval/README.md b/eval/README.md new file mode 100644 index 0000000..5fe7f5f --- /dev/null +++ b/eval/README.md @@ -0,0 +1,62 @@ +# InfraWatch evaluation harness + +A small, reproducible harness that measures the parts of InfraWatch that can be measured **without paid API calls or labeled imagery**, and provides a ready-to-run harness for the part that cannot (vision accuracy). + +## Run + +```bash +cd server && npm install && cd .. # the failover test imports the real server code +cd eval && npm test # or from repo root: node --test "eval/tests/*.test.js" +``` + +No API keys, network, or database are needed. Each test writes its numbers to `eval/results/*.json`. Last run: 14 tests, 14 pass (Node 22). + +## What is measured, and what it means + +### 1. SLA tier routing (`tests/sla.test.js`, real code) +The confidence-to-SLA mapping used by the crisis feed (`server/routes/violations.js`) was extracted verbatim into `server/services/sla.js` (behaviour-preserving) so it can be tested. + +| Check | Result | +|---|---| +| 9 boundary confidences (100, 95, 90, 89.99, 85, 80, 79.99, 50, 0) map to 4h / 12h / 24h as documented | 9/9 | +| Sweep 0-100 in steps of 0.5 (201 points): higher confidence never gets a longer SLA | pass | +| Status thresholds: ON_TIME up to 80% of SLA, AT_RISK up to 100%, BREACHED beyond | pass | + +Observed edge behaviour (documented, not judged): `null`, `undefined`, `NaN`, and a 0-1 fraction such as `0.95` all fall into the 24h tier, because the 0-100 scale is implicit. Also note the crisis feed's candidate filter uses `confidence >= 85` while the 4h tier starts at 90 (`violations.js`), so 85-89 cases are listed but get the 12h SLA. + +### 2. AI provider failover (`tests/failover.test.js`, real code, stubbed network) +Runs the real dispatcher in `server/services/visionAI.js` with dummy keys and a stubbed `fetch` (the Gemini SDK and the Groq call both go through `fetch`). 7 scenarios: + +| Scenario | Outcome observed | +|---|---| +| Gemini healthy | Gemini answers, Groq called 0 times | +| Gemini 429 | falls over, Groq answers (1 call) | +| Gemini 503 | falls over, Groq answers | +| Gemini hangs past timeout (300 ms in test; 30 s default) | falls over, Groq answers | +| Gemini 400 (non-retryable) | no failover, error surfaced | +| Both fail with 429 | error `Both AI providers unavailable...` | +| 429 with no Groq key configured | no failover, error surfaced | + +Finding: after a failover, the parsed result's `_provider`/`_model` metadata still reports `gemini` (it is read from config in `parseVisionResponse`, not from the provider that actually answered). The dispatcher itself returns the correct provider; the metadata attached to the parsed payload does not. Scope: only the `callAI` path in `visionAI.js` fails over. `generateAINotice` in `services/gemini.js` (the `/api/notices/generate-ai` route) uses a single provider chosen by `AI_PROVIDER` (default `groq` in that file) with no failover. + +### 3. Citation guardrail (`tests/citations.test.js`, `tests/guardrail_passthrough.test.js`) +Two different questions, deliberately separated: + +**a) Does the app catch fabricated citations in code?** The guardrail today is a **prompt instruction only** (`visionAI.js` `generateLegalNoticeFromAnalysis`, `gemini.js` `buildPrompt`). There is no server-side validation of the model output. With a stubbed provider returning notices that cite fabricated provisions, the real function returned the fabricated text unchanged in **20 of 20** cases. App-side catch rate: **0%**. The test asserts this observed behaviour; if validation is added, it will fail and should be updated. + +**b) What would a deterministic post-check against the same allowlist catch?** `lib/citations.js` is a reference validator (not part of the app). Allowlist = the sections named in the app's own prompt (KMC Act 1976 s.308/321/321A/322; Bye-laws 2003 2.2/2.5/3.0/4.6; KTCP Act 1961 s.14/15/76-FF). On 20 hand-written fabricated snippets (non-existent sections, wrong-Act sections, IPC/Constitution/case-law citations, a fake statute, a real-looking fallback phrase followed by a fabricated section) and 20 valid snippets: **20/20 caught (100%), 0/20 false positives**. + +Caveats: the fixtures are synthetic and written by the same author as the validator, so 100% is a sanity check, not a generalisation claim. The allowlist comes from the app's prompt and has **not** been verified against the statutes. Neither result says anything about how often a real LLM hallucinates a citation; that needs a live run (see below). + +### 4. Vision detection accuracy: PENDING +Not measured. It needs human-labeled satellite tiles and a Gemini or Groq key. No accuracy numbers are reported anywhere in this repo. + +To run once you have data: +1. Copy `vision/labels.template.csv` to `vision/labels.csv`; add images under `vision/images/`; set `has_violation` to 1/0 from human ground truth (aim for a balanced set, e.g. 50/50). +2. `export GEMINI_API_KEY=...` (optionally `GROQ_API_KEY`), then `node eval/vision/run.js --limit 100`. +3. Output `eval/results/vision.json`: confusion matrix, precision, recall, F1, false-positive rate, per-image predictions. + +The harness calls `analyzeSingleImage` directly, bypassing the seeded demo detections in `routes/vision.js`. Expect a false-positive bias worth measuring: that prompt tells the model to flag at least one suspicious area whenever buildings are visible. The metric code itself is unit-tested on a toy example (`tests/metrics.test.js`), which is not a detection result. + +## Not covered +Auth/RBAC, notice generation quality, penalty formula correctness, officer assignment, and the client. The README's "round-robin" officer assignment claim could not be found in server code (see the main README). diff --git a/eval/fixtures/citations.json b/eval/fixtures/citations.json new file mode 100644 index 0000000..f7a7d23 --- /dev/null +++ b/eval/fixtures/citations.json @@ -0,0 +1,47 @@ +{ + "note": "Hand-written synthetic notice snippets. 'valid' = cites only allowlisted provisions (the allowlist is the app's own prompt list, not independently verified against the Acts). 'fabricated' = cites a provision outside the allowlist or an outside authority. Fixtures test the checker, NOT the LLM.", + "valid": [ + "Karnataka Municipal Corporations Act, 1976 - Section 321", + "Karnataka Municipal Corporations Act, 1976 - Section 308", + "Karnataka Municipal Corporations Act, 1976 - Section 322", + "Karnataka Municipal Corporations Act, 1976 - Section 321A", + "BBMP Building Bye-laws, 2003 - Bye-law 2.2", + "BBMP Building Bye-laws, 2003 - Bye-law 2.5", + "BBMP Building Bye-laws, 2003 - Bye-law 3.0", + "BBMP Building Bye-laws, 2003 - Bye-law 4.6", + "Karnataka Town and Country Planning Act, 1961 - Section 14", + "Karnataka Town and Country Planning Act, 1961 - Section 15", + "Karnataka Town and Country Planning Act, 1961 - Section 76-FF", + "LEGAL PROVISIONS CITED: Section 308 of the Karnataka Municipal Corporations Act, 1976; Bye-law 2.5 of the BBMP Building Bye-laws, 2003", + "CONSEQUENCES OF NON-COMPLIANCE: demolition under Sec 321 of the BBMP Act; prosecution under Section 76-FF of the Karnataka Town and Country Planning Act, 1961", + "You are directed to comply. Refer to applicable BBMP Building Bye-laws, 2003", + "LEGAL PROVISIONS CITED: Refer to applicable BBMP Building Bye-laws, 2003", + "NATURE OF VIOLATION: A fourth floor was added without sanction. No statute is cited in this paragraph.", + "Stop-work notice under Sec. 322 of the Karnataka Municipal Corporations Act, 1976", + "Penalty under BBMP Act Section 321", + "Compounding under Karnataka Municipal Corporations Act, 1976 Section 321A where permissible", + "Setback per BBMP Building Bye-laws 2003, Bye-law 2.2; FAR per Bye-law 2.5" + ], + "fabricated": [ + "Karnataka Municipal Corporations Act, 1976 - Section 999", + "Karnataka Municipal Corporations Act, 1976 - Section 412", + "Karnataka Municipal Corporations Act, 1976 - Section 320", + "Karnataka Municipal Corporations Act, 1976 - Section 323", + "Karnataka Municipal Corporations Act, 1976 - Section 308B", + "Karnataka Municipal Corporations Act, 1976 - Section 76-FF", + "BBMP Building Bye-laws, 2003 - Bye-law 7.9", + "BBMP Building Bye-laws, 2003 - Bye-law 2.3", + "BBMP Building Bye-laws, 2003 - Bye-law 11.4", + "Karnataka Town and Country Planning Act, 1961 - Section 99", + "Karnataka Town and Country Planning Act, 1961 - Section 76-FG", + "Karnataka Town and Country Planning Act, 1961 - Section 321", + "LEGAL PROVISIONS CITED: Section 505 of the Karnataka Municipal Corporations Act, 1976; Bye-law 2.5 of the BBMP Building Bye-laws, 2003", + "CONSEQUENCES OF NON-COMPLIANCE: demolition under Sec 321 of the BBMP Act; prosecution under Section 188 of the Indian Penal Code", + "Penalty under Section 44 of the Karnataka Building Regulation Act", + "As held by the Supreme Court in State of Karnataka v. Sharma, AIR 2011 SC 4021, demolition is mandatory", + "Article 300A of the Constitution of India entitles the corporation to act", + "Stop-work notice under Sec. 322D of the Karnataka Municipal Corporations Act, 1976", + "Refer to applicable BBMP Building Bye-laws, 2003; also Section 512 of the KMC Act, 1976", + "Under Section 17 of the Karnataka Municipal Corporations Act, 1976 the owner must pay a fine of Rs 5 lakh" + ] +} diff --git a/eval/lib/citations.js b/eval/lib/citations.js new file mode 100644 index 0000000..95849e7 --- /dev/null +++ b/eval/lib/citations.js @@ -0,0 +1,56 @@ +/** + * Reference citation validator used by the eval. + * + * IMPORTANT: this is NOT part of the InfraWatch runtime. The app's only + * anti-hallucination guard today is a prompt instruction (see + * server/services/visionAI.js, generateLegalNoticeFromAnalysis, and + * server/services/gemini.js buildPrompt). This module shows what a + * deterministic post-generation check against the SAME allowlist would catch, + * and the eval measures it. The allowlist is copied from the app's prompt; it + * has NOT been verified against the statutes themselves. + */ +export const ALLOWED = { + KMC: new Set(['308', '321', '321A', '322']), // Karnataka Municipal Corporations Act, 1976 + BYELAW: new Set(['2.2', '2.5', '3.0', '4.6']), // BBMP Building Bye-laws, 2003 + KTCP: new Set(['14', '15', '76-FF']), // Karnataka Town and Country Planning Act, 1961 +}; + +const FOREIGN = /\b(IPC|Indian Penal Code|Penal Code|Constitution|Supreme Court|High Court|AIR\s?\d|SCC|Companies Act|Consumer Protection|Environment Protection)\b|\bv\.\s/i; +const FALLBACK = /Refer to applicable BBMP Building Bye-?laws,? 2003/i; + +function actOf(clause) { + if (/Municipal Corporations? Act|\bKMC\b|BBMP Act/i.test(clause)) return 'KMC'; + if (/Bye-?laws?/i.test(clause)) return 'BYELAW'; + if (/Town and Country Planning|\bKTCP\b|\bTCP Act\b/i.test(clause)) return 'KTCP'; + return null; +} + +/** Returns { ok, violations: [...], citations: n } for a notice text (or array of cite strings). */ +export function validateCitations(input) { + const text = Array.isArray(input) ? input.join('\n') : String(input); + // Split into clauses; also split after "prosecution under"/"; also" style joins by sentence-ish breaks. + const clauses = text.split(/[\n;]+|,\s*(?=prosecution|also|and)/i); + const violations = []; + let citations = 0; + for (const clause of clauses) { + const refs = [...clause.matchAll(/(?:Section|Sec\.?|§|Bye-?law|Article)\s*(\d+(?:\.\d+)?(?:-[A-Z]{1,2}|[A-Z])?)/gi)]; + if (refs.length === 0 && !FOREIGN.test(clause)) continue; + if (FOREIGN.test(clause)) { + citations++; + violations.push({ clause: clause.trim(), reason: 'authority outside allowlist' }); + continue; + } + const act = actOf(clause); + for (const m of refs) { + citations++; + const id = m[1].toUpperCase(); + const key = /Bye-?law/i.test(m[0]) ? 'BYELAW' : act; + if (!key) { + violations.push({ clause: clause.trim(), reason: `section ${id} with no identifiable Act` }); + continue; + } + if (!ALLOWED[key].has(id)) violations.push({ clause: clause.trim(), reason: `${key} ${id} not in allowlist` }); + } + } + return { ok: violations.length === 0, violations, citations, usedFallbackPhrase: FALLBACK.test(text) }; +} diff --git a/eval/lib/metrics.js b/eval/lib/metrics.js new file mode 100644 index 0000000..01930c9 --- /dev/null +++ b/eval/lib/metrics.js @@ -0,0 +1,21 @@ +/** Binary detection metrics. rows: [{ truth: boolean, pred: boolean }] */ +export function confusion(rows) { + const c = { tp: 0, fp: 0, tn: 0, fn: 0 }; + for (const { truth, pred } of rows) { + if (truth && pred) c.tp++; + else if (!truth && pred) c.fp++; + else if (!truth && !pred) c.tn++; + else c.fn++; + } + return c; +} + +const div = (a, b) => (b === 0 ? null : a / b); + +export function summarize(rows) { + const c = confusion(rows); + const precision = div(c.tp, c.tp + c.fp); + const recall = div(c.tp, c.tp + c.fn); + const f1 = precision === null || recall === null || precision + recall === 0 ? null : (2 * precision * recall) / (precision + recall); + return { n: rows.length, ...c, precision, recall, f1, false_positive_rate: div(c.fp, c.fp + c.tn), accuracy: div(c.tp + c.tn, rows.length) }; +} diff --git a/eval/lib/results.js b/eval/lib/results.js new file mode 100644 index 0000000..78320d1 --- /dev/null +++ b/eval/lib/results.js @@ -0,0 +1,11 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const dir = path.join(path.dirname(fileURLToPath(import.meta.url)), '..', 'results'); + +export function record(name, data) { + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, `${name}.json`), JSON.stringify(data, null, 2) + '\n'); + console.log(`# ${name}: ${JSON.stringify(data)}`); +} diff --git a/eval/package.json b/eval/package.json new file mode 100644 index 0000000..8729fd3 --- /dev/null +++ b/eval/package.json @@ -0,0 +1,9 @@ +{ + "name": "infrawatch-eval", + "private": true, + "type": "module", + "scripts": { + "test": "node --test \"tests/*.test.js\"", + "vision": "node vision/run.js" + } +} diff --git a/eval/results/app_guardrail_passthrough.json b/eval/results/app_guardrail_passthrough.json new file mode 100644 index 0000000..6838422 --- /dev/null +++ b/eval/results/app_guardrail_passthrough.json @@ -0,0 +1,6 @@ +{ + "fabricated_n": 20, + "app_returned_fabricated_text_unchanged": 20, + "app_side_catch_rate": 0, + "would_be_flagged_by_reference_validator": 20 +} diff --git a/eval/results/citation_validator.json b/eval/results/citation_validator.json new file mode 100644 index 0000000..31a77d5 --- /dev/null +++ b/eval/results/citation_validator.json @@ -0,0 +1,10 @@ +{ + "fabricated_n": 20, + "caught": 20, + "catch_rate": 1, + "valid_n": 20, + "false_positives": 0, + "false_positive_rate": 0, + "missed": [], + "false_positive_examples": [] +} diff --git a/eval/results/failover.json b/eval/results/failover.json new file mode 100644 index 0000000..eb98fb7 --- /dev/null +++ b/eval/results/failover.json @@ -0,0 +1,67 @@ +{ + "scenarios": [ + { + "name": "primary_ok", + "ok": true, + "provider_reported": "gemini", + "calls": { + "gemini": 1, + "groq": 0 + } + }, + { + "name": "gemini_429", + "ok": true, + "provider_reported": "gemini", + "calls": { + "gemini": 1, + "groq": 1 + } + }, + { + "name": "gemini_503", + "ok": true, + "provider_reported": "gemini", + "calls": { + "gemini": 1, + "groq": 1 + } + }, + { + "name": "gemini_timeout", + "ok": true, + "provider_reported": "gemini", + "calls": { + "gemini": 1, + "groq": 1 + } + }, + { + "name": "gemini_400", + "ok": false, + "error": "[GoogleGenerativeAI Error]: Error fetching from https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent: [400 ] HTTP 400", + "calls": { + "gemini": 1, + "groq": 0 + } + }, + { + "name": "both_fail", + "ok": false, + "error": "Both AI providers unavailable. gemini: [GoogleGenerativeAI Error]: Error fetching from https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:", + "calls": { + "gemini": 1, + "groq": 1 + } + }, + { + "name": "no_secondary_key", + "ok": false, + "error": "[GoogleGenerativeAI Error]: Error fetching from https://generativelanguage.googleapis.com/v1beta/models/gemini-2.5-flash:generateContent: [429 ] HTTP 429", + "calls": { + "gemini": 1, + "groq": 0 + } + } + ] +} diff --git a/eval/results/sla_tiers.json b/eval/results/sla_tiers.json new file mode 100644 index 0000000..b7ddf73 --- /dev/null +++ b/eval/results/sla_tiers.json @@ -0,0 +1,5 @@ +{ + "boundary_cases": 9, + "sweep_points": 201, + "all_pass": true +} diff --git a/eval/tests/citations.test.js b/eval/tests/citations.test.js new file mode 100644 index 0000000..fd46299 --- /dev/null +++ b/eval/tests/citations.test.js @@ -0,0 +1,24 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { validateCitations } from '../lib/citations.js'; +import { record } from '../lib/results.js'; + +const fx = JSON.parse(fs.readFileSync(new URL('../fixtures/citations.json', import.meta.url))); + +test('reference validator: catch rate on fabricated, false-positive rate on valid', () => { + const missed = fx.fabricated.filter((t) => validateCitations(t).ok); + const falsePos = fx.valid.filter((t) => !validateCitations(t).ok); + record('citation_validator', { + fabricated_n: fx.fabricated.length, + caught: fx.fabricated.length - missed.length, + catch_rate: (fx.fabricated.length - missed.length) / fx.fabricated.length, + valid_n: fx.valid.length, + false_positives: falsePos.length, + false_positive_rate: falsePos.length / fx.valid.length, + missed, + false_positive_examples: falsePos, + }); + assert.equal(missed.length, 0, `missed: ${JSON.stringify(missed)}`); + assert.equal(falsePos.length, 0, `false positives: ${JSON.stringify(falsePos)}`); +}); diff --git a/eval/tests/failover.test.js b/eval/tests/failover.test.js new file mode 100644 index 0000000..cb6168d --- /dev/null +++ b/eval/tests/failover.test.js @@ -0,0 +1,99 @@ +/** + * Failover eval: runs the REAL dispatcher (server/services/visionAI.js callAI, reached via the + * exported analyzeSingleImage) with dummy API keys and a stubbed global fetch. No network, no cost. + * The stub stands in for both the Gemini SDK (which uses fetch) and Groq (raw fetch). + */ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { record } from '../lib/results.js'; + +process.env.AI_TIMEOUT_MS = '300'; +process.env.GEMINI_API_KEY = 'dummy-gemini'; +process.env.GROQ_API_KEY = 'dummy-groq'; +process.env.AI_PROVIDER = 'gemini'; +const { analyzeSingleImage } = await import('../../server/services/visionAI.js'); + +const GOOD = JSON.stringify({ suspiciousAreasDetected: true, totalAreas: 1, detections: [], overallConfidence: 80 }); +const realFetch = globalThis.fetch; +let calls; + +function stub({ gemini, groq }) { + calls = { gemini: 0, groq: 0 }; + globalThis.fetch = async (url) => { + const u = String(url); + const which = u.includes('generativelanguage') ? 'gemini' : u.includes('groq.com') ? 'groq' : null; + if (!which) throw new Error('unexpected fetch ' + u); + calls[which]++; + const beh = which === 'gemini' ? gemini : groq; + if (beh === 'hang') return new Promise(() => {}); + if (typeof beh === 'number' && beh !== 200) { + return new Response(JSON.stringify({ error: { message: `HTTP ${beh}` } }), { status: beh }); + } + const body = which === 'gemini' + ? { candidates: [{ content: { parts: [{ text: GOOD }], role: 'model' }, finishReason: 'STOP' }] } + : { choices: [{ message: { content: GOOD } }] }; + return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } }); + }; +} + +const scenarios = []; +async function scenario(name, cfg) { + stub(cfg); + let outcome; + try { + const r = await analyzeSingleImage({ image: null, locationContext: {} }); + outcome = { ok: true, provider_reported: r._provider, calls: { ...calls } }; + } catch (e) { + outcome = { ok: false, error: String(e.message).slice(0, 160), calls: { ...calls } }; + } + scenarios.push({ name, ...outcome }); + return outcome; +} + +test.after(() => { + globalThis.fetch = realFetch; + record('failover', { scenarios }); +}); + +test('primary healthy: Gemini answers, Groq never called', async () => { + const o = await scenario('primary_ok', { gemini: 200, groq: 200 }); + assert.ok(o.ok, JSON.stringify(o)); + assert.equal(o.calls.groq, 0); + assert.equal(o.calls.gemini, 1); +}); +test('Gemini 429 -> Groq answers', async () => { + const o = await scenario('gemini_429', { gemini: 429, groq: 200 }); + assert.ok(o.ok, JSON.stringify(o)); + assert.equal(o.calls.groq, 1); +}); +test('Gemini 503 -> Groq answers', async () => { + const o = await scenario('gemini_503', { gemini: 503, groq: 200 }); + assert.ok(o.ok, JSON.stringify(o)); + assert.equal(o.calls.groq, 1); +}); +test('Gemini hangs past timeout -> Groq answers', async () => { + const o = await scenario('gemini_timeout', { gemini: 'hang', groq: 200 }); + assert.ok(o.ok, JSON.stringify(o)); + assert.equal(o.calls.groq, 1); +}); +test('Gemini 400 (non-retryable): no failover, error surfaces', async () => { + const o = await scenario('gemini_400', { gemini: 400, groq: 200 }); + assert.equal(o.ok, false); + assert.equal(o.calls.groq, 0); +}); +test('both providers fail: combined error', async () => { + const o = await scenario('both_fail', { gemini: 429, groq: 429 }); + assert.equal(o.ok, false); + assert.match(o.error, /Both AI providers unavailable/); +}); +test('no Groq key: no failover even on 429', async () => { + const k = process.env.GROQ_API_KEY; + delete process.env.GROQ_API_KEY; + try { + const o = await scenario('no_secondary_key', { gemini: 429, groq: 200 }); + assert.equal(o.ok, false); + assert.equal(o.calls.groq, 0); + } finally { + process.env.GROQ_API_KEY = k; + } +}); diff --git a/eval/tests/guardrail_passthrough.test.js b/eval/tests/guardrail_passthrough.test.js new file mode 100644 index 0000000..23e6252 --- /dev/null +++ b/eval/tests/guardrail_passthrough.test.js @@ -0,0 +1,52 @@ +/** + * Measures what the APP ITSELF does (not the reference validator) when the model returns a + * notice with a fabricated citation: runs the real generateLegalNoticeFromAnalysis with a + * stubbed provider. The app's guardrail is a prompt instruction only, so we expect the + * fabricated text to pass through untouched. This test asserts that observed behaviour so the + * result is reproducible; if someone adds server-side validation, this test will fail and + * should be updated (that would be an improvement). + */ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import { record } from '../lib/results.js'; +import { validateCitations } from '../lib/citations.js'; + +process.env.AI_PROVIDER = 'groq'; +process.env.GROQ_API_KEY = 'dummy-groq'; +delete process.env.GEMINI_API_KEY; +const { generateLegalNoticeFromAnalysis } = await import('../../server/services/visionAI.js'); + +const fx = JSON.parse(fs.readFileSync(new URL('../fixtures/citations.json', import.meta.url))); +const realFetch = globalThis.fetch; +let current; +globalThis.fetch = async () => new Response(JSON.stringify({ + choices: [{ message: { content: JSON.stringify({ + noticeType: 'Show Cause', + noticeContent: `LEGAL PROVISIONS CITED: ${current}`, + legalSectionsCited: [current], + }) } }], +}), { status: 200 }); + +test.after(() => { globalThis.fetch = realFetch; }); + +test('app passes fabricated citations through unchanged (prompt-only guardrail)', async () => { + const violation = { id: '#IW-9999', owner_name: 'Test', address: 'x', ward: 'Ward 1' }; + const analysis = { primaryViolationType: 'Unauthorized Floor Addition', overallConfidence: 90, riskLevel: 'HIGH', totalEstimatedAreaSqFt: 100, visualEvidence: [] }; + const penalty = { calculatedPenaltyINR: 1, calculatedPenaltyLakhs: 1, legalBasis: 'x', formula: 'x' }; + let passedThrough = 0; + let flaggedByRef = 0; + for (const fab of fx.fabricated) { + current = fab; + const out = await generateLegalNoticeFromAnalysis({ violation, analysisResult: analysis, penaltyCalculation: penalty }); + if (out.noticeContent.includes(fab)) passedThrough++; + if (!validateCitations(out.noticeContent).ok) flaggedByRef++; + } + record('app_guardrail_passthrough', { + fabricated_n: fx.fabricated.length, + app_returned_fabricated_text_unchanged: passedThrough, + app_side_catch_rate: 1 - passedThrough / fx.fabricated.length, + would_be_flagged_by_reference_validator: flaggedByRef, + }); + assert.equal(passedThrough, fx.fabricated.length); +}); diff --git a/eval/tests/metrics.test.js b/eval/tests/metrics.test.js new file mode 100644 index 0000000..c4433bb --- /dev/null +++ b/eval/tests/metrics.test.js @@ -0,0 +1,18 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { summarize } from '../lib/metrics.js'; + +// Correctness check of the scoring code on a hand-computed toy example. +// These are NOT detection results for InfraWatch. +test('metrics arithmetic on a toy example', () => { + const rows = [ + { truth: true, pred: true }, { truth: true, pred: true }, { truth: true, pred: false }, + { truth: false, pred: true }, { truth: false, pred: false }, { truth: false, pred: false }, + ]; + const s = summarize(rows); + assert.deepEqual([s.tp, s.fp, s.tn, s.fn], [2, 1, 2, 1]); + assert.equal(s.precision, 2 / 3); + assert.equal(s.recall, 2 / 3); + assert.equal(s.false_positive_rate, 1 / 3); + assert.equal(summarize([]).precision, null); +}); diff --git a/eval/tests/sla.test.js b/eval/tests/sla.test.js new file mode 100644 index 0000000..917d90c --- /dev/null +++ b/eval/tests/sla.test.js @@ -0,0 +1,42 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { getSlaHours, getSlaStatus } from '../../server/services/sla.js'; +import { record } from '../lib/results.js'; + +const tierCases = [ + [100, 4], [95, 4], [90, 4], [89.99, 12], [85, 12], [80, 12], [79.99, 24], [50, 24], [0, 24], +]; + +test('SLA tier by confidence (0-100 scale) at boundaries', () => { + for (const [c, h] of tierCases) assert.equal(getSlaHours(c), h, `confidence ${c}`); +}); + +test('SLA tier sweep 0..100 step 0.5 is monotone non-increasing', () => { + let prev = Infinity; + let n = 0; + for (let c = 0; c <= 100; c += 0.5) { + const h = getSlaHours(c); + assert.ok(h <= prev || c === 0, `non-monotone at ${c}`); + if (c === 0) prev = h; else prev = Math.min(prev, h); + // higher confidence must never get a longer SLA than lower confidence + if (c > 0) assert.ok(h <= getSlaHours(c - 0.5)); + n++; + } + record('sla_tiers', { boundary_cases: tierCases.length, sweep_points: n, all_pass: true }); +}); + +test('SLA status thresholds', () => { + assert.equal(getSlaStatus(0, 4), 'ON_TIME'); + assert.equal(getSlaStatus(3.2, 4), 'ON_TIME'); // 80% exactly is not yet at risk + assert.equal(getSlaStatus(3.21, 4), 'AT_RISK'); + assert.equal(getSlaStatus(4, 4), 'AT_RISK'); // exactly at SLA is not yet breached + assert.equal(getSlaStatus(4.01, 4), 'BREACHED'); +}); + +test('documents behaviour on bad input: null/undefined/NaN/fractional confidence gets the 24h tier', () => { + assert.equal(getSlaHours(null), 24); + assert.equal(getSlaHours(undefined), 24); + assert.equal(getSlaHours(NaN), 24); + // A 0-1 fraction (e.g. 0.95) is silently treated as low confidence: the 0-100 scale is implicit. + assert.equal(getSlaHours(0.95), 24); +}); diff --git a/eval/vision/labels.template.csv b/eval/vision/labels.template.csv new file mode 100644 index 0000000..a7571f7 --- /dev/null +++ b/eval/vision/labels.template.csv @@ -0,0 +1,7 @@ +image,has_violation,violation_type,labeler,notes +# has_violation: 1 = image contains an unauthorized construction, 0 = clean. Ground truth must come from +# a human (ideally checked against sanctioned plans / field inspection), not from the model. +# Aim for a balanced set (e.g. 50 positive / 50 negative) across different wards and zoom levels. +# Delete these comment lines' examples and replace with real rows, e.g.: +# ward12_tile_001.png,1,Unauthorized Floor Addition,alice,slab visible on 4th floor +# ward12_tile_002.png,0,,alice,compliant apartment block diff --git a/eval/vision/run.js b/eval/vision/run.js new file mode 100644 index 0000000..82b807b --- /dev/null +++ b/eval/vision/run.js @@ -0,0 +1,61 @@ +/** + * Vision detection eval (REQUIRES labeled imagery + a real API key; makes paid/quota-limited calls). + * + * 1. Put images in eval/vision/images/ and describe them in eval/vision/labels.csv + * (see labels.template.csv). + * 2. export GEMINI_API_KEY=... (and optionally GROQ_API_KEY=...) ; AI_PROVIDER=gemini + * 3. node eval/vision/run.js [--limit N] + * + * It calls the same analyzeSingleImage() the app uses (bypassing the seeded-demo shortcut in + * routes/vision.js) and scores the binary decision `suspiciousAreasDetected` against your label. + * Output: eval/results/vision.json. If no labels/keys are present it exits without writing numbers. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { summarize } from '../lib/metrics.js'; +import { record } from '../lib/results.js'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const labelsPath = path.join(here, 'labels.csv'); + +if (!fs.existsSync(labelsPath)) { + console.log('No eval/vision/labels.csv found. Copy labels.template.csv, add labeled images. Results: PENDING (no data).'); + process.exit(0); +} +if (!process.env.GEMINI_API_KEY && !process.env.GROQ_API_KEY) { + console.log('No GEMINI_API_KEY / GROQ_API_KEY set. Results: PENDING (no key).'); + process.exit(0); +} + +const limitIdx = process.argv.indexOf('--limit'); +const limit = limitIdx > -1 ? Number(process.argv[limitIdx + 1]) : Infinity; +const { analyzeSingleImage } = await import('../../server/services/visionAI.js'); + +const lines = fs.readFileSync(labelsPath, 'utf8').split(/\r?\n/).filter((l) => l.trim() && !l.startsWith('#')); +const header = lines.shift().split(',').map((s) => s.trim()); +const rows = lines.map((l) => Object.fromEntries(l.split(',').map((v, i) => [header[i], v.trim()]))).slice(0, limit); + +const mime = { '.png': 'image/png', '.jpg': 'image/jpeg', '.jpeg': 'image/jpeg', '.webp': 'image/webp' }; +const scored = []; +const errors = []; +for (const r of rows) { + const file = path.join(here, 'images', r.image); + try { + const dataUrl = `data:${mime[path.extname(file).toLowerCase()] || 'image/png'};base64,${fs.readFileSync(file).toString('base64')}`; + const out = await analyzeSingleImage({ image: { dataUrl }, locationContext: { address: r.image, zone: 'Residential' } }); + scored.push({ image: r.image, truth: r.has_violation === '1', pred: Boolean(out.suspiciousAreasDetected), confidence: out.overallConfidence ?? null }); + } catch (e) { + errors.push({ image: r.image, error: String(e.message).slice(0, 200) }); + } +} + +record('vision', { + status: 'measured', + provider: process.env.AI_PROVIDER || 'gemini', + model: process.env.GEMINI_MODEL || null, + run_at: new Date().toISOString(), + metrics: summarize(scored), + errors, + per_image: scored, +}); diff --git a/server/routes/violations.js b/server/routes/violations.js index cdabec5..06c0590 100644 --- a/server/routes/violations.js +++ b/server/routes/violations.js @@ -2,6 +2,7 @@ import { Router } from 'express'; import { getDb } from '../db/connection.js'; import { authenticate, requireRole } from '../middleware/auth.js'; import { buildWardScope, canAccessWard } from '../middleware/access.js'; +import { getSlaHours, getSlaStatus } from '../services/sla.js'; import { analyzeSatelliteImagery, generateCaseReviewDossier } from '../services/gemini.js'; const router = Router(); @@ -275,10 +276,8 @@ router.get('/crisis-feed', (req, res) => { const enrichedCases = criticalCases.map((c) => { const detectedAt = new Date(c.created_at + 'Z'); // SQLite UTC const ageHours = (Date.now() - detectedAt.getTime()) / 3600000; - const slaHours = c.confidence >= 90 ? 4 : c.confidence >= 80 ? 12 : 24; - const slaStatus = ageHours > slaHours ? 'BREACHED' - : ageHours > slaHours * 0.8 ? 'AT_RISK' - : 'ON_TIME'; + const slaHours = getSlaHours(c.confidence); + const slaStatus = getSlaStatus(ageHours, slaHours); return { ...c, ageHours: Math.round(ageHours * 10) / 10, diff --git a/server/services/sla.js b/server/services/sla.js new file mode 100644 index 0000000..f3b59a8 --- /dev/null +++ b/server/services/sla.js @@ -0,0 +1,14 @@ +/** + * SLA tier + status helpers (extracted verbatim from routes/violations.js so + * the logic can be unit-tested; behaviour is unchanged). + * Confidence is on a 0-100 scale. + */ +export function getSlaHours(confidence) { + return confidence >= 90 ? 4 : confidence >= 80 ? 12 : 24; +} + +export function getSlaStatus(ageHours, slaHours) { + return ageHours > slaHours ? 'BREACHED' + : ageHours > slaHours * 0.8 ? 'AT_RISK' + : 'ON_TIME'; +} From 0adff00b159468a3e6e1aed55dac07ab36198ef2 Mon Sep 17 00:00:00 2001 From: Afnan Date: Mon, 21 Sep 2026 21:31:59 +0530 Subject: [PATCH 2/3] Rewrite README with truthful architecture, design decisions, contributions Co-Authored-By: Claude Sonnet 5 --- README.md | 322 +++++++++++++++--------------------------------------- 1 file changed, 90 insertions(+), 232 deletions(-) diff --git a/README.md b/README.md index e5b3e14..edde601 100644 --- a/README.md +++ b/README.md @@ -1,277 +1,135 @@ -InfraWatch +
-# InfraWatch -
+InfraWatch banner -**Satellite-based detection of illegal construction, with an SLA-tracked enforcement workflow, built for Indian municipal corporations.** +# InfraWatch -[Live Demo](https://infrawatch-sepia.vercel.app) · [Citizen Report Portal](https://infrawatch-sepia.vercel.app/?report=1) · [Backend Health](https://infrawatch-backend-odou.onrender.com/health) · [Report a Bug](https://github.com/LordDevdeep/Infrawatch/issues) +**Satellite imagery in, SLA-tracked enforcement notice out.** +A vision-AI pipeline for flagging illegal construction in Indian municipal corporations, with confidence-based SLA routing, multi-provider AI failover, and a legal-notice generator constrained to a fixed statute list. -MIT License · React 18 + Express · Gemini 2.5 Flash · Built for Google Solution Challenge 2026 +[![Live demo](https://img.shields.io/badge/live_demo-infrawatch--sepia.vercel.app-0b0f14?style=for-the-badge&logo=vercel&logoColor=white)](https://infrawatch-sepia.vercel.app) +![React](https://img.shields.io/badge/React_18-0b0f14?style=flat-square&logo=react&logoColor=61DAFB) +![Express](https://img.shields.io/badge/Express_4-0b0f14?style=flat-square&logo=express&logoColor=white) +![Gemini](https://img.shields.io/badge/Gemini_2.5_Flash-0b0f14?style=flat-square&logo=googlegemini&logoColor=8AB4F8) +![Groq](https://img.shields.io/badge/Groq_failover-0b0f14?style=flat-square&logoColor=white) +![SQLite](https://img.shields.io/badge/sql.js-0b0f14?style=flat-square&logo=sqlite&logoColor=74C0FC) +![Eval](https://img.shields.io/badge/eval_tests-14%2F14_passing-0b0f14?style=flat-square) +![License](https://img.shields.io/badge/license-MIT-0b0f14?style=flat-square) ---- +Google Solution Challenge 2026 project, team of two. Prototype, not a validated deployment. -## Table of Contents -- [The Problem](#the-problem) -- [What It Does](#what-it-does) -- [Why It Matters](#why-it-matters) -- [Key Features](#key-features) -- [Architecture](#architecture) -- [Tech Stack](#tech-stack) -- [Quickstart](#quickstart) -- [Demo Access](#demo-access) -- [Project Structure](#project-structure) -- [Environment Variables](#environment-variables) -- [API Reference](#api-reference) -- [Roadmap](#roadmap) -- [License](#license) +
--- -## The Problem - -Bengaluru has roughly **2.5 million buildings** across **198 wards**, monitored by around **840 BBMP enforcement officers**. Manual inspection can't keep pace with construction. Two concrete consequences: - -- The 2019 Dharwad building collapse killed 19 people in an unauthorized structure that went unflagged. -- Bellandur Lake's wetlands have been degraded by unchecked construction-linked sewage discharge. - -BBMP's own figures put average detection-to-notice time at 21+ days — long enough for a violation to be finished, occupied, or already causing damage before enforcement even starts. - -## What It Does - -InfraWatch closes the loop from satellite image to legal notice in one pipeline: - -1. **Detect** — Gemini 2.5 Flash analyzes satellite tiles and returns structured violation candidates: bounding boxes, classification, confidence score. -2. **Triage** — Each detection is auto-assigned an SLA tier: 4 hours at ≥90% confidence, 12 hours at ≥80%, 24 hours otherwise. -3. **Dispatch** — Round-robin assignment routes the case to the nearest available field officer. -4. **Draft** — Gemini drafts a formal enforcement notice citing real statute (Karnataka Municipal Corporations Act 1976 §308/321/322, BBMP Building Bye-laws 2003). If it can't confidently match a citation, it falls back to "Refer to applicable BBMP Bye-laws" instead of inventing a section number. -5. **Track** — Every action is timestamped and logged. -6. **Crowdsource** — A public reporting portal takes anonymous photo + GPS submissions into the same officer queue, no login required. +## Why -## Why It Matters +Bengaluru has an enormous stock of buildings and a small enforcement team, so detection-to-notice time is measured in weeks. InfraWatch explores closing that loop in software: a satellite tile goes to a vision model, a detection becomes a case with a response deadline, and the case gets a draft legal notice for an officer to review. -This is a hackathon prototype, not a validated deployment — we haven't run a pilot, so we're not going to hand you a fabricated ROI table. What we can say concretely: +No pilot has been run, so there are **no impact or accuracy claims** here. Numbers that can be measured honestly are in [`eval/`](eval/README.md); vision accuracy is explicitly pending. -- Manual review scales linearly with officer headcount; satellite-triggered detection doesn't. -- Detection-to-notice time drops from weeks (someone has to physically notice and report a violation) to however long the AI pipeline + officer review takes — realistically minutes to hours once a tile is scanned. -- Every notice and action is logged, which is closer to RTI-auditable than the current largely undocumented process. +## Features -Actual impact numbers are a Phase 2 pilot deliverable, not something we're claiming today. - -## Key Features - -**AI Vision** -- Gemini 2.5 Flash for satellite tile analysis — bounding boxes + confidence scores -- Automatic failover to Groq Llama 4 Scout Vision on Gemini rate-limit/quota errors -- AI Insights panel surfaces hot wards, SLA performance, false-positive trends -- Chat copilot with read access to the violations DB, cites real ward names and case IDs - -**Crisis Response Workflow** -- Live SLA-breach banner, polls every 15 seconds -- One-click "City Scan" — detects, files, and dispatches across 4 hotspots in under 30 seconds -- Role-based access control, round-robin officer assignment - -**Legal Drafting** -- Notices grounded in named statutes (KMC Act 1976, BBMP Building Bye-laws 2003, Karnataka Town & Country Planning Act 1961) -- Explicit guardrail against hallucinated section numbers -- Standard 10-section notice format: notice number, date, addressee, address, violation, legal provision, required action, deadline, consequences, officer - -**Civic Participation** -- Public reporting portal, anonymous, photo + GPS, IP-rate-limited - -**Transparency** -- Every case action is audit-logged -- Public `/health` endpoint reporting DB, AI provider, and uptime status +| Area | What the code does | Where | +|---|---|---| +| Vision analysis | Single-image and before/after analysis via Gemini; returns bounding boxes, violation type, confidence, risk level as JSON | `server/services/visionAI.js` | +| Confidence-based SLA | 4h if confidence >= 90, 12h if >= 80, else 24h; status ON_TIME / AT_RISK (past 80% of SLA) / BREACHED | `server/services/sla.js`, `server/routes/violations.js` | +| AI failover | Gemini primary; on 429 / quota / 503 / timeout the same request is retried on Groq (Llama 4 Scout for images, Llama 3.3 70B for text) | `visionAI.js` `callAI` | +| Legal notices | Draft notices from a prompt that restricts citations to a fixed statute list and tells the model to use a fallback phrase when unsure | `visionAI.js`, `gemini.js` | +| Case workflow | Roles (admin, commissioner, inspector, field officer), ward-scoped access, notice templates, activity log of case actions | `server/routes/*`, `server/middleware/*` | +| Crisis feed | Dashboard banner polling every 15 s for high-confidence unresolved cases and SLA breaches | `CrisisResponseBanner.jsx` | +| Citizen portal | Public, no-login report endpoint, in-memory limit of 5 reports/minute per IP | `server/routes/citizen.js` | +| Other | Case-review dossier, chat over a violation, compliance report, analytics, PWA shell, `/health` page | `server/routes/vision.js`, `analytics.js`, `client/` | ## Architecture -``` -┌──────────────────────┐ -│ VERCEL │ -│ React 18 + Vite │ -│ Leaflet · PWA │ -└──────────┬───────────┘ - │ /api proxy - ▼ -┌──────────────────────┐ ┌──────────────────────────┐ -│ RENDER │────►│ Google AI Studio │ -│ Node.js 20 + Express │ │ Gemini 2.5 Flash │ -│ sql.js · JWT auth │ │ (vision + text gen) │ -└──────────┬───────────┘ └──────────┬───────────────┘ - │ │ if 429 / quota - │ ▼ - │ ┌──────────────────────────┐ - │ │ Groq Llama 4 Scout │ - │ │ (auto-failover) │ - │ └──────────────────────────┘ - ▼ -┌──────────────────────┐ -│ ESRI World Imagery │ -│ (satellite tiles) │ -└──────────────────────┘ +```mermaid +flowchart LR + subgraph Client["Client - Vercel"] + UI["React 18 + Vite
Leaflet map, Recharts, PWA"] + end + subgraph Server["Server - Render (Express)"] + R["Routes
violations / vision / notices / citizen"] + AUTH["JWT auth + role and ward checks"] + SLA["sla.js
confidence to SLA tier"] + AI["visionAI.js callAI
timeout + failover"] + DB[("sql.js SQLite
auto-seeded on first boot")] + end + G["Gemini 2.5 Flash
(primary)"] + Q["Groq Llama 4 Scout / Llama 3.3 70B
(fallback)"] + UI -- "/api/* rewrite (vercel.json)" --> R + R --> AUTH --> DB + R --> SLA + R --> AI + AI -- "try first" --> G + AI -. "429 / 503 / quota / timeout" .-> Q ``` -- No vendor lock-in — Node-compatible with any host; Render was picked because it's free. -- AI layer has a single dispatcher with transparent failover and timeouts, not scattered try/catch. -- sql.js persists to disk for the demo; swapping to Postgres is a single connection-layer change, not a rewrite. +Vercel rewrites `/api/*` to the Render backend (`client/vercel.json`); locally, Vite proxies `/api` to the server. -## Tech Stack +## Design decisions -| Layer | Technology | Why | -|---|---|---| -| Frontend | React 18, Vite, Leaflet, PWA | fast iteration, real map rendering, installable | -| Backend | Node.js 20, Express | lightweight, fits free-tier hosting | -| Database | sql.js (SQLite) | zero setup for demo, trivial Postgres migration | -| Auth | JWT + bcrypt | role-based: admin / commissioner / inspector / field_officer | -| AI — primary | Gemini 2.5 Flash | multimodal vision, structured JSON output, grounded generation | -| AI — fallback | Groq Llama 4 Scout Vision + Llama 3.3 70B | failover on Gemini quota errors | -| Satellite | ESRI World Imagery | free CDN tiles | -| Hosting | Vercel + Render | free tier, zero-config GitHub deploy | -| Monitoring | UptimeRobot | keeps Render warm | - -## Quickstart - -### Prerequisites -- Node.js ≥ 18, npm ≥ 9 -- A free Gemini API key from [aistudio.google.com/apikey](https://aistudio.google.com/apikey) +**Confidence-based SLA tiers.** Higher-confidence detections get shorter response deadlines, so scarce officer time goes to the likeliest true positives first. The mapping is a 10-line pure function (`sla.js`) so it is testable; boundary and monotonicity tests are in `eval/`. Known rough edges, documented rather than hidden: the scale is implicitly 0-100 (a 0-1 fraction lands in the 24h tier), and the crisis feed lists cases from 85 while the 4h tier starts at 90. -```bash -git clone https://github.com/LordDevdeep/Infrawatch.git -cd Infrawatch +**AI failover in one place.** Every vision/text call in `visionAI.js` goes through a single dispatcher with a per-call timeout (default 30 s, `AI_TIMEOUT_MS`). Only retryable errors (429, quota, 503, unavailable, timeout) trigger failover; a 400 surfaces immediately instead of burning the second provider. Verified against stubbed providers in `eval/tests/failover.test.js`. Limitation: the notice route in `gemini.js` uses one provider and does not fail over. -npm install -cd client && npm install && cd .. -cd server && npm install && cd .. +**Citation guardrail, honestly.** The prompt names the only statutes and sections the model may cite (Karnataka Municipal Corporations Act 1976 s.308/321/321A/322, BBMP Building Bye-laws 2003, Karnataka Town and Country Planning Act 1961) and instructs a fallback phrase when unsure. That reduces, but does not eliminate, hallucination: there is **no server-side check** of the model's output, and the eval confirms fabricated sections pass through unchanged. A reference validator in `eval/lib/citations.js` shows a post-generation allowlist check is straightforward (20/20 synthetic fabrications caught, 0/20 false positives) and is the obvious next step. Every notice is a draft for officer review, and the section allowlist has not been verified against the statutes by a lawyer. -cp server/.env.example server/.env -# edit server/.env, paste your GEMINI_API_KEY +## Run locally -npm run dev # client on :5173, server on :3002 -``` +Requires Node >= 18 (Render config pins 20.11.1). A free Gemini key is optional for browsing the seeded demo; AI calls need it. -Open [http://localhost:5173](http://localhost:5173). The database auto-seeds on first boot with 214 realistic Bengaluru violations across 15 wards. - -### Production build ```bash -cd client && npm run build -cd ../server && node index.js +git clone https://github.com/tchxm/Infrawatch.git && cd Infrawatch +npm install +(cd client && npm install) +(cd server && npm install) +cp server/.env.example server/.env # add GEMINI_API_KEY (and optionally GROQ_API_KEY) +npm run dev # client :5173, server :3002 ``` -## Demo Access +The database auto-seeds on first boot if empty: 214 synthetic violations across 15 wards, users, and notice templates. Seed login is printed by the seed script (`server/db/seed.js`). Run the eval with `cd eval && npm test` (after the `server` install). -The live deploy has seeded demo accounts for admin, inspector, and field-officer roles. Credentials aren't published here — request them via [GitHub Issues](https://github.com/LordDevdeep/Infrawatch/issues) or run the project locally, where the seed script prints them to the console on first boot. +Key env vars (`server/.env.example`): `AI_PROVIDER`, `GEMINI_API_KEY`, `GEMINI_MODEL`, `GROQ_API_KEY`, `GROQ_MODEL`, `JWT_SECRET`, `PORT`, `AI_TIMEOUT_MS`. -OTP login is also wired up; codes print to the server console in development mode. - -## Project Structure - -``` -infrawatch/ -├── client/ React + Vite frontend -│ ├── public/ PWA manifest, static assets -│ └── src/ -│ ├── api/ centralised API client -│ ├── components/ -│ │ ├── dashboard/ CrisisResponseBanner, ImpactCard, AIInsightsPanel -│ │ ├── layout/ Sidebar, Footer -│ │ ├── map/ WardMap (Leaflet satellite layer) -│ │ └── ui/ KPI, EmptyState, AICityScanModal -│ ├── context/ Auth + Toast contexts -│ └── pages/ Dashboard, Violations, Detail, Map, Live Detection, -│ AI Tools, SDG, Settings, Citizen Report, Login -│ -├── server/ Node + Express backend -│ ├── db/ -│ │ ├── connection.js sql.js wrapper -│ │ ├── schema.sql 11 tables incl. citizen_reports -│ │ └── seed.js auto-seed (218 violations) -│ ├── middleware/ -│ │ ├── auth.js JWT + role enforcement -│ │ └── access.js ward-scoped access control -│ ├── routes/ -│ │ ├── auth.js, violations.js, vision.js, citizen.js, -│ │ ├── analytics.js, officers.js, notices.js, settings.js, logs.js -│ ├── services/ -│ │ ├── visionAI.js Gemini + Groq dispatcher, auto-failover -│ │ └── gemini.js legal notice generation -│ └── index.js server entry, /health, auto-seed on boot -│ -├── render.yaml Render deploy config -├── client/vercel.json Vercel proxy config -└── LICENSE MIT -``` +## Screenshots -## Environment Variables + +| Dashboard + crisis banner | Map view | Vision analysis | +|---|---|---| +| _TODO: docs/screenshots/dashboard.png_ | _TODO: docs/screenshots/map.png_ | _TODO: docs/screenshots/analysis.png_ | -`server/.env`: +| AI notice draft | Citizen report portal | Analytics | +|---|---|---| +| _TODO: docs/screenshots/notice.png_ | _TODO: docs/screenshots/citizen.png_ | _TODO: docs/screenshots/analytics.png_ | -```env -PORT=3002 -JWT_SECRET=replace_with_a_long_random_string +## Known limitations -AI_PROVIDER=gemini -GEMINI_API_KEY=AIzaSy... -GEMINI_MODEL=gemini-2.5-flash +- The demo's "city scan" and hotspot detections come from **seeded canned results** (`server/routes/vision.js`, `getSeededDetection`), so demo detections are not live model output. Non-hotspot images do go to the model. +- The single-image prompt tells the model to flag something whenever buildings are visible, which likely biases toward false positives. Vision accuracy is **not measured** (pending labeled data, see `eval/`). +- Officer assignment: the UI text mentions round-robin, but no round-robin logic exists in the server code; `full-pipeline` assigns new cases to the requesting user, and manual assignment is via the violations API. +- Model defaults are inconsistent: `.env.example`/`render.yaml` use `gemini-2.5-flash`, but `gemini.js` hardcodes `gemini-2.0-flash` for its text path and `visionAI.js` falls back to `gemini-2.0-flash` if `GEMINI_MODEL` is unset. +- After a failover, the `_provider`/`_model` metadata on parsed vision results still says Gemini. +- Data is synthetic; sql.js persists to a local file, so hosted state is ephemeral on free tiers. -GROQ_API_KEY=gsk_... -GROQ_MODEL=llama-3.3-70b-versatile +## My contributions -AUTO_SEED=true # set false to skip auto-seed on boot -AI_TIMEOUT_MS=30000 # per-AI-call timeout -``` +Mohammed Afnan (GitHub: [tchxm](https://github.com/tchxm)). -`.env` is gitignored. Never commit it. +**What the git history supports** (fork `tchxm/Infrawatch` of `LordDevdeep/Infrawatch`; 9 commits total, 2 author identities): -## API Reference +- 4 commits authored as `Afnan ` on 2026-07-18, all touching only `README.md` (documentation rewrite and fixes). +- 5 commits authored by `LordDevdeep` (2026-04-20 to 2026-05-03) including both "first commit" commits that contain the entire codebase (server, client, AI services) and a README rewrite. +- Work added in this fork after that: the `eval/` harness (SLA, failover, citation-guardrail and vision-eval tooling) and extraction of the SLA mapping into `server/services/sla.js` (behaviour-preserving). -### Public (no auth) +**What the history does not show.** The upstream history contains no commits authored under the Afnan identity for `server/` or the AI integration layer. Backend and AI-layer work (provider failover, notice generation, SLA routing) may have been written by Mohammed and committed under the upstream owner's account (the code arrives in "first commit" imports), pair-programmed, or written by the co-author. **Git history cannot distinguish these, so this README does not claim authorship of specific modules.** Mohammed should replace this paragraph with an accurate first-person account. -| Method | Path | Purpose | -|---|---|---| -| GET | `/api/health` | JSON service status | -| GET | `/health` | HTML status dashboard | -| POST | `/api/citizen/report` | submit a citizen violation report | + -### Authenticated (JWT) +## Evaluation -| Method | Path | Role required | -|---|---|---| -| POST | `/api/auth/login` | open | -| POST | `/api/auth/otp-request` | open | -| POST | `/api/auth/otp-verify` | open | -| GET | `/api/auth/me` | any | -| GET | `/api/violations` | any | -| POST | `/api/violations` | inspector / commissioner / admin | -| GET | `/api/violations/crisis-feed` | any | -| PATCH | `/api/violations/:id` | inspector / commissioner / admin | -| POST | `/api/violations/bulk-action` | inspector / commissioner / admin | -| POST | `/api/vision/analyze-single` | inspector / commissioner / admin | -| POST | `/api/vision/full-pipeline` | inspector / commissioner / admin | -| POST | `/api/vision/chat` | inspector / commissioner / admin | -| GET | `/api/vision/city-scan/plan` | any | -| GET | `/api/citizen/reports` | inspector / commissioner / admin | -| GET | `/api/officers` | any | -| GET | `/api/analytics/*` | any | -| GET/PUT | `/api/settings` | admin | - -## Roadmap - -**Phase 1 — Prototype (current)** -Working full-stack app, real Gemini integration, 214 seeded Bengaluru violations across 15 wards, Gemini/Groq auto-failover, live on Vercel + Render, MIT-licensed. - -**Phase 2 — Pilot (target: mid-2026)** -Sentinel-2 satellite feed via ESA Copernicus, WhatsApp Business API for officer dispatch, Kannada notice translation, Postgres migration, single-ward pilot with real BBMP officers and *measured* impact numbers. - -**Phase 3 — Multi-city (2027)** -Mumbai / Delhi / Chennai / Hyderabad expansion, permit-PDF parsing against satellite cross-check, public RTI dashboard, drone-imagery ingestion, court-ready evidence export. - -**Phase 4 — Open civic infrastructure (2028+)** -Open SDK for any Indian municipal corporation, predictive hot-zone flagging, India Stack integration (Aadhaar-verified reports, DigiLocker permits). +See [`eval/README.md`](eval/README.md) for methodology and results: SLA routing (9/9 boundary cases, monotone over a 201-point sweep), failover (7/7 scenarios behave as designed with stubbed providers), citation guardrail (app-side catch 0/20, reference validator 20/20 on synthetic fixtures), and vision accuracy (**pending**, harness and labeling template included). ## License -MIT — fork it, modify it, deploy it for your own city. Attribution appreciated, not required. - ---- - -Built for [Google Solution Challenge 2026](https://developers.google.com/community/gdsc-solution-challenge). \ No newline at end of file +MIT, see [LICENSE](LICENSE). Upstream: [LordDevdeep/Infrawatch](https://github.com/LordDevdeep/Infrawatch). From c5f53b541aaef131ec7fdd4e0211498dcd8dea9c Mon Sep 17 00:00:00 2001 From: Afnan Date: Mon, 21 Sep 2026 21:37:48 +0530 Subject: [PATCH 3/3] Enforce citation allowlist server-side; eval now measures the enforced guard Co-Authored-By: Claude Sonnet 5 --- README.md | 4 +-- eval/README.md | 6 ++-- eval/results/app_guardrail_enforcement.json | 6 ++++ eval/tests/citations.test.js | 2 +- ....test.js => guardrail_enforcement.test.js} | 21 ++++++----- .../services/citationGuard.js | 35 ++++++++++++++----- server/services/gemini.js | 9 +++-- server/services/visionAI.js | 12 +++++++ 8 files changed, 67 insertions(+), 28 deletions(-) create mode 100644 eval/results/app_guardrail_enforcement.json rename eval/tests/{guardrail_passthrough.test.js => guardrail_enforcement.test.js} (67%) rename eval/lib/citations.js => server/services/citationGuard.js (64%) diff --git a/README.md b/README.md index edde601..6c4419f 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Vercel rewrites `/api/*` to the Render backend (`client/vercel.json`); locally, **AI failover in one place.** Every vision/text call in `visionAI.js` goes through a single dispatcher with a per-call timeout (default 30 s, `AI_TIMEOUT_MS`). Only retryable errors (429, quota, 503, unavailable, timeout) trigger failover; a 400 surfaces immediately instead of burning the second provider. Verified against stubbed providers in `eval/tests/failover.test.js`. Limitation: the notice route in `gemini.js` uses one provider and does not fail over. -**Citation guardrail, honestly.** The prompt names the only statutes and sections the model may cite (Karnataka Municipal Corporations Act 1976 s.308/321/321A/322, BBMP Building Bye-laws 2003, Karnataka Town and Country Planning Act 1961) and instructs a fallback phrase when unsure. That reduces, but does not eliminate, hallucination: there is **no server-side check** of the model's output, and the eval confirms fabricated sections pass through unchanged. A reference validator in `eval/lib/citations.js` shows a post-generation allowlist check is straightforward (20/20 synthetic fabrications caught, 0/20 false positives) and is the obvious next step. Every notice is a draft for officer review, and the section allowlist has not been verified against the statutes by a lawyer. +**Citation guardrail, two layers.** The prompt names the only statutes and sections the model may cite (Karnataka Municipal Corporations Act 1976 s.308/321/321A/322, BBMP Building Bye-laws 2003, Karnataka Town and Country Planning Act 1961) and instructs a fallback phrase when unsure. Prompts alone cannot prevent hallucination, so `server/services/citationGuard.js` also checks every AI-drafted notice against the allowlist and replaces any off-allowlist citation with the fallback phrase, reporting what it removed in `citationsRemoved`. Measured before/after: the app passed 20/20 fabricated citations through with no guard, and catches 20/20 with it (0/20 false positives on valid text; synthetic fixtures, not a claim about real model hallucination rates). Every notice is still a draft for officer review, and the allowlist has not been verified against the statutes by a lawyer. ## Run locally @@ -128,7 +128,7 @@ Mohammed Afnan (GitHub: [tchxm](https://github.com/tchxm)). ## Evaluation -See [`eval/README.md`](eval/README.md) for methodology and results: SLA routing (9/9 boundary cases, monotone over a 201-point sweep), failover (7/7 scenarios behave as designed with stubbed providers), citation guardrail (app-side catch 0/20, reference validator 20/20 on synthetic fixtures), and vision accuracy (**pending**, harness and labeling template included). +See [`eval/README.md`](eval/README.md) for methodology and results: SLA routing (9/9 boundary cases, monotone over a 201-point sweep), failover (7/7 scenarios behave as designed with stubbed providers), citation guardrail (app-side catch 0/20 before the guard was added, 20/20 after; 0/20 false positives on synthetic fixtures), and vision accuracy (**pending**, harness and labeling template included). ## License diff --git a/eval/README.md b/eval/README.md index 5fe7f5f..3a1704e 100644 --- a/eval/README.md +++ b/eval/README.md @@ -39,12 +39,12 @@ Runs the real dispatcher in `server/services/visionAI.js` with dummy keys and a Finding: after a failover, the parsed result's `_provider`/`_model` metadata still reports `gemini` (it is read from config in `parseVisionResponse`, not from the provider that actually answered). The dispatcher itself returns the correct provider; the metadata attached to the parsed payload does not. Scope: only the `callAI` path in `visionAI.js` fails over. `generateAINotice` in `services/gemini.js` (the `/api/notices/generate-ai` route) uses a single provider chosen by `AI_PROVIDER` (default `groq` in that file) with no failover. -### 3. Citation guardrail (`tests/citations.test.js`, `tests/guardrail_passthrough.test.js`) +### 3. Citation guardrail (`tests/citations.test.js`, `tests/guardrail_enforcement.test.js`) Two different questions, deliberately separated: -**a) Does the app catch fabricated citations in code?** The guardrail today is a **prompt instruction only** (`visionAI.js` `generateLegalNoticeFromAnalysis`, `gemini.js` `buildPrompt`). There is no server-side validation of the model output. With a stubbed provider returning notices that cite fabricated provisions, the real function returned the fabricated text unchanged in **20 of 20** cases. App-side catch rate: **0%**. The test asserts this observed behaviour; if validation is added, it will fail and should be updated. +**a) Does the app catch fabricated citations in code?** Originally the guardrail was a **prompt instruction only**, and with a stubbed provider returning 20 notices that cite fabricated provisions the real function returned the fabricated text unchanged in **20 of 20** cases (catch rate **0%**). That prompted `server/services/citationGuard.js`, now called from `visionAI.js` and `gemini.js`. Re-running the same test, the app removes the fabricated citation in **20 of 20** cases (catch rate **100%**). -**b) What would a deterministic post-check against the same allowlist catch?** `lib/citations.js` is a reference validator (not part of the app). Allowlist = the sections named in the app's own prompt (KMC Act 1976 s.308/321/321A/322; Bye-laws 2003 2.2/2.5/3.0/4.6; KTCP Act 1961 s.14/15/76-FF). On 20 hand-written fabricated snippets (non-existent sections, wrong-Act sections, IPC/Constitution/case-law citations, a fake statute, a real-looking fallback phrase followed by a fabricated section) and 20 valid snippets: **20/20 caught (100%), 0/20 false positives**. +**b) What would a deterministic post-check against the same allowlist catch?** `server/services/citationGuard.js` is the validator, now enforced in the app. Allowlist = the sections named in the app's own prompt (KMC Act 1976 s.308/321/321A/322; Bye-laws 2003 2.2/2.5/3.0/4.6; KTCP Act 1961 s.14/15/76-FF). On 20 hand-written fabricated snippets (non-existent sections, wrong-Act sections, IPC/Constitution/case-law citations, a fake statute, a real-looking fallback phrase followed by a fabricated section) and 20 valid snippets: **20/20 caught (100%), 0/20 false positives**. Caveats: the fixtures are synthetic and written by the same author as the validator, so 100% is a sanity check, not a generalisation claim. The allowlist comes from the app's prompt and has **not** been verified against the statutes. Neither result says anything about how often a real LLM hallucinates a citation; that needs a live run (see below). diff --git a/eval/results/app_guardrail_enforcement.json b/eval/results/app_guardrail_enforcement.json new file mode 100644 index 0000000..2f018b2 --- /dev/null +++ b/eval/results/app_guardrail_enforcement.json @@ -0,0 +1,6 @@ +{ + "fabricated_n": 20, + "app_returned_fabricated_text_unchanged": 0, + "app_side_catch_rate": 1, + "still_flagged_by_validator_after_guard": 0 +} diff --git a/eval/tests/citations.test.js b/eval/tests/citations.test.js index fd46299..ca705f0 100644 --- a/eval/tests/citations.test.js +++ b/eval/tests/citations.test.js @@ -1,7 +1,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; -import { validateCitations } from '../lib/citations.js'; +import { validateCitations } from '../../server/services/citationGuard.js'; import { record } from '../lib/results.js'; const fx = JSON.parse(fs.readFileSync(new URL('../fixtures/citations.json', import.meta.url))); diff --git a/eval/tests/guardrail_passthrough.test.js b/eval/tests/guardrail_enforcement.test.js similarity index 67% rename from eval/tests/guardrail_passthrough.test.js rename to eval/tests/guardrail_enforcement.test.js index 23e6252..47e2e6d 100644 --- a/eval/tests/guardrail_passthrough.test.js +++ b/eval/tests/guardrail_enforcement.test.js @@ -1,16 +1,14 @@ /** - * Measures what the APP ITSELF does (not the reference validator) when the model returns a - * notice with a fabricated citation: runs the real generateLegalNoticeFromAnalysis with a - * stubbed provider. The app's guardrail is a prompt instruction only, so we expect the - * fabricated text to pass through untouched. This test asserts that observed behaviour so the - * result is reproducible; if someone adds server-side validation, this test will fail and - * should be updated (that would be an improvement). + * Measures what the APP ITSELF does when the model returns a notice with a fabricated + * citation: runs the real generateLegalNoticeFromAnalysis with a stubbed provider. + * Before the server-side citation guard existed this passed 20/20 fabrications through + * unchanged (0% catch rate). The guard now strips them; this test tracks the catch rate. */ import test from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import { record } from '../lib/results.js'; -import { validateCitations } from '../lib/citations.js'; +import { validateCitations } from '../../server/services/citationGuard.js'; process.env.AI_PROVIDER = 'groq'; process.env.GROQ_API_KEY = 'dummy-groq'; @@ -30,7 +28,7 @@ globalThis.fetch = async () => new Response(JSON.stringify({ test.after(() => { globalThis.fetch = realFetch; }); -test('app passes fabricated citations through unchanged (prompt-only guardrail)', async () => { +test('app strips fabricated citations before returning the notice', async () => { const violation = { id: '#IW-9999', owner_name: 'Test', address: 'x', ward: 'Ward 1' }; const analysis = { primaryViolationType: 'Unauthorized Floor Addition', overallConfidence: 90, riskLevel: 'HIGH', totalEstimatedAreaSqFt: 100, visualEvidence: [] }; const penalty = { calculatedPenaltyINR: 1, calculatedPenaltyLakhs: 1, legalBasis: 'x', formula: 'x' }; @@ -42,11 +40,12 @@ test('app passes fabricated citations through unchanged (prompt-only guardrail)' if (out.noticeContent.includes(fab)) passedThrough++; if (!validateCitations(out.noticeContent).ok) flaggedByRef++; } - record('app_guardrail_passthrough', { + record('app_guardrail_enforcement', { fabricated_n: fx.fabricated.length, app_returned_fabricated_text_unchanged: passedThrough, app_side_catch_rate: 1 - passedThrough / fx.fabricated.length, - would_be_flagged_by_reference_validator: flaggedByRef, + still_flagged_by_validator_after_guard: flaggedByRef, }); - assert.equal(passedThrough, fx.fabricated.length); + assert.equal(passedThrough, 0); + assert.equal(flaggedByRef, 0); }); diff --git a/eval/lib/citations.js b/server/services/citationGuard.js similarity index 64% rename from eval/lib/citations.js rename to server/services/citationGuard.js index 95849e7..3d748e2 100644 --- a/eval/lib/citations.js +++ b/server/services/citationGuard.js @@ -1,13 +1,10 @@ /** - * Reference citation validator used by the eval. - * - * IMPORTANT: this is NOT part of the InfraWatch runtime. The app's only - * anti-hallucination guard today is a prompt instruction (see - * server/services/visionAI.js, generateLegalNoticeFromAnalysis, and - * server/services/gemini.js buildPrompt). This module shows what a - * deterministic post-generation check against the SAME allowlist would catch, - * and the eval measures it. The allowlist is copied from the app's prompt; it - * has NOT been verified against the statutes themselves. + * Deterministic post-generation citation guard. + * Prompts alone cannot stop a model from inventing statute sections, so every AI-drafted + * notice is checked against an allowlist. Offending clauses are replaced with the safe + * fallback phrase and reported so an officer can see what was removed. + * NOTE: the allowlist mirrors the prompt in visionAI.js/gemini.js and has not been + * verified against the statutes themselves. */ export const ALLOWED = { KMC: new Set(['308', '321', '321A', '322']), // Karnataka Municipal Corporations Act, 1976 @@ -54,3 +51,23 @@ export function validateCitations(input) { } return { ok: violations.length === 0, violations, citations, usedFallbackPhrase: FALLBACK.test(text) }; } + +export const FALLBACK_PHRASE = 'Refer to applicable BBMP Building Bye-laws, 2003'; + +/** Replaces any clause citing an off-allowlist authority with the fallback phrase. */ +export function sanitizeCitations(text) { + const input = String(text ?? ''); + const removed = []; + const lines = input.split(/\r?\n/).map((line) => { + const clauses = line.split(/(;)/); + return clauses.map((clause) => { + if (clause === ';') return clause; + const { ok } = validateCitations(clause); + if (ok) return clause; + removed.push(clause.trim()); + const label = clause.match(/^\s*([A-Z][A-Z &]+:)/); + return (label ? label[1] + ' ' : '') + FALLBACK_PHRASE; + }).join(''); + }); + return { text: lines.join('\n'), removed, ok: removed.length === 0 }; +} diff --git a/server/services/gemini.js b/server/services/gemini.js index 7da0eb3..057c8be 100644 --- a/server/services/gemini.js +++ b/server/services/gemini.js @@ -1,5 +1,6 @@ import { GoogleGenerativeAI } from '@google/generative-ai'; import Groq from 'groq-sdk'; +import { sanitizeCitations } from './citationGuard.js'; const provider = (process.env.AI_PROVIDER || 'groq').toLowerCase(); const geminiKey = process.env.GEMINI_API_KEY; @@ -109,9 +110,11 @@ async function generateWithGroq(violation, templateContent = null) { console.log('[ai] Groq response received'); + const guarded = sanitizeCitations(text); return { success: true, - content: text, + content: guarded.text, + citationsRemoved: guarded.removed, model: GROQ_MODEL, provider: 'groq', generatedAt: new Date().toISOString(), @@ -137,9 +140,11 @@ async function generateWithGemini(violation, templateContent = null) { throw new Error('Gemini returned an empty notice draft.'); } + const guarded = sanitizeCitations(text); return { success: true, - content: text, + content: guarded.text, + citationsRemoved: guarded.removed, model: GEMINI_MODEL, provider: 'gemini', generatedAt: new Date().toISOString(), diff --git a/server/services/visionAI.js b/server/services/visionAI.js index 3fda3c2..9b27e2f 100644 --- a/server/services/visionAI.js +++ b/server/services/visionAI.js @@ -4,6 +4,8 @@ * Primary: Google Gemini 2.0 Flash (via @google/generative-ai SDK + AI Studio API key) * Fallback: Groq Llama 4 Scout Vision (only used if Gemini rate-limits) */ +import { sanitizeCitations, validateCitations } from './citationGuard.js'; + const GROQ_VISION_MODEL = 'meta-llama/llama-4-scout-17b-16e-instruct'; const GEMINI_VISION_MODEL = process.env.GEMINI_MODEL || 'gemini-2.0-flash'; @@ -532,8 +534,18 @@ JSON schema: const text = await callTextAI(prompt); const parsed = parseVisionResponse(text, 'legalNotice'); + const guardedContent = sanitizeCitations(parsed.noticeContent); + const guardedConsequences = sanitizeCitations(parsed.consequencesIfIgnored); + const guardedSections = (Array.isArray(parsed.legalSectionsCited) ? parsed.legalSectionsCited : []) + .filter((c) => validateCitations(String(c)).ok); + const citationsRemoved = [...guardedContent.removed, ...guardedConsequences.removed]; + return { ...parsed, + noticeContent: guardedContent.text, + consequencesIfIgnored: guardedConsequences.text, + legalSectionsCited: guardedSections, + citationsRemoved, generatedAt: new Date().toISOString(), caseReference: violation.id, issuedTo: violation.owner_name || 'Property Owner',