diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..48a20b8 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,41 @@ +name: Deploy Lighthouse to Cloudflare + +on: + workflow_dispatch: + push: + branches: + - main + +jobs: + gate: + if: github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '[deploy lighthouse]') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v6 + with: + node-version: 20 + cache: npm + - run: npm ci + - run: npm run typecheck + - run: npm test + + deploy: + needs: gate + runs-on: ubuntu-latest + permissions: + contents: read + deployments: write + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-node@v6 + with: + node-version: 20 + cache: npm + - run: npm ci + - name: Deploy Worker while preserving provisioned secrets + uses: cloudflare/wrangler-action@v3 + with: + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + command: deploy diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d109a1..4b7ac55 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # Changelog +## [1.26.0] - 2026-07-18 + +- Added the protected `GET /report?view=tgc` commercial analytics view with today, 7-day, and 30-day acquisition, audience, engagement, funnel, performance, content, and health summaries. +- Enforced a TGC-specific site/event allowlist, production-origin matching, path/URL consistency, bounded context, and query/fragment stripping at ingestion. +- Added explicit support for consent-created TGC visitor/session IDs while keeping all identifiers out of operator reports and downstream-summary contracts. +- Replaced stored unsalted site-event IP/user-agent hashes with minute-scoped keyed abuse identifiers kept only in the two-day rate table; raw events no longer store IP hashes, user-agent hashes, or request IDs, and scheduled maintenance scrubs those legacy columns from existing rows. +- Added raw site-event retention: 90 days for `tgc_site`, 30 days for other site-event properties. +- Reused the existing D1 schema; no migration was added or applied. +- Added a gated Cloudflare deployment workflow that runs the full validation suite and deploys only on manual dispatch or an explicitly marked release merge, preserving provisioned Worker secrets. + ## [1.25.0] - 2026-07-18 - Added `BUS_CORE_TRAFFIC_TRUTH.md` as the authoritative definition of artifact traffic, successful responses, HMAC client-network buckets, inferred download intent, confirmed product signals, lead separation, privacy, retention, rollout thresholds, rollback, evidence, and blind spots. diff --git a/README.md b/README.md index 19d3a3e..c38e008 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,13 @@ # buscore-lighthouse +## TGC website analytics + +Version 1.26.0 adds an explicitly consented commercial analytics lane for `site_key=tgc_site`. Lighthouse is the raw-event and aggregate-report source of truth; the protected operator view is `GET /report?view=tgc` using the existing `X-Admin-Token` contract. + +The view reports today, 7-day, and 30-day page execution, new/returning and session counts, acquisition, content, engagement, service-funnel outcomes, performance, and health. It never returns visitor/session/rate identifiers or form contents. Raw TGC events are retained for 90 days; rotating keyed rate identifiers are retained for two days. See `TGC_SITE_ANALYTICS_POLICY.md` for the product-specific justification and prohibitions. + +Production deploys use the gated `.github/workflows/deploy.yml`: full tests first, then Wrangler only on manual dispatch or an explicitly marked release merge. Migrations remain separate and are never implied by deployment. + BUS Core artifact delivery and demand semantics are defined in `BUS_CORE_TRAFFIC_TRUTH.md`. Version 1.25.0 keeps downloads public while separating raw Worker traffic, successful artifact responses, privacy-preserving daily client-network buckets, probable-human intent proxies, confirmed product telemetry, and leads. Migration `0014_add_artifact_traffic_truth.sql` was applied remotely before the 2026-07-18 v1.25.0 deployment. ## BUS Core transition direction diff --git a/SOT.md b/SOT.md index a568c89..986b7c0 100644 --- a/SOT.md +++ b/SOT.md @@ -1,5 +1,17 @@ # Lighthouse — Source of Truth +## TGC consented commercial analytics — v1.26.0 branch implementation + +The protected `GET /report?view=tgc` view is the canonical on-demand source for True Good Craft website analytics. It reads existing `site_events_raw` storage and returns consented page-execution metrics for today, 7 days, and 30 days: page views, sessions, visitors, first/returning visits, commercial intent, form funnel outcomes, deep-scroll and engaged-time milestones, average page-load/LCP/CLS, top events, paths, sources, campaigns, and sections. It never returns visitor IDs, session IDs, rate identifiers, user-agent hashes, request IDs, or form contents. + +TGC site ingestion is an explicitly approved exception to the company-wide aggregate-only default. Random first-party visitor and session IDs are justified because aggregate counters alone cannot measure new versus returning visits, sessions per visitor, multi-page journeys, attribution continuity, or funnel progression. The visitor ID is created only after explicit analytics consent and persists in the browser for at most 395 days; the session ID renews after 30 minutes of inactivity. They are used only for TGC website measurement, are not linked to intake identity or other properties, and are not exposed downstream. + +The server enforces the TGC event allowlist, production-origin match, path/URL consistency, origin-and-path-only URL storage, bounded context, and test-mode exclusion. Form values, typed content, keystrokes, raw IP addresses, user-agent hashes, exact location, fingerprints, cross-site advertising identifiers, and session replay are prohibited. Minute-scoped abuse identifiers use keyed HMAC and are retained for two days; they are not copied into raw events. Scheduled maintenance also nulls legacy IP-hash, user-agent-hash, and request-ID columns in existing site-event rows. Raw TGC events are pruned after 90 days; other site-event raw rows are pruned after 30 days. No new D1 migration is required because the existing site-event schema is reused. + +Lighthouse remains the source of truth. Agent Smith may present this protected aggregate view through `/tgc`. Airtable may receive curated periodic KPI/campaign/content/experiment summaries later, but must not receive raw events or stable identifiers. + +Production deployment is governed by `.github/workflows/deploy.yml`. It runs the complete typecheck/test gate and deploys only on manual dispatch or a main-branch commit explicitly marked `[deploy lighthouse]`; ordinary pushes do not deploy. Wrangler deployment preserves separately provisioned Worker secrets. Schema migrations remain a separate, explicit operation and this release requires none. + ## BUS Core traffic truth and bounded delivery work — v1.25.0 deployed `BUS_CORE_TRAFFIC_TRUTH.md` is the authoritative metric/privacy/retention/rollout contract for the new additive fields. Lighthouse now distinguishes Worker-visible artifact requests, successful 200/206 handoffs, full and partial responses, HEAD and Range traffic, declared response bytes, cache outcomes, daily HMAC/IP/version client-network buckets, repeats excluded from that proxy, inferred download intent, confirmed product events, and voluntary leads. None of these fields may be renamed to people, users, installations, completed downloads, or revenue. diff --git a/TGC_SITE_ANALYTICS_POLICY.md b/TGC_SITE_ANALYTICS_POLICY.md new file mode 100644 index 0000000..26089f0 --- /dev/null +++ b/TGC_SITE_ANALYTICS_POLICY.md @@ -0,0 +1,65 @@ +# True Good Craft Website Analytics Policy + +## Status + +This is the product-specific declaration for `site_key=tgc_site`. It records the user-approved commercial analytics exception to the company-wide aggregate-only and no-persistent-identifier defaults. It does not change BUS Core, Star Map, or service-intake data policy. + +## Levels and purposes + +- Page level: consented browser execution, acquisition, content interest, navigation, engagement, funnel outcomes, performance, and sanitized reliability. +- Host level: Cloudflare traffic as a separate broad traffic source that may include bots and must not be described as human engagement. +- Internal: protected Lighthouse and Agent Smith aggregate reporting. +- User level: no analytics-only collection. Information intentionally submitted through an intake remains a separate business relationship path. + +The purpose is to improve TGC content, acquisition, commercial offers, inquiry flow, and site reliability. Each allowed event must answer one of those questions. + +## Explicit identifier exception + +After explicit analytics consent, the TGC site creates: + +- `anon_user_id`: a random first-party value retained in the browser for at most 395 days. +- `session_id`: a random first-party value renewed after 30 minutes of inactivity. + +Aggregate-only measurement is insufficient for new-versus-returning analysis, sessions per visitor, multi-page journeys, attribution continuity, and service-funnel progression. These identifiers are used only for those TGC website questions. + +They are not derived from IP address, user agent, account data, form data, device characteristics, or another property. They are not linked to intake identity, BUS Core, Star Map, advertising networks, or external profiles. They are not exposed in operator reports, Airtable summaries, logs, or exports. + +Essential-only choice, Global Privacy Control, and Do Not Track keep optional analytics disabled. Withdrawing consent deletes the browser-side analytics identity. + +## Allowed event data + +- production origin and path, with query and fragment removed +- origin-and-path-only referrer +- `src` and bounded UTM attribution +- allowlisted semantic event name and bounded event value +- random visitor/session IDs and new/returning state +- coarse device, viewport, language, timezone, and edge country +- scroll/engaged-time/section milestones +- form identifier, field identifier, validation state, and submit outcome +- bounded page-load, FCP, LCP, CLS, and sanitized error category +- test-mode marker + +## Prohibited data + +- form values, names, emails, phone numbers, messages, typed content, or keystrokes +- passwords, credentials, intake payloads, or business records +- raw IP retention, stored user-agent hashes, request IDs in raw TGC events, or exact geolocation +- fingerprinting, session replay, cross-site advertising IDs, account linking, or enrichment +- full URL query strings/fragments, arbitrary event names, or arbitrary context keys +- visitor/session/rate identifiers in operator reports or Airtable + +## Retention + +- raw accepted/dropped TGC site events: 90 days +- minute-scoped keyed abuse-control identifiers: 2 days +- browser visitor ID: at most 395 days +- browser session ID: 30 minutes of inactivity +- longer-lived analytics: aggregate summaries only, without visitor/session/rate identifiers + +## Reporting and downstream use + +Lighthouse is the source of truth. `GET /report?view=tgc` is the protected aggregate contract and Agent Smith `/tgc` is the on-demand presentation surface. Airtable may later receive curated daily/weekly KPI, campaign, content, and experiment rows. Airtable must not be a raw-event sink. + +## Safety + +Collection is consented, fail-soft, production-origin restricted, rate-limited, and server-allowlisted. Analytics failure must never block navigation, forms, intake delivery, or mailto fallback. diff --git a/package-lock.json b/package-lock.json index 0fb14a3..6133857 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "buscore-lighthouse", - "version": "1.25.0", + "version": "1.26.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "buscore-lighthouse", - "version": "1.25.0", + "version": "1.26.0", "license": "ISC", "devDependencies": { "@cloudflare/workers-types": "^4.20260305.0", diff --git a/package.json b/package.json index c66b76c..e7e2c30 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "buscore-lighthouse", - "version": "1.25.0", + "version": "1.26.0", "description": "Standalone deterministic metrics worker: manifest proxy + fixed daily counters + protected on-demand reporting.", "scripts": { "dev": "wrangler dev", diff --git a/src/index.ts b/src/index.ts index 89346ee..5a889a1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -350,7 +350,7 @@ type SiteSectionAvailability = { identity: boolean; read: boolean; }; -type ReportView = "legacy" | "fleet" | "site" | "source_health" | "asset" | "monthly"; +type ReportView = "legacy" | "fleet" | "site" | "tgc" | "source_health" | "asset" | "monthly"; type ReportWindow = { start_day: string; end_day: string; @@ -438,6 +438,7 @@ type ReportRequestResolution = | { ok: true; view: "legacy"; siteEventFilter: SiteEventFilter | null } | { ok: true; view: "fleet" } | { ok: true; view: "site"; siteEventFilter: SiteEventFilter } + | { ok: true; view: "tgc" } | { ok: true; view: "source_health" } | { ok: true; view: "asset" } | { ok: true; view: "monthly" } @@ -541,7 +542,7 @@ const PAGEVIEW_ALLOWED_ORIGINS: Set = new Set( TRACKED_SITES.find((s) => s.site_key === "buscore")?.allowed_origins ?? [] ); const PAGEVIEW_INGEST_VERSION = "1.9.0"; -const SITE_EVENT_INGEST_VERSION = "1.11.0"; +const SITE_EVENT_INGEST_VERSION = "1.12.0"; const PAGEVIEW_INVALID_JSON_DEBUG_ENABLED = true; const PAGEVIEW_INVALID_JSON_DEBUG_PREVIEW_CHARS = 500; const PAGEVIEW_RATE_LIMIT_PER_MINUTE = 50; @@ -549,6 +550,8 @@ const PAGEVIEW_RAW_RETENTION_DAYS = 30; const PAGEVIEW_RATE_LIMIT_RETENTION_DAYS = 2; const SITE_EVENT_RATE_LIMIT_PER_MINUTE = 50; const SITE_EVENT_RATE_LIMIT_RETENTION_DAYS = 2; +const SITE_EVENT_RAW_RETENTION_DAYS = 30; +const TGC_SITE_EVENT_RAW_RETENTION_DAYS = 90; const TOP_PAGEVIEW_DIMENSION_LIMIT = 5; const DIRECT_SOURCE_LABEL = "(direct)"; const EARLIEST_REPORT_DAY = "0000-01-01"; @@ -561,6 +564,17 @@ const VERSIONED_ARTIFACT_CACHE_CONTROL = "public, max-age=31536000, s-maxage=315 const UPDATE_CHECK_REQUIRED_QUERY_KEYS = ["current_version", "channel", "first_check"] as const; const UPDATE_CHECK_ALLOWED_CHANNELS = new Set(BUSCORE_TELEMETRY_RELEASE_CHANNELS); const UUID_V4_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const TGC_ANONYMOUS_ID_PATTERN = /^[vs]_(?:[0-9a-f]{32}|[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})$/i; +const TGC_SITE_EVENT_ALLOWLIST = new Set([ + "page_view", "session_start", "first_visit", "returning_visit", + "internal_navigation", "outbound_click", "contact_click", "email_click", "buscore_outbound_click", + "services_interest", "infrastructure_cta_click", "infrastructure_package_interest", "ops_care_interest", + "audit_cta_click", "infrastructure_form_start", "infrastructure_form_submit", "audit_form_start", "audit_form_submit", + "form_start", "form_field_complete", "form_validation_error", "form_submit_attempt", + "form_submit_success", "form_submit_failure", "form_submit_fallback", + "scroll_depth", "engaged_time", "section_view", + "web_vital_page_load_ms", "web_vital_fcp_ms", "web_vital_lcp_ms", "web_vital_cls", "js_error", +]); const PAGEVIEW_ALLOWED_DEVICES = new Set(["desktop", "mobile", "tablet"]); const PAGEVIEW_VIEWPORT_PATTERN = /^\d+x\d+$/; const BUSCORE_TRAFFIC_QUERY = `query DailyBuscoreTraffic($zoneTag: string, $start: Time!, $end: Time!, $host: string!) { @@ -1093,6 +1107,7 @@ export function normalizeReportView(value: string | null): ReportView | null { if ( normalized === "fleet" || normalized === "site" || + normalized === "tgc" || normalized === "source_health" || normalized === "asset" || normalized === "monthly" @@ -1180,7 +1195,7 @@ export function normalizeOptionalAnonymousId(value: unknown): string | null { } // Keep ingest permissive for backward compatibility while filtering obvious garbage. - if (!UUID_V4_PATTERN.test(normalized)) { + if (!UUID_V4_PATTERN.test(normalized) && !TGC_ANONYMOUS_ID_PATTERN.test(normalized)) { return null; } @@ -1313,84 +1328,82 @@ export function parseCanonicalPageviewPayload(payload: unknown): PageviewInput | }; } +export function sanitizeAnalyticsLocation(value: string, allowEmpty: boolean = false): string | null { + if (!value.trim()) { + return allowEmpty ? "" : null; + } + try { + const parsed = new URL(value); + if (parsed.protocol !== "https:" && parsed.protocol !== "http:") { + return null; + } + return `${parsed.origin}${parsed.pathname}`; + } catch { + return null; + } +} + export function parseCanonicalEventPayload(payload: unknown): SiteEventInput | null { const root = typeof payload === "object" && payload !== null ? (payload as Record) : {}; const siteKey = readRequiredString(root, "site_key"); const eventName = readRequiredString(root, "event_name"); const clientTs = readRequiredString(root, "client_ts"); const path = readRequiredString(root, "path"); - const url = readRequiredString(root, "url"); - const referrer = readRequiredString(root, "referrer", true); + const rawUrl = readRequiredString(root, "url"); + const rawReferrer = readRequiredString(root, "referrer", true); const device = readRequiredString(root, "device"); const viewport = readRequiredString(root, "viewport"); const lang = readRequiredString(root, "lang", true); const tz = readRequiredString(root, "tz", true); const utmRaw = root.utm; - if ( - !siteKey || - !eventName || - !clientTs || - !path || - !url || - referrer === null || - !device || - !viewport || - lang === null || - tz === null || - typeof utmRaw !== "object" || - utmRaw === null || - Array.isArray(utmRaw) - ) { - return null; - } - - if (!Number.isFinite(Date.parse(clientTs))) { - return null; - } - - if (!path.startsWith("/")) { + if (!siteKey || !eventName || !clientTs || !path || !rawUrl || rawReferrer === null || !device || !viewport + || lang === null || tz === null || typeof utmRaw !== "object" || utmRaw === null || Array.isArray(utmRaw)) { return null; } - if (!isValidAbsoluteUrl(url)) { - return null; - } - - if (!PAGEVIEW_ALLOWED_DEVICES.has(device)) { + const site = getSiteByKey(siteKey); + const url = sanitizeAnalyticsLocation(rawUrl); + const referrer = sanitizeAnalyticsLocation(rawReferrer, true); + if (!site || !url || referrer === null || !Number.isFinite(Date.parse(clientTs)) || !path.startsWith("/") + || !PAGEVIEW_ALLOWED_DEVICES.has(device) || !PAGEVIEW_VIEWPORT_PATTERN.test(viewport)) { return null; } - if (!PAGEVIEW_VIEWPORT_PATTERN.test(viewport)) { + const parsedUrl = new URL(url); + if (!site.allowed_origins.includes(parsedUrl.origin) || parsedUrl.pathname !== path) { return null; } - - if (!getSiteByKey(siteKey)) { + if (siteKey === "tgc_site" && !TGC_SITE_EVENT_ALLOWLIST.has(eventName)) { return null; } const utm = utmRaw as Record; + const bounded = (value: unknown, max: number): string | null => { + const normalized = readOptionalString(value); + return normalized ? normalized.slice(0, max) : null; + }; return { site_key: siteKey, - event_name: eventName, + event_name: eventName.slice(0, 80), client_ts: clientTs, - path, + path: path.slice(0, 500), url, referrer, - src: readOptionalString(root.src), - utm_source: readOptionalString(utm.source), - utm_medium: readOptionalString(utm.medium), - utm_campaign: readOptionalString(utm.campaign), - utm_content: readOptionalString(utm.content), + src: bounded(root.src, 120), + utm_source: bounded(utm.source, 120), + utm_medium: bounded(utm.medium, 120), + utm_campaign: bounded(utm.campaign, 160), + utm_content: bounded(utm.content, 160), device, viewport, - lang, - tz, + lang: lang.slice(0, 35), + tz: tz.slice(0, 80), anon_user_id: normalizeOptionalAnonymousId(root.anon_user_id), session_id: normalizeOptionalAnonymousId(root.session_id), is_new_user: coerceBooleanLikeToInt(root.is_new_user), - event_value: readOptionalString(root.event_value), + event_value: bounded(root.event_value, 160), test_mode: coerceBooleanLikeToInt(root.test_mode), }; } @@ -1425,6 +1438,19 @@ async function sha256Hex(value: string): Promise { return Array.from(new Uint8Array(digest), (chunk) => chunk.toString(16).padStart(2, "0")).join(""); } +async function keyedRateIdentifier(secret: string, minuteBucket: string, clientIp: string): Promise { + const encoder = new TextEncoder(); + const key = await crypto.subtle.importKey( + "raw", + encoder.encode(secret), + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign"] + ); + const signature = await crypto.subtle.sign("HMAC", key, encoder.encode(`${minuteBucket}:${clientIp}`)); + return Array.from(new Uint8Array(signature), (chunk) => chunk.toString(16).padStart(2, "0")).join(""); +} + async function incrementCounter(db: D1Database, day: string, column: CounterColumn): Promise { await db .prepare( @@ -3087,11 +3113,16 @@ async function prunePageviewData(db: D1Database, now: Date = new Date()): Promis const siteEventRateLimitCutoffMinute = utcMinuteBucket( new Date(now.getTime() - SITE_EVENT_RATE_LIMIT_RETENTION_DAYS * 24 * 60 * 60 * 1000) ); + const siteEventRawCutoffDay = utcDay(addUtcDays(now, -SITE_EVENT_RAW_RETENTION_DAYS)); + const tgcRawCutoffDay = utcDay(addUtcDays(now, -TGC_SITE_EVENT_RAW_RETENTION_DAYS)); await Promise.all([ db.prepare("DELETE FROM pageview_events_raw WHERE received_day < ?").bind(rawCutoffDay).run(), db.prepare("DELETE FROM pageview_rate_limit WHERE minute_bucket < ?").bind(rateLimitCutoffMinute).run(), db.prepare("DELETE FROM site_event_rate_limit WHERE minute_bucket < ?").bind(siteEventRateLimitCutoffMinute).run(), + db.prepare("UPDATE site_events_raw SET ip_hash = NULL, user_agent_hash = NULL, request_id = NULL WHERE ip_hash IS NOT NULL OR user_agent_hash IS NOT NULL OR request_id IS NOT NULL").run(), + db.prepare("DELETE FROM site_events_raw WHERE site_key = 'tgc_site' AND received_day < ?").bind(tgcRawCutoffDay).run(), + db.prepare("DELETE FROM site_events_raw WHERE site_key <> 'tgc_site' AND received_day < ?").bind(siteEventRawCutoffDay).run(), ]); } @@ -3414,10 +3445,7 @@ async function processSiteEventIngest( const receivedAt = new Date(); const receivedAtIso = receivedAt.toISOString(); const receivedDay = utcDay(receivedAt); - const [ipHash, userAgentHash] = await Promise.all([ - requestContext.clientIp ? sha256Hex(requestContext.clientIp) : Promise.resolve(null), - requestContext.userAgent ? sha256Hex(requestContext.userAgent) : Promise.resolve(null), - ]); + const minuteBucket = utcMinuteBucket(receivedAt); const parsedBody = readAndParsePageviewBody(capture.raw); if (!parsedBody.ok) { @@ -3425,15 +3453,23 @@ async function processSiteEventIngest( } const normalized = parseCanonicalEventPayload(parsedBody.payload); - if (!normalized) { + const site = normalized ? getSiteByKey(normalized.site_key) : undefined; + if (!normalized || !site) { + return; + } + if ( + normalized.site_key === "tgc_site" + && (!requestContext.origin || !site.allowed_origins.includes(requestContext.origin)) + ) { return; } let accepted = 1; let dropReason: string | null = null; - - if (ipHash) { - const rateLimitCount = await incrementSiteEventRateLimitBucket(env.DB, utcMinuteBucket(receivedAt), ipHash); + const rateLimitSecret = env.TELEMETRY_RATE_LIMIT_SECRET?.trim(); + if (requestContext.clientIp && rateLimitSecret) { + const rateIdentifier = await keyedRateIdentifier(rateLimitSecret, minuteBucket, requestContext.clientIp); + const rateLimitCount = await incrementSiteEventRateLimitBucket(env.DB, minuteBucket, rateIdentifier); if (rateLimitCount > SITE_EVENT_RATE_LIMIT_PER_MINUTE) { accepted = 0; dropReason = "rate_limited"; @@ -3447,11 +3483,11 @@ async function processSiteEventIngest( received_day: receivedDay, referrer_domain: parseReferrerDomain(normalized.referrer), country: requestContext.country, - ip_hash: ipHash, - user_agent_hash: userAgentHash, + ip_hash: null, + user_agent_hash: null, accepted, drop_reason: dropReason, - request_id: requestContext.requestId, + request_id: null, ingest_version: SITE_EVENT_INGEST_VERSION, }; @@ -3945,6 +3981,149 @@ async function buildSiteReport( }); } +type TgcAnalyticsAggregateRow = { + events: number | null; + page_views: number | null; + sessions: number | null; + visitors: number | null; + first_visits: number | null; + returning_visits: number | null; + service_interest: number | null; + form_starts: number | null; + submit_attempts: number | null; + submit_successes: number | null; + submit_failures: number | null; + scroll_90: number | null; + engaged_60: number | null; + avg_page_load_ms: number | null; + avg_lcp_ms: number | null; + avg_cls: number | null; +}; + +type TgcTopRow = { value: string | null; events: number | null }; + +async function queryTgcAnalyticsWindow(db: D1Database, startDay: string, endDay: string) { + const row = await db.prepare( + `SELECT + COUNT(*) AS events, + SUM(CASE WHEN event_name = 'page_view' THEN 1 ELSE 0 END) AS page_views, + COUNT(DISTINCT CASE WHEN session_id IS NOT NULL THEN session_id END) AS sessions, + COUNT(DISTINCT CASE WHEN anon_user_id IS NOT NULL THEN anon_user_id END) AS visitors, + SUM(CASE WHEN event_name = 'first_visit' THEN 1 ELSE 0 END) AS first_visits, + SUM(CASE WHEN event_name = 'returning_visit' THEN 1 ELSE 0 END) AS returning_visits, + SUM(CASE WHEN event_name IN ('services_interest','infrastructure_cta_click','infrastructure_package_interest','ops_care_interest','audit_cta_click') THEN 1 ELSE 0 END) AS service_interest, + SUM(CASE WHEN event_name IN ('form_start','infrastructure_form_start','audit_form_start') THEN 1 ELSE 0 END) AS form_starts, + SUM(CASE WHEN event_name = 'form_submit_attempt' THEN 1 ELSE 0 END) AS submit_attempts, + SUM(CASE WHEN event_name = 'form_submit_success' THEN 1 ELSE 0 END) AS submit_successes, + SUM(CASE WHEN event_name IN ('form_submit_failure','form_submit_fallback') THEN 1 ELSE 0 END) AS submit_failures, + SUM(CASE WHEN event_name = 'scroll_depth' AND CAST(event_value AS INTEGER) >= 90 THEN 1 ELSE 0 END) AS scroll_90, + SUM(CASE WHEN event_name = 'engaged_time' AND CAST(event_value AS INTEGER) >= 60 THEN 1 ELSE 0 END) AS engaged_60, + AVG(CASE WHEN event_name = 'web_vital_page_load_ms' THEN CAST(event_value AS REAL) END) AS avg_page_load_ms, + AVG(CASE WHEN event_name = 'web_vital_lcp_ms' THEN CAST(event_value AS REAL) END) AS avg_lcp_ms, + AVG(CASE WHEN event_name = 'web_vital_cls' THEN CAST(event_value AS REAL) END) AS avg_cls + FROM site_events_raw + WHERE site_key = 'tgc_site' AND accepted = 1 AND test_mode = 0 AND received_day BETWEEN ? AND ?` + ).bind(startDay, endDay).first(); + + const count = (value: number | null | undefined) => Number(value ?? 0); + const attempts = count(row?.submit_attempts); + const successes = count(row?.submit_successes); + return { + start_day: startDay, + end_day: endDay, + events: count(row?.events), + page_views: count(row?.page_views), + sessions: count(row?.sessions), + visitors: count(row?.visitors), + first_visits: count(row?.first_visits), + returning_visits: count(row?.returning_visits), + commercial_intent: count(row?.service_interest), + funnel: { + form_starts: count(row?.form_starts), + submit_attempts: attempts, + submit_successes: successes, + submit_failures_or_fallbacks: count(row?.submit_failures), + submit_success_rate: attempts > 0 ? successes / attempts : null, + }, + engagement: { + scroll_90: count(row?.scroll_90), + engaged_60_seconds: count(row?.engaged_60), + }, + performance: { + avg_page_load_ms: row?.avg_page_load_ms === null || row?.avg_page_load_ms === undefined ? null : Number(row.avg_page_load_ms), + avg_lcp_ms: row?.avg_lcp_ms === null || row?.avg_lcp_ms === undefined ? null : Number(row.avg_lcp_ms), + avg_cls: row?.avg_cls === null || row?.avg_cls === undefined ? null : Number(row.avg_cls), + }, + }; +} + +async function queryTgcTop( + db: D1Database, + expression: "event_name" | "path" | "source" | "utm_campaign" | "section", + startDay: string, + endDay: string +) { + const sqlExpression = expression === "source" + ? "COALESCE(NULLIF(src, ''), NULLIF(utm_source, ''), '(direct)')" + : expression === "section" + ? "event_value" + : expression; + const extra = expression === "section" ? " AND event_name = 'section_view'" : ""; + const result = await db.prepare( + `SELECT ${sqlExpression} AS value, COUNT(*) AS events + FROM site_events_raw + WHERE site_key = 'tgc_site' AND accepted = 1 AND test_mode = 0 + AND received_day BETWEEN ? AND ?${extra} + AND ${sqlExpression} IS NOT NULL AND ${sqlExpression} <> '' + GROUP BY ${sqlExpression} + ORDER BY events DESC, value ASC + LIMIT 8` + ).bind(startDay, endDay).all(); + return (result.results ?? []).map((row) => ({ value: row.value ?? "(unknown)", events: Number(row.events ?? 0) })); +} + +async function buildTgcAnalyticsReport(db: D1Database, now: Date) { + const today = utcDay(now); + const last7Start = utcDay(addUtcDays(now, -6)); + const last30Start = utcDay(addUtcDays(now, -29)); + const [todayWindow, last7, last30, topEvents, topPaths, topSources, topCampaigns, topSections, health] = await Promise.all([ + queryTgcAnalyticsWindow(db, today, today), + queryTgcAnalyticsWindow(db, last7Start, today), + queryTgcAnalyticsWindow(db, last30Start, today), + queryTgcTop(db, "event_name", last30Start, today), + queryTgcTop(db, "path", last30Start, today), + queryTgcTop(db, "source", last30Start, today), + queryTgcTop(db, "utm_campaign", last30Start, today), + queryTgcTop(db, "section", last30Start, today), + db.prepare( + "SELECT MAX(received_at) AS last_received_at, SUM(CASE WHEN accepted = 0 AND drop_reason = 'rate_limited' THEN 1 ELSE 0 END) AS dropped_rate_limited FROM site_events_raw WHERE site_key = 'tgc_site' AND received_day BETWEEN ? AND ?" + ).bind(last30Start, today).first<{ last_received_at: string | null; dropped_rate_limited: number | null }>(), + ]); + + return { + view: "tgc" as const, + generated_at: now.toISOString(), + site_key: "tgc_site", + semantics: "consented_page_execution_events_not_edge_traffic", + windows: { today: todayWindow, last_7_days: last7, last_30_days: last30 }, + top_30_days: { + events: topEvents, + paths: topPaths, + sources: topSources, + campaigns: topCampaigns, + sections: topSections, + }, + health: { + last_received_at: health?.last_received_at ?? null, + dropped_rate_limited_30d: Number(health?.dropped_rate_limited ?? 0), + test_mode_excluded: true, + identifiers_exposed: false, + raw_event_retention_days: TGC_SITE_EVENT_RAW_RETENTION_DAYS, + rate_identifier_retention_days: SITE_EVENT_RATE_LIMIT_RETENTION_DAYS, + }, + }; +} + async function buildSourceHealthReport( db: D1Database, now: Date @@ -5792,7 +5971,8 @@ export default { if ( reportRequest.view !== "source_health" && reportRequest.view !== "asset" && - reportRequest.view !== "monthly" + reportRequest.view !== "monthly" && + reportRequest.view !== "tgc" ) { await refreshPreviousCompletedTrafficBestEffort(env, now); } @@ -5804,6 +5984,8 @@ export default { ? await buildFleetReport(env.DB, now) : reportRequest.view === "site" ? await buildSiteReport(env.DB, env.BUSCORE_LEADS_DB, now, reportRequest.siteEventFilter) + : reportRequest.view === "tgc" + ? await buildTgcAnalyticsReport(env.DB, now) : reportRequest.view === "asset" ? await buildAssetReport(env.DB, env.BUSCORE_LEADS_DB, now) : reportRequest.view === "monthly" diff --git a/tests/tgc-analytics.test.mjs b/tests/tgc-analytics.test.mjs new file mode 100644 index 0000000..e9bbf6f --- /dev/null +++ b/tests/tgc-analytics.test.mjs @@ -0,0 +1,75 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; + +import { + normalizeReportView, + resolveReportRequest, + parseCanonicalEventPayload, + sanitizeAnalyticsLocation, +} from "../dist/index.js"; + +function payload(overrides = {}) { + return { + type: "event", + site_key: "tgc_site", + event_name: "page_view", + client_ts: "2026-07-18T12:00:00.000Z", + path: "/services.html", + url: "https://truegoodcraft.ca/services.html?utm_source=test#offers", + referrer: "https://example.com/article?person=value", + src: "newsletter", + utm: { source: "newsletter", medium: "email", campaign: "summer" }, + device: "desktop", + viewport: "1440x900", + lang: "en-CA", + tz: "America/Toronto", + anon_user_id: "v_a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", + session_id: "s_1eebc999-9c0b-4ef8-bb6d-6bb9bd380a11", + is_new_user: true, + event_value: "services", + test_mode: false, + ...overrides, + }; +} + +test("TGC report view resolves without a site key", () => { + assert.equal(normalizeReportView("tgc"), "tgc"); + assert.deepEqual(resolveReportRequest(new URL("https://lighthouse.test/report?view=tgc")), { + ok: true, + view: "tgc", + }); +}); + +test("TGC payload accepts consent-created IDs and strips URL detail", () => { + const parsed = parseCanonicalEventPayload(payload()); + assert.ok(parsed); + assert.equal(parsed.url, "https://truegoodcraft.ca/services.html"); + assert.equal(parsed.referrer, "https://example.com/article"); + assert.equal(parsed.anon_user_id, "v_a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"); + assert.equal(parsed.session_id, "s_1eebc999-9c0b-4ef8-bb6d-6bb9bd380a11"); +}); + +test("TGC payload rejects unknown events, mismatched paths, and foreign origins", () => { + assert.equal(parseCanonicalEventPayload(payload({ event_name: "capture_everything" })), null); + assert.equal(parseCanonicalEventPayload(payload({ path: "/contact.html" })), null); + assert.equal(parseCanonicalEventPayload(payload({ url: "https://evil.example/services.html" })), null); +}); + +test("analytics location permits only HTTP(S) origin and path", () => { + assert.equal(sanitizeAnalyticsLocation("https://truegoodcraft.ca/a?x=1#b"), "https://truegoodcraft.ca/a"); + assert.equal(sanitizeAnalyticsLocation("javascript:alert(1)"), null); + assert.equal(sanitizeAnalyticsLocation("", true), ""); +}); + +test("source declares bounded retention and no raw request identifiers for TGC events", () => { + const source = fs.readFileSync(new URL("../src/index.ts", import.meta.url), "utf8"); + assert.match(source, /TGC_SITE_EVENT_RAW_RETENTION_DAYS = 90/); + assert.match(source, /SITE_EVENT_RATE_LIMIT_RETENTION_DAYS = 2/); + assert.match(source, /ip_hash: null/); + assert.match(source, /user_agent_hash: null/); + assert.match(source, /request_id: null/); + assert.match(source, /UPDATE site_events_raw SET ip_hash = NULL, user_agent_hash = NULL, request_id = NULL/); + assert.match(source, /view: "tgc" as const/); + assert.match(source, /identifiers_exposed: false/); +});