From f722f63da757f52c8d93c353a9b25b46b7c2a25c Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Thu, 29 Oct 2026 04:25:56 +0000 Subject: [PATCH] search: one crawlproof ad line, and our own sites backfill as Featured (0.12.0) Search results now carry pit.world's crawlproof text_link unit after the third web result (after the grid/list on media tabs): the same sandboxed cross-origin frame as the other pages, origin-only referrer, so the ad never sees the query. Analytics still never runs on search. A search that matches none of our ring sites gets one of them as a Featured box below the results (stable per query, rotating across queries). Featured is ours and unpaid; paid ads come only from the crawlproof unit. Privacy page updated. Co-Authored-By: Claude Opus 5.5 --- package.json | 2 +- src/featured.ts | 15 +++++++++++++++ src/html.ts | 40 +++++++++++++++++++++++++--------------- src/server.ts | 9 ++++++--- test/featured.test.ts | 11 ++++++++++- test/pit.test.ts | 22 ++++++++++++++++++---- 6 files changed, 75 insertions(+), 24 deletions(-) diff --git a/package.json b/package.json index abea64b..73f6245 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "pitworld", - "version": "0.11.2", + "version": "0.12.0", "mcpName": "world.pit/pit", "description": "pit.world MCP server: private web search and Moshpit name lookups for AI agents (stdio, no API key). Also a web app, API, CLI and TUI.", "license": "MIT", diff --git a/src/featured.ts b/src/featured.ts index 685ab8b..4b58087 100644 --- a/src/featured.ts +++ b/src/featured.ts @@ -46,3 +46,18 @@ export function featuredFor(query: string, max = 2, sites: RingSite[] = RING): R .slice(0, max) .map((s) => s.site) } + +/** + * One of our own sites for a search that matched none of them, so the space never goes to waste + * (Anthony 2026-10-29: search "is a good place to backfill our properties listings, which are + * featured, not ads"). Labelled Featured like a match, never Sponsored: paid ads come only from the + * crawlproof unit. The pick is stable for a query and spreads across queries; pit.world itself and + * claimed listings are left out. + */ +export function backfillFor(query: string, sites: RingSite[] = RING): RingSite | undefined { + const pool = sites.filter((s) => s.name !== 'pit.world' && !s.listing) + if (!pool.length) return undefined + let h = 2166136261 + for (const ch of flat(query)) h = Math.imul(h ^ ch.charCodeAt(0), 16777619) >>> 0 + return pool[h % pool.length] +} diff --git a/src/html.ts b/src/html.ts index 5888c74..443df20 100644 --- a/src/html.ts +++ b/src/html.ts @@ -51,9 +51,11 @@ const fmt = (n: number) => n.toLocaleString('en-US') /** * crawlproof analytics and ads, on pit.world's own pages only: the *.pit.world relay - * (proxy.ts) serves other people's sites and never gets either. Neither runs on search - * results, whose URL is the query. Ads are a cross-origin sandboxed iframe, so - * the ad sees pit.world's origin and nothing else. CRAWLPROOF_AD_SLOT=off turns ads off. + * (proxy.ts) serves other people's sites and never gets either. Analytics never runs on + * search results, whose URL is the query. The ad does (one line after the third result, + * Anthony 2026-10-29), because it is a cross-origin sandboxed iframe sent with an + * origin-only referrer: the ad sees pit.world's origin and nothing else, never the query. + * CRAWLPROOF_AD_SLOT=off turns ads off. */ const TRACKER_SITE = '81a8a3cd-8d5b-4c5f-bb29-c21a1bbb6a05' export const AD_SLOT = process.env.CRAWLPROOF_AD_SLOT ?? '782907b4-bf27-44bf-b202-df196c4b207e' @@ -290,7 +292,7 @@ function mediaList(vertical: Exclude, hits: Media[]): string { .join('')}` } -export function mediaPage(options: { query: string; vertical: Exclude; result: MediaResult | null; promos: Promo[]; error: string | null; featured?: RingSite[] }): string { +export function mediaPage(options: { query: string; vertical: Exclude; result: MediaResult | null; promos: Promo[]; error: string | null; featured?: RingSite[]; backfill?: RingSite }): string { const { query, vertical, result, promos, error } = options const hits = result?.hits ?? [] const label = VERTICALS.find((v) => v.id === vertical)!.label @@ -304,13 +306,26 @@ ${tabs(query, vertical)} ${promoStrip(promos, vertical)} ${error ? `

${esc(error)}

` : ''} ${hits.length ? mediaList(vertical, hits) : ''} + ${hits.length ? adUnit() : ''} + ${featuredBox(options.backfill ? [options.backfill] : undefined)} ${!error && hits.length === 0 ? `

No ${label.toLowerCase()} found. Try other words.

` : ''} ${result ? `

${label} from the Google index, fetched by pit.world. Thumbnails are loaded through pit.world too, so Google and YouTube never see your IP address, browser or cookies.

` : ''} ` return layout({ title: `${query} · ${label} · pit.world`, description: `Private ${label.toLowerCase()} results for ${query}`, path: '/search', body, bodyClass: 'serp', noindex: true, track: false }) } -export function resultsPage(options: { query: string; result: SearchResult | null; resolution: Resolution | null; error: string | null; card?: string; featured?: RingSite[] }): string { +/** Web results before the one ad line: in the content, never at the top, never by the footer. */ +const AD_AFTER = 3 + +function hitItem(h: SearchResult['hits'][number]): string { + return `
  • + ${esc(h.title)} +
    ${esc(h.host)}${(h.sources ?? []).map((src) => `${esc(LABELS[src] ?? src)}`).join('')}
    + ${h.description ? `

    ${esc(h.description)}

    ` : ''} +
    • ` +} + +export function resultsPage(options: { query: string; result: SearchResult | null; resolution: Resolution | null; error: string | null; card?: string; featured?: RingSite[]; backfill?: RingSite }): string { const { query, result, resolution, error } = options const hits = result?.hits ?? [] const domain = !resolution?.registered && /^[a-z0-9.-]+\.[a-z]{2,}$/i.test(query) ? query.toLowerCase() : null @@ -326,16 +341,11 @@ ${tabs(query, 'web')} ${domain ? `

    Go straight to ${esc(domain)}${options.card ? '' : ` · business listing`}

    ` : ''} ${error ? `

    ${esc(error)}

    ` : ''}
      - ${hits - .map( - (h) => `
    1. - ${esc(h.title)} -
      ${esc(h.host)}${(h.sources ?? []).map((src) => `${esc(LABELS[src] ?? src)}`).join('')}
      - ${h.description ? `

      ${esc(h.description)}

      ` : ''} -
      2. `, - ) - .join('')} + ${hits.slice(0, AD_AFTER).map(hitItem).join('')}
    + ${hits.length ? adUnit() : ''} + ${hits.length > AD_AFTER ? `
      ${hits.slice(AD_AFTER).map(hitItem).join('')}
    ` : ''} + ${featuredBox(options.backfill ? [options.backfill] : undefined)} ${!error && hits.length === 0 ? '

    Nothing found. Try other words, or a bang like !ddg.

    ' : ''}

    Try this search on DuckDuckGo, Brave or Google, without a referrer.

    ${result?.provider ? `

    Results from the ${listFormat.format(result.provider.split('+').map((p) => LABELS[p as Provider] ?? p))} ${result.provider.includes('+') ? 'indexes' : 'index'}, fetched by pit.world and merged. Your IP address, browser and cookies were not sent.

    ` : ''} @@ -361,7 +371,7 @@ export function privacyPage(): string {
  • Your searches, to anyone else. Search results pages load nothing from a third party.

    • Analytics and ads, outside search

    -

    pit.world's other pages (this one, the home page, the docs) load crawlproof's page-view counter and show a crawlproof ad in a sandboxed frame. The counter reports the page address and keeps a random visitor id in your browser's local storage (not a cookie); neither one runs on search results, so neither ever sees a query. The ad frame is told only that it is on pit.world. Fonts and every other script still come from pit.world.

    +

    pit.world's other pages (this one, the home page, the docs) load crawlproof's page-view counter and show a crawlproof ad in a sandboxed frame. The counter reports the page address and keeps a random visitor id in your browser's local storage (not a cookie); it never runs on search results, so it never sees a query. Search results show one crawlproof ad line, marked Sponsored, in the same sandboxed frame: the frame is told only that it is on pit.world, never what you searched for. Boxes marked "Featured on pit.world" are our own sites, not ads, and nobody pays for them. Fonts and every other script still come from pit.world.

    What happens to a search

    1. Your browser sends the words to pit.world over HTTPS.
      2. diff --git a/src/server.ts b/src/server.ts index 9bc2b93..6d2dd69 100644 --- a/src/server.ts +++ b/src/server.ts @@ -17,7 +17,7 @@ import { snippetFor } from './footer.js' import { neighbors, randomSite, ringJson, ringSites, siteHref, useListings } from './ring.js' import { RingSync } from './ring-sync.js' import { liveCerts, proxiedName, proxyRequest, registrySweep, Targets } from './proxy.js' -import { featuredFor } from './featured.js' +import { backfillFor, featuredFor } from './featured.js' import { Promoted } from './promoted.js' import { listingCard } from './listings-html.js' import { mountListings } from './listings-routes.js' @@ -326,12 +326,15 @@ export function createApp(deps: Deps) { c.header('cache-control', 'no-store') if (vertical !== 'web') { const [{ result, error }, promos] = await Promise.all([runMedia(vertical, query), promoted.for(query, vertical)]) - return c.html(mediaPage({ query, vertical, result, promos, error, featured: featuredFor(query) }), result || promos.length ? 200 : 503) + const mediaFeatured = featuredFor(query) + const mediaBackfill = mediaFeatured.length ? undefined : backfillFor(query) + return c.html(mediaPage({ query, vertical, result, promos, error, featured: mediaFeatured, ...(mediaBackfill ? { backfill: mediaBackfill } : {}) }), result || promos.length ? 200 : 503) } const [{ result, error }, resolution] = await Promise.all([runSearch(query), looksLikeName(query) ? registry.resolve(query) : Promise.resolve(null)]) const claimed = deps.listings?.claimedOnly(query) const featured = featuredFor(query) - return c.html(resultsPage({ query, result, resolution, error, featured, ...(claimed ? { card: listingCard(claimed) } : {}) }), result || resolution || claimed || featured.length ? 200 : 503) + const backfill = featured.length ? undefined : backfillFor(query) + return c.html(resultsPage({ query, result, resolution, error, featured, ...(backfill ? { backfill } : {}), ...(claimed ? { card: listingCard(claimed) } : {}) }), result || resolution || claimed || featured.length ? 200 : 503) }) // --- JSON API --------------------------------------------------------------- diff --git a/test/featured.test.ts b/test/featured.test.ts index 32b9a5a..e2d8e93 100644 --- a/test/featured.test.ts +++ b/test/featured.test.ts @@ -1,5 +1,14 @@ import { describe, expect, test } from 'bun:test' -import { featuredFor } from '../src/featured.ts' +import { backfillFor, featuredFor } from '../src/featured.ts' + +describe('backfill for a search that matched none of our sites', () => { + test('is stable for a query, varies across queries, and is never pit.world itself', () => { + expect(backfillFor('kitchen remodel')?.name).toBe(backfillFor('Kitchen remodel')?.name) + const picks = new Set(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'].map((q) => backfillFor(`query ${q}`)?.name)) + expect(picks.size).toBeGreaterThan(2) + expect(picks.has('pit.world')).toBe(false) + }) +}) const names = (q: string) => featuredFor(q).map((s) => s.name) diff --git a/test/pit.test.ts b/test/pit.test.ts index 1593205..4536e4b 100644 --- a/test/pit.test.ts +++ b/test/pit.test.ts @@ -204,10 +204,23 @@ describe('crawlproof analytics and ads', () => { expect(csp).toContain("img-src 'self' data: https://built.indie.pit.world https://pwamart.com;") }) - it('never runs on search results, whose URL is the query', async () => { + it('runs no analytics on search results, and the one ad never sees the query', async () => { const { request } = app() const html = await (await request('/search?q=moshpit+network')).text() - expect(html).not.toContain('crawlproof.com') + expect(html).not.toContain('crawlproof.com/stats.js') + const frames = html.match(/