search: crawlproof ad line + our sites backfill as Featured (0.12.0)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "pitworld",
-  "version": "0.11.2",
+  "version": "0.12.0",
   "mcpName": "world.pit/pit",
   "description": "pit.world MCP server: private web search and Moshpit name lookups for AI agents (stdio, no API key). Also a web app, API, CLI and TUI.",
   "license": "MIT",

src/featured.ts

@@ -46,3 +46,18 @@ export function featuredFor(query: string, max = 2, sites: RingSite[] = RING): R
     .slice(0, max)
     .map((s) => s.site)
 }
+
+/**
+ * One of our own sites for a search that matched none of them, so the space never goes to waste
+ * (Anthony 2026-10-29: search "is a good place to backfill our properties listings, which are
+ * featured, not ads"). Labelled Featured like a match, never Sponsored: paid ads come only from the
+ * crawlproof unit. The pick is stable for a query and spreads across queries; pit.world itself and
+ * claimed listings are left out.
+ */
+export function backfillFor(query: string, sites: RingSite[] = RING): RingSite | undefined {
+  const pool = sites.filter((s) => s.name !== 'pit.world' && !s.listing)
+  if (!pool.length) return undefined
+  let h = 2166136261
+  for (const ch of flat(query)) h = Math.imul(h ^ ch.charCodeAt(0), 16777619) >>> 0
+  return pool[h % pool.length]
+}

src/html.ts

@@ -51,9 +51,11 @@ const fmt = (n: number) => n.toLocaleString('en-US')
 
 /**
  * crawlproof analytics and ads, on pit.world's own pages only: the *.pit.world relay
- * (proxy.ts) serves other people's sites and never gets either. Neither runs on search
- * results, whose URL is the query. Ads are a cross-origin sandboxed iframe, so
- * the ad sees pit.world's origin and nothing else. CRAWLPROOF_AD_SLOT=off turns ads off.
+ * (proxy.ts) serves other people's sites and never gets either. Analytics never runs on
+ * search results, whose URL is the query. The ad does (one line after the third result,
+ * Anthony 2026-10-29), because it is a cross-origin sandboxed iframe sent with an
+ * origin-only referrer: the ad sees pit.world's origin and nothing else, never the query.
+ * CRAWLPROOF_AD_SLOT=off turns ads off.
  */
 const TRACKER_SITE = '81a8a3cd-8d5b-4c5f-bb29-c21a1bbb6a05'
 export const AD_SLOT = process.env.CRAWLPROOF_AD_SLOT ?? '782907b4-bf27-44bf-b202-df196c4b207e'
@@ -290,7 +292,7 @@ function mediaList(vertical: Exclude<Vertical, 'web'>, hits: Media[]): string {
     .join('')}</ol>`
 }
 
-export function mediaPage(options: { query: string; vertical: Exclude<Vertical, 'web'>; result: MediaResult | null; promos: Promo[]; error: string | null; featured?: RingSite[] }): string {
+export function mediaPage(options: { query: string; vertical: Exclude<Vertical, 'web'>; result: MediaResult | null; promos: Promo[]; error: string | null; featured?: RingSite[]; backfill?: RingSite }): string {
   const { query, vertical, result, promos, error } = options
   const hits = result?.hits ?? []
   const label = VERTICALS.find((v) => v.id === vertical)!.label
@@ -304,13 +306,26 @@ ${tabs(query, vertical)}
   ${promoStrip(promos, vertical)}
   ${error ? `<p class="notice">${esc(error)}</p>` : ''}
   ${hits.length ? mediaList(vertical, hits) : ''}
+  ${hits.length ? adUnit() : ''}
+  ${featuredBox(options.backfill ? [options.backfill] : undefined)}
   ${!error && hits.length === 0 ? `<p class="notice">No ${label.toLowerCase()} found. Try other words.</p>` : ''}
   ${result ? `<p class="via">${label} from the Google index, fetched by pit.world. Thumbnails are loaded through pit.world too, so Google and YouTube never see your IP address, browser or cookies.</p>` : ''}
 </main>`
   return layout({ title: `${query} · ${label} · pit.world`, description: `Private ${label.toLowerCase()} results for ${query}`, path: '/search', body, bodyClass: 'serp', noindex: true, track: false })
 }
 
-export function resultsPage(options: { query: string; result: SearchResult | null; resolution: Resolution | null; error: string | null; card?: string; featured?: RingSite[] }): string {
+/** Web results before the one ad line: in the content, never at the top, never by the footer. */
+const AD_AFTER = 3
+
+function hitItem(h: SearchResult['hits'][number]): string {
+  return `<li>
+      <a class="title" href="${esc(h.url)}" rel="noreferrer noopener">${esc(h.title)}</a>
+      <div class="url">${esc(h.host)}${(h.sources ?? []).map((src) => `<span class="src">${esc(LABELS[src] ?? src)}</span>`).join('')}</div>
+      ${h.description ? `<p>${esc(h.description)}</p>` : ''}
+    </li>`
+}
+
+export function resultsPage(options: { query: string; result: SearchResult | null; resolution: Resolution | null; error: string | null; card?: string; featured?: RingSite[]; backfill?: RingSite }): string {
   const { query, result, resolution, error } = options
   const hits = result?.hits ?? []
   const domain = !resolution?.registered && /^[a-z0-9.-]+\.[a-z]{2,}$/i.test(query) ? query.toLowerCase() : null
@@ -326,16 +341,11 @@ ${tabs(query, 'web')}
   ${domain ? `<p class="go">Go straight to <a href="https://${esc(domain)}/" rel="noreferrer noopener">${esc(domain)}</a>${options.card ? '' : ` · <a href="/b/${encodeURIComponent(domain)}">business listing</a>`}</p>` : ''}
   ${error ? `<p class="notice">${esc(error)}</p>` : ''}
   <ol class="hits">
-    ${hits
-      .map(
-        (h) => `<li>
-      <a class="title" href="${esc(h.url)}" rel="noreferrer noopener">${esc(h.title)}</a>
-      <div class="url">${esc(h.host)}${(h.sources ?? []).map((src) => `<span class="src">${esc(LABELS[src] ?? src)}</span>`).join('')}</div>
-      ${h.description ? `<p>${esc(h.description)}</p>` : ''}
-    </li>`,
-      )
-      .join('')}
+    ${hits.slice(0, AD_AFTER).map(hitItem).join('')}
   </ol>
+  ${hits.length ? adUnit() : ''}
+  ${hits.length > AD_AFTER ? `<ol class="hits" start="${AD_AFTER + 1}">${hits.slice(AD_AFTER).map(hitItem).join('')}</ol>` : ''}
+  ${featuredBox(options.backfill ? [options.backfill] : undefined)}
   ${!error && hits.length === 0 ? '<p class="notice">Nothing found. Try other words, or a bang like <code>!ddg</code>.</p>' : ''}
   <p class="elsewhere">Try this search on <a href="/search?q=${encodeURIComponent(`${query} !ddg`)}">DuckDuckGo</a>, <a href="/search?q=${encodeURIComponent(`${query} !b`)}">Brave</a> or <a href="/search?q=${encodeURIComponent(`${query} !g`)}">Google</a>, without a referrer.</p>
   ${result?.provider ? `<p class="via">Results from the ${listFormat.format(result.provider.split('+').map((p) => LABELS[p as Provider] ?? p))} ${result.provider.includes('+') ? 'indexes' : 'index'}, fetched by pit.world and merged. Your IP address, browser and cookies were not sent.</p>` : ''}
@@ -361,7 +371,7 @@ export function privacyPage(): string {
 <li><b>Your searches, to anyone else.</b> Search results pages load nothing from a third party.</li>
 </ul>
 <h2 id="analytics">Analytics and ads, outside search</h2>
-<p>pit.world's other pages (this one, the home page, the docs) load <a href="https://crawlproof.com">crawlproof</a>'s page-view counter and show a crawlproof ad in a sandboxed frame. The counter reports the page address and keeps a random visitor id in your browser's local storage (not a cookie); neither one runs on search results, so neither ever sees a query. The ad frame is told only that it is on pit.world. Fonts and every other script still come from pit.world.</p>
+<p>pit.world's other pages (this one, the home page, the docs) load <a href="https://crawlproof.com">crawlproof</a>'s page-view counter and show a crawlproof ad in a sandboxed frame. The counter reports the page address and keeps a random visitor id in your browser's local storage (not a cookie); it never runs on search results, so it never sees a query. Search results show one crawlproof ad line, marked Sponsored, in the same sandboxed frame: the frame is told only that it is on pit.world, never what you searched for. Boxes marked "Featured on pit.world" are our own sites, not ads, and nobody pays for them. Fonts and every other script still come from pit.world.</p>
 <h2>What happens to a search</h2>
 <ol>
 <li>Your browser sends the words to pit.world over HTTPS.</li>

src/server.ts

@@ -17,7 +17,7 @@ import { snippetFor } from './footer.js'
 import { neighbors, randomSite, ringJson, ringSites, siteHref, useListings } from './ring.js'
 import { RingSync } from './ring-sync.js'
 import { liveCerts, proxiedName, proxyRequest, registrySweep, Targets } from './proxy.js'
-import { featuredFor } from './featured.js'
+import { backfillFor, featuredFor } from './featured.js'
 import { Promoted } from './promoted.js'
 import { listingCard } from './listings-html.js'
 import { mountListings } from './listings-routes.js'
@@ -326,12 +326,15 @@ export function createApp(deps: Deps) {
     c.header('cache-control', 'no-store')
     if (vertical !== 'web') {
       const [{ result, error }, promos] = await Promise.all([runMedia(vertical, query), promoted.for(query, vertical)])
-      return c.html(mediaPage({ query, vertical, result, promos, error, featured: featuredFor(query) }), result || promos.length ? 200 : 503)
+      const mediaFeatured = featuredFor(query)
+      const mediaBackfill = mediaFeatured.length ? undefined : backfillFor(query)
+      return c.html(mediaPage({ query, vertical, result, promos, error, featured: mediaFeatured, ...(mediaBackfill ? { backfill: mediaBackfill } : {}) }), result || promos.length ? 200 : 503)
     }
     const [{ result, error }, resolution] = await Promise.all([runSearch(query), looksLikeName(query) ? registry.resolve(query) : Promise.resolve(null)])
     const claimed = deps.listings?.claimedOnly(query)
     const featured = featuredFor(query)
-    return c.html(resultsPage({ query, result, resolution, error, featured, ...(claimed ? { card: listingCard(claimed) } : {}) }), result || resolution || claimed || featured.length ? 200 : 503)
+    const backfill = featured.length ? undefined : backfillFor(query)
+    return c.html(resultsPage({ query, result, resolution, error, featured, ...(backfill ? { backfill } : {}), ...(claimed ? { card: listingCard(claimed) } : {}) }), result || resolution || claimed || featured.length ? 200 : 503)
   })
 
   // --- JSON API ---------------------------------------------------------------

test/featured.test.ts

@@ -1,5 +1,14 @@
 import { describe, expect, test } from 'bun:test'
-import { featuredFor } from '../src/featured.ts'
+import { backfillFor, featuredFor } from '../src/featured.ts'
+
+describe('backfill for a search that matched none of our sites', () => {
+  test('is stable for a query, varies across queries, and is never pit.world itself', () => {
+    expect(backfillFor('kitchen remodel')?.name).toBe(backfillFor('Kitchen  remodel')?.name)
+    const picks = new Set(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'].map((q) => backfillFor(`query ${q}`)?.name))
+    expect(picks.size).toBeGreaterThan(2)
+    expect(picks.has('pit.world')).toBe(false)
+  })
+})
 
 const names = (q: string) => featuredFor(q).map((s) => s.name)
 

test/pit.test.ts

@@ -204,10 +204,23 @@ describe('crawlproof analytics and ads', () => {
     expect(csp).toContain("img-src 'self' data: https://built.indie.pit.world https://pwamart.com;")
   })
 
-  it('never runs on search results, whose URL is the query', async () => {
+  it('runs no analytics on search results, and the one ad never sees the query', async () => {
     const { request } = app()
     const html = await (await request('/search?q=moshpit+network')).text()
-    expect(html).not.toContain('crawlproof.com')
+    expect(html).not.toContain('crawlproof.com/stats.js')
+    const frames = html.match(/<iframe [^>]*>/g) ?? []
+    expect(frames).toHaveLength(1)
+    expect(frames[0]).toContain('https://crawlproof.com/api/ads/frame?slot=')
+    expect(frames[0]).toContain('format=text_link')
+    expect(frames[0]).toContain('sandbox=')
+    expect(frames[0]).not.toMatch(/moshpit|network/i)
+  })
+
+  it('backfills a search that matches none of our sites with one Featured site, never an ad', async () => {
+    const { request } = app()
+    const html = await (await request('/search?q=' + encodeURIComponent('zzqx unrelated words'))).text()
+    expect(html.match(/class="promo featured"/g)?.length).toBe(1)
+    expect(html.indexOf('class="ad-unit"')).toBeLessThan(html.indexOf('class="promo featured"'))
   })
 })
 
@@ -404,8 +417,9 @@ describe('categories: images, videos, music, live', () => {
     expect(html).toContain('src="/img?u=https%3A%2F%2Fencrypted-tbn0.gstatic.com%2Fimages%3Fq%3Dtbn%3Aabc"')
     // A thumbnail host outside the allowlist gets no <img> at all.
     expect(html).not.toContain('tracker.example/pixel')
-    // Nothing loads from another origin, except the footer's built.indie and pwamart badges, which are ours.
-    expect(html).not.toMatch(/src="https:\/\/(?!built\.indie\.pit\.world\/|pwamart\.com\/)/)
+    // Nothing loads from another origin, except the footer's built.indie and pwamart badges, which are
+    // ours, and the one sandboxed crawlproof ad frame, which is told only that it is on pit.world.
+    expect(html).not.toMatch(/src="https:\/\/(?!built\.indie\.pit\.world\/|pwamart\.com\/|crawlproof\.com\/api\/ads\/frame\?)/)
   })
 
   it('puts PairUX first, then nixamp, above live results', async () => {