diff --git a/.github/workflows/indexnow.yml b/.github/workflows/indexnow.yml new file mode 100644 index 0000000..74c7779 --- /dev/null +++ b/.github/workflows/indexnow.yml @@ -0,0 +1,29 @@ +name: IndexNow + +on: + deployment_status: + workflow_dispatch: + inputs: + all: + description: Submit every URL in the sitemap, not only recently changed pages + type: boolean + default: false + +permissions: + contents: read + +jobs: + submit: + if: >- + github.event_name == 'workflow_dispatch' || + (github.event.deployment_status.state == 'success' && + github.event.deployment.environment == 'Production') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.deployment.sha || github.ref }} + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: node scripts/indexnow.mjs ${{ inputs.all && '--all' || '' }} diff --git a/public/f45f7e891bc3827c5178c9ff112a900c.txt b/public/f45f7e891bc3827c5178c9ff112a900c.txt new file mode 100644 index 0000000..5ca577d --- /dev/null +++ b/public/f45f7e891bc3827c5178c9ff112a900c.txt @@ -0,0 +1 @@ +f45f7e891bc3827c5178c9ff112a900c \ No newline at end of file diff --git a/scripts/indexnow.mjs b/scripts/indexnow.mjs new file mode 100644 index 0000000..9e2c8f1 --- /dev/null +++ b/scripts/indexnow.mjs @@ -0,0 +1,67 @@ +import { pathToFileURL } from "node:url"; +import { siteConfig } from "../src/lib/site.js"; + +export const ENDPOINT = "https://api.indexnow.org/indexnow"; +const DAY = 24 * 60 * 60 * 1000; + +export function locsFromSitemap(xml) { + return [...xml.matchAll(/([\s\S]*?)<\/url>/g)] + .map(([, block]) => ({ + loc: /([^<]+)<\/loc>/.exec(block)?.[1].trim(), + lastmod: /([^<]+)<\/lastmod>/.exec(block)?.[1].trim() ?? null, + })) + .filter((entry) => entry.loc); +} + +export function pickUrls(entries, { all = false, sinceDays = 7, now = Date.now() } = {}) { + if (all) return entries.map((entry) => entry.loc); + const cutoff = now - sinceDays * DAY; + return entries + .filter((entry) => entry.lastmod && Date.parse(entry.lastmod) >= cutoff) + .map((entry) => entry.loc); +} + +export function payload({ base, key, urls }) { + const { host, origin } = new URL(base); + return { + host, + key, + keyLocation: `${origin}/${key}.txt`, + urlList: urls.filter((url) => new URL(url).host === host), + }; +} + +async function main(argv) { + const flags = new Set(argv.filter((arg) => arg.startsWith("--"))); + const base = (argv.find((arg) => !arg.startsWith("--")) ?? siteConfig.url).replace(/\/$/, ""); + const sitemap = await fetch(`${base}/sitemap.xml`); + if (!sitemap.ok) throw new Error(`${base}/sitemap.xml returned ${sitemap.status}`); + + const urls = pickUrls(locsFromSitemap(await sitemap.text()), { all: flags.has("--all") }); + const body = payload({ base: siteConfig.url, key: siteConfig.indexNowKey, urls }); + if (body.urlList.length === 0) { + console.log("No recently changed pages to submit."); + return; + } + if (flags.has("--dry-run")) { + console.log(JSON.stringify(body, null, 2)); + return; + } + + const response = await fetch(ENDPOINT, { + method: "POST", + headers: { "content-type": "application/json; charset=utf-8" }, + body: JSON.stringify(body), + }); + if (response.status !== 200 && response.status !== 202) { + throw new Error(`IndexNow returned ${response.status}: ${await response.text()}`); + } + console.log(`IndexNow accepted ${body.urlList.length} URLs (${response.status}).`); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(process.argv.slice(2)).catch((error) => { + console.error(error.message); + process.exit(1); + }); +} diff --git a/src/lib/site.js b/src/lib/site.js index c74940c..c2da8e0 100644 --- a/src/lib/site.js +++ b/src/lib/site.js @@ -21,6 +21,7 @@ export const siteConfig = { }, repo: "https://github.com/salsadsid/visualizer", feedbackUrl: `https://github.com/salsadsid/visualizer/issues/new?${feedbackParams}`, + indexNowKey: "f45f7e891bc3827c5178c9ff112a900c", keywords: [ "DSA visualizer", "array visualizer", diff --git a/tests/indexnow.test.mjs b/tests/indexnow.test.mjs new file mode 100644 index 0000000..2295a4f --- /dev/null +++ b/tests/indexnow.test.mjs @@ -0,0 +1,61 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { locsFromSitemap, payload, pickUrls } from "../scripts/indexnow.mjs"; +import { siteConfig } from "../src/lib/site.js"; + +const SITEMAP = ` + + +https://visualizer-gold.vercel.app/ +2026-10-20 +weekly +1 + + +https://visualizer-gold.vercel.app/roadmap +2026-09-19 + + +https://visualizer-gold.vercel.app/algorithms + +`; + +test("reads every url and its lastmod from a sitemap", () => { + assert.deepEqual(locsFromSitemap(SITEMAP), [ + { loc: "https://visualizer-gold.vercel.app/", lastmod: "2026-10-20" }, + { loc: "https://visualizer-gold.vercel.app/roadmap", lastmod: "2026-09-19" }, + { loc: "https://visualizer-gold.vercel.app/algorithms", lastmod: null }, + ]); +}); + +test("submits only pages changed within the last seven days unless asked for all", () => { + const entries = locsFromSitemap(SITEMAP); + const edge = Date.parse("2026-10-27T00:00:00Z"); + assert.deepEqual(pickUrls(entries, { now: edge }), ["https://visualizer-gold.vercel.app/"]); + assert.deepEqual(pickUrls(entries, { now: edge + 1000 }), []); + assert.deepEqual( + pickUrls(entries, { all: true, now: edge }), + entries.map((entry) => entry.loc) + ); +}); + +test("builds a payload whose urls all sit on the key's host", () => { + const body = payload({ + base: "https://visualizer-gold.vercel.app", + key: "abc123", + urls: ["https://visualizer-gold.vercel.app/", "https://elsewhere.example/page"], + }); + assert.deepEqual(body, { + host: "visualizer-gold.vercel.app", + key: "abc123", + keyLocation: "https://visualizer-gold.vercel.app/abc123.txt", + urlList: ["https://visualizer-gold.vercel.app/"], + }); +}); + +test("the key file in public/ holds exactly the configured key", () => { + assert.match(siteConfig.indexNowKey, /^[a-f0-9]{32}$/); + const file = new URL(`../public/${siteConfig.indexNowKey}.txt`, import.meta.url); + assert.equal(readFileSync(file, "utf8"), siteConfig.indexNowKey); +});