From 2d2de0af4696b154bda73638aa3f815f5dcfea55 Mon Sep 17 00:00:00 2001
From: Matt Miermans <m.miermans@gmail.com>
Date: Mon, 6 Jul 2026 13:11:34 +0200
Subject: [PATCH] feat(HNT-2086): add Redis key names for crawl state

---
 packages/crawl-common/src/index.ts           |  9 ++++
 packages/crawl-common/src/redis/index.ts     | 10 ++++
 packages/crawl-common/src/redis/keys.spec.ts | 45 ++++++++++++++++++
 packages/crawl-common/src/redis/keys.ts      | 48 ++++++++++++++++++++
 4 files changed, 112 insertions(+)
 create mode 100644 packages/crawl-common/src/redis/index.ts
 create mode 100644 packages/crawl-common/src/redis/keys.spec.ts
 create mode 100644 packages/crawl-common/src/redis/keys.ts

diff --git a/packages/crawl-common/src/index.ts b/packages/crawl-common/src/index.ts
index cd7138b..8b7950e 100644
--- a/packages/crawl-common/src/index.ts
+++ b/packages/crawl-common/src/index.ts
@@ -1,4 +1,13 @@
 export { normalizeText } from './utils/index.js';
+export {
+  pageFetchKey,
+  pageLockKey,
+  pageEnqueuedKey,
+  articleEnqueuedKey,
+  articleFetchKey,
+  articleLockKey,
+  articleContentKey,
+} from './redis/index.js';
 export {
   initCorpusApiClient,
   updateApprovedCorpusItem,
diff --git a/packages/crawl-common/src/redis/index.ts b/packages/crawl-common/src/redis/index.ts
new file mode 100644
index 0000000..1422571
--- /dev/null
+++ b/packages/crawl-common/src/redis/index.ts
@@ -0,0 +1,10 @@
+export {
+  hashUrl,
+  pageFetchKey,
+  pageLockKey,
+  pageEnqueuedKey,
+  articleEnqueuedKey,
+  articleFetchKey,
+  articleLockKey,
+  articleContentKey,
+} from './keys.js';
diff --git a/packages/crawl-common/src/redis/keys.spec.ts b/packages/crawl-common/src/redis/keys.spec.ts
new file mode 100644
index 0000000..9d8ed79
--- /dev/null
+++ b/packages/crawl-common/src/redis/keys.spec.ts
@@ -0,0 +1,45 @@
+import { describe, expect, it } from 'vitest';
+import {
+  articleContentKey,
+  articleFetchKey,
+  articleLockKey,
+  hashUrl,
+  pageEnqueuedKey,
+  pageFetchKey,
+  pageLockKey,
+} from './keys.js';
+
+const URL = 'https://example.com/news/article-1';
+
+describe('hashUrl', () => {
+  it('produces a 64-char hex SHA-256 digest', () => {
+    expect(hashUrl(URL)).toMatch(/^[0-9a-f]{64}$/);
+  });
+
+  it('is stable for the same URL', () => {
+    expect(hashUrl(URL)).toBe(hashUrl(URL));
+  });
+
+  it('ignores surrounding whitespace', () => {
+    expect(hashUrl(`  ${URL}\n`)).toBe(hashUrl(URL));
+  });
+
+  it('differs for different URLs', () => {
+    expect(hashUrl(URL)).not.toBe(hashUrl(`${URL}-2`));
+  });
+});
+
+describe('key builders', () => {
+  const hash = hashUrl(URL);
+
+  it.each([
+    [pageFetchKey, 'page:fetch'],
+    [pageLockKey, 'page:lock'],
+    [pageEnqueuedKey, 'page:enqueued'],
+    [articleFetchKey, 'article:fetch'],
+    [articleLockKey, 'article:lock'],
+    [articleContentKey, 'article:content'],
+  ])('%o builds the %s namespace', (build, prefix) => {
+    expect(build(URL)).toBe(`${prefix}:${hash}`);
+  });
+});
diff --git a/packages/crawl-common/src/redis/keys.ts b/packages/crawl-common/src/redis/keys.ts
new file mode 100644
index 0000000..daf57c5
--- /dev/null
+++ b/packages/crawl-common/src/redis/keys.ts
@@ -0,0 +1,48 @@
+import { createHash } from 'node:crypto';
+
+/**
+ * Hash a URL into the stable token used in Redis keys. The URL is
+ * trimmed first so that incidental surrounding whitespace cannot
+ * produce two keys for the same article; this is the normalization
+ * deferred from the message-validation boundary, kept here so the
+ * single place that derives keys also owns it. SHA-256 keeps keys a
+ * fixed length regardless of URL length.
+ */
+export function hashUrl(url: string): string {
+  return createHash('sha256').update(url.trim()).digest('hex');
+}
+
+/** Last time a page was fetched (crawled) by the discovery worker. */
+export function pageFetchKey(url: string): string {
+  return `page:fetch:${hashUrl(url)}`;
+}
+
+/** Guard against concurrent fetches of the same page. */
+export function pageLockKey(url: string): string {
+  return `page:lock:${hashUrl(url)}`;
+}
+
+/** Last time the agent enqueued a discovery job for a page. */
+export function pageEnqueuedKey(url: string): string {
+  return `page:enqueued:${hashUrl(url)}`;
+}
+
+/** Last time the agent enqueued a crawl job for a live article. */
+export function articleEnqueuedKey(url: string): string {
+  return `article:enqueued:${hashUrl(url)}`;
+}
+
+/** Last time an article was fetched (extracted) by the article worker. */
+export function articleFetchKey(url: string): string {
+  return `article:fetch:${hashUrl(url)}`;
+}
+
+/** Guard against concurrent fetches of the same article. */
+export function articleLockKey(url: string): string {
+  return `article:lock:${hashUrl(url)}`;
+}
+
+/** Content hash of an article, for change detection across crawls. */
+export function articleContentKey(url: string): string {
+  return `article:content:${hashUrl(url)}`;
+}