feat(HNT-2888): add Redis key names for crawl state

packages/crawl-common/src/index.ts

@@ -1,4 +1,13 @@
 export { normalizeText } from './utils/index.js';
+export {
+  pageFetchKey,
+  pageLockKey,
+  pageEnqueuedKey,
+  articleEnqueuedKey,
+  articleFetchKey,
+  articleLockKey,
+  articleContentKey,
+} from './redis/index.js';
 export {
   initCorpusApiClient,
   updateApprovedCorpusItem,

packages/crawl-common/src/redis/index.ts

@@ -0,0 +1,10 @@
+export {
+  hashUrl,
+  pageFetchKey,
+  pageLockKey,
+  pageEnqueuedKey,
+  articleEnqueuedKey,
+  articleFetchKey,
+  articleLockKey,
+  articleContentKey,
+} from './keys.js';

packages/crawl-common/src/redis/keys.spec.ts

@@ -0,0 +1,45 @@
+import { describe, expect, it } from 'vitest';
+import {
+  articleContentKey,
+  articleFetchKey,
+  articleLockKey,
+  hashUrl,
+  pageEnqueuedKey,
+  pageFetchKey,
+  pageLockKey,
+} from './keys.js';
+
+const URL = 'https://example.com/news/article-1';
+
+describe('hashUrl', () => {
+  it('produces a 64-char hex SHA-256 digest', () => {
+    expect(hashUrl(URL)).toMatch(/^[0-9a-f]{64}$/);
+  });
+
+  it('is stable for the same URL', () => {
+    expect(hashUrl(URL)).toBe(hashUrl(URL));
+  });
+
+  it('ignores surrounding whitespace', () => {
+    expect(hashUrl(`  ${URL}\n`)).toBe(hashUrl(URL));
+  });
+
+  it('differs for different URLs', () => {
+    expect(hashUrl(URL)).not.toBe(hashUrl(`${URL}-2`));
+  });
+});
+
+describe('key builders', () => {
+  const hash = hashUrl(URL);
+
+  it.each([
+    [pageFetchKey, 'page:fetch'],
+    [pageLockKey, 'page:lock'],
+    [pageEnqueuedKey, 'page:enqueued'],
+    [articleFetchKey, 'article:fetch'],
+    [articleLockKey, 'article:lock'],
+    [articleContentKey, 'article:content'],
+  ])('%o builds the %s namespace', (build, prefix) => {
+    expect(build(URL)).toBe(`${prefix}:${hash}`);
+  });
+});

packages/crawl-common/src/redis/keys.ts

@@ -0,0 +1,48 @@
+import { createHash } from 'node:crypto';
+
+/**
+ * Hash a URL into the stable token used in Redis keys. The URL is
+ * trimmed first so that incidental surrounding whitespace cannot
+ * produce two keys for the same article; this is the normalization
+ * deferred from the message-validation boundary, kept here so the
+ * single place that derives keys also owns it. SHA-256 keeps keys a
+ * fixed length regardless of URL length.
+ */
+export function hashUrl(url: string): string {
+  return createHash('sha256').update(url.trim()).digest('hex');
+}
+
+/** Last time a page was fetched (crawled) by the discovery worker. */
+export function pageFetchKey(url: string): string {
+  return `page:fetch:${hashUrl(url)}`;
+}
+
+/** Guard against concurrent fetches of the same page. */
+export function pageLockKey(url: string): string {
+  return `page:lock:${hashUrl(url)}`;
+}
+
+/** Last time the agent enqueued a discovery job for a page. */
+export function pageEnqueuedKey(url: string): string {
+  return `page:enqueued:${hashUrl(url)}`;
+}
+
+/** Last time the agent enqueued a crawl job for a live article. */
+export function articleEnqueuedKey(url: string): string {
+  return `article:enqueued:${hashUrl(url)}`;
+}
+
+/** Last time an article was fetched (extracted) by the article worker. */
+export function articleFetchKey(url: string): string {
+  return `article:fetch:${hashUrl(url)}`;
+}
+
+/** Guard against concurrent fetches of the same article. */
+export function articleLockKey(url: string): string {
+  return `article:lock:${hashUrl(url)}`;
+}
+
+/** Content hash of an article, for change detection across crawls. */
+export function articleContentKey(url: string): string {
+  return `article:content:${hashUrl(url)}`;
+}