Skip to content

Commit 5d64958

Browse files
authored
v0.8.58: desktop app catchas, insights dashboard, kb fixes
2 parents 3975ae5 + aba70b2 commit 5d64958

167 files changed

Lines changed: 71234 additions & 2661 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/test-build.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ jobs:
102102
bunx vitest run
103103
scripts/retired-columns.postgres.test.ts
104104
scripts/connector-sync-schedule-precision.postgres.test.ts
105+
scripts/database-failure-classification.postgres.test.ts
105106
106107
- name: Verify OAuth lifecycle and SCIM membership guards in PostgreSQL
107108
working-directory: apps/sim
@@ -267,6 +268,7 @@ jobs:
267268
lib/knowledge/__integration__/search-source-pagination.integration.ts
268269
lib/knowledge/__integration__/search-reference-batching.integration.ts
269270
lib/knowledge/__integration__/embedding-insert-batches.integration.ts
271+
lib/knowledge/__integration__/processing-lock-scope.integration.ts
270272
lib/knowledge/__integration__/connector-lifecycle-locks.integration.ts
271273
lib/knowledge/__integration__/connector-deferral.integration.ts
272274
lib/knowledge/__integration__/stored-document-recovery.integration.ts
@@ -275,10 +277,13 @@ jobs:
275277
lib/knowledge/__integration__/listing-continuation.integration.ts
276278
lib/knowledge/__integration__/member-scope-renewal.integration.ts
277279
lib/knowledge/__integration__/member-document-lifecycle.integration.ts
280+
lib/knowledge/__integration__/connector-lease-pages.integration.ts
278281
lib/knowledge/__integration__/slack-empty-threads.integration.ts
279282
lib/knowledge/__integration__/kb-block-search.integration.ts
280283
lib/knowledge/__integration__/gitlab-workspace.integration.ts
281284
lib/knowledge/__integration__/unfilled-projection-source.integration.ts
285+
lib/knowledge/__integration__/knowledge-projection.integration.ts
286+
lib/knowledge/__integration__/async-projection-processing.integration.ts
282287
lib/knowledge/__integration__/purged-detach-reservation.integration.ts
283288
lib/core/outbox/service.integration.ts
284289
lib/knowledge/__integration__/connector-upload.integration.ts

‎apps/desktop/e2e/smoke.spec.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,13 @@ const PAGES: Record<string, string> = {
2121
'/login': '<!doctype html><html><body><h1 id="login">fixture-login</h1></body></html>',
2222
}
2323

24+
/** `User-Agent` of every request the fixture origin has served, in arrival order. */
25+
const requestUserAgents: string[] = []
26+
2427
function startFixtureServer(): Promise<{ server: Server; origin: string }> {
2528
return new Promise((resolvePromise) => {
2629
const server = createServer((request, response) => {
30+
requestUserAgents.push(request.headers['user-agent'] ?? '')
2731
const path = new URL(request.url ?? '/', 'http://127.0.0.1').pathname
2832
const sessionCookie = request.headers.cookie
2933
?.split(';')
@@ -85,6 +89,21 @@ test.describe('desktop shell smoke', () => {
8589
expect(window.url()).toBe(`${origin}/home`)
8690
})
8791

92+
test('presents one stock Chrome user agent on every request from the first load', async () => {
93+
requestUserAgents.length = 0
94+
app = await launchApp(origin)
95+
const window = await app.firstWindow()
96+
await expect(window.locator('#app')).toHaveText('fixture-app')
97+
await window.evaluate(() => fetch('/home').then((response) => response.text()))
98+
99+
const pageUserAgent = await window.evaluate(() => navigator.userAgent)
100+
expect(pageUserAgent).toMatch(
101+
/^Mozilla\/5\.0 \(.+\) AppleWebKit\/537\.36 \(KHTML, like Gecko\) Chrome\/\d+\.0\.0\.0 Safari\/537\.36$/
102+
)
103+
expect(requestUserAgents.length).toBeGreaterThanOrEqual(2)
104+
expect(new Set(requestUserAgents)).toEqual(new Set([pageUserAgent]))
105+
})
106+
88107
test('internal window.open creates an independent full Sim window', async () => {
89108
app = await launchApp(origin)
90109
const window = await app.firstWindow()

‎apps/desktop/src/main/browser-agent/session.test.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ interface MockView {
3737
session: {
3838
setPermissionRequestHandler: ReturnType<typeof vi.fn>
3939
setPermissionCheckHandler: ReturnType<typeof vi.fn>
40+
setUserAgent: ReturnType<typeof vi.fn>
4041
webRequest: { onBeforeRequest: ReturnType<typeof vi.fn> }
4142
}
4243
on: ReturnType<typeof vi.fn>
@@ -358,15 +359,14 @@ describe('browser-agent session', () => {
358359
expect(onTabNavigated).toHaveBeenCalledWith(contents, true)
359360
})
360361

361-
it('gives every tab a user agent with no Electron token in it', () => {
362+
it('leaves every tab on the process-wide user agent instead of overriding it', () => {
362363
const first = session.ensureTab()
363364
const second = session.addTab()
364365

365366
for (const tab of [first, second]) {
366367
const contents = (tab.view as unknown as MockView).webContents
367-
const agent = contents.setUserAgent.mock.calls.at(-1)?.[0] as string | undefined
368-
expect(agent).toMatch(/^Mozilla\/5\.0 \(.+\) .*Chrome\/\d+\.0\.0\.0 Safari\/537\.36$/)
369-
expect(agent).not.toMatch(/Electron|Sim\//)
368+
expect(contents.setUserAgent).not.toHaveBeenCalled()
369+
expect(contents.session.setUserAgent).not.toHaveBeenCalled()
370370
}
371371
})
372372

‎apps/desktop/src/main/browser-agent/session.ts‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,6 @@ import {
6767
isBlockedSubresourceUrl,
6868
subresourceNeedsResolution,
6969
} from '@/main/browser-agent/url-guard'
70-
import { browserUserAgent } from '@/main/browser-agent/user-agent'
7170
import type { BrowserSessionSnapshot } from '@/main/desktop-chat-session-store'
7271
import { suggestedFilename, uniqueDownloadPath } from '@/main/downloads'
7372
import {
@@ -1413,11 +1412,6 @@ function configureAgentPartition(ses: Session): void {
14131412
}
14141413
return ALLOWED_SITE_PERMISSIONS.has(permission)
14151414
})
1416-
// Service workers do not inherit a tab's user agent. With only the tab's set,
1417-
// the document request carries the browser string while the worker's own
1418-
// script request still announces Electron — and on a site that routes its
1419-
// fetches through a worker, that is the one the server sees.
1420-
ses.setUserAgent(browserUserAgent())
14211415
// SSRF choke point for the agent partition. Document navigations (top-level +
14221416
// iframes) get the full DNS-resolving check — the one seam every navigation
14231417
// passes through, including page-initiated ones the driver never sees (server
@@ -1965,10 +1959,6 @@ function initializeTabView(view: WebContentsView, scopeId: string): WebContentsV
19651959
const contents = view.webContents
19661960
registerAgentWebContents(contents)
19671961
configureAgentPartition(contents.session)
1968-
// The session default does not reach a WebContents that already exists, and
1969-
// the first tab is what brings the session into being, so each tab sets its
1970-
// own as well — otherwise tab one browses as Electron and the rest as Chrome.
1971-
contents.setUserAgent(browserUserAgent())
19721962
attachAgentContextMenu(contents, {
19731963
addToChat: (text) => withBrowserScope(scopeId, () => addPageSelectionToChat(contents, text)),
19741964
openTab: (url) => withBrowserScope(scopeId, () => openTabWithUrl(url, { agentOwned: false })),

‎apps/desktop/src/main/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ import { attachTelemetryPolicy } from '@/main/telemetry-policy'
8383
import { TerminalRegistry } from '@/main/terminal/registry'
8484
import { installTray, type TrayHandle } from '@/main/tray'
8585
import { checkForUpdatesInteractive, initUpdater, type UpdaterHandle } from '@/main/updater'
86+
import { installBrowserUserAgent } from '@/main/user-agent'
8687
import { createMainWindow, setupPermissionHandlers } from '@/main/window'
8788
import { attachWindowOpenPolicy, isPopupContents } from '@/main/windows'
8889

@@ -899,6 +900,7 @@ app.setName(APP_NAME_FOR_CHANNEL[channelForOrigin(DEFAULT_ORIGIN)])
899900
if (process.env.SIM_DESKTOP_USER_DATA) {
900901
app.setPath('userData', process.env.SIM_DESKTOP_USER_DATA)
901902
}
903+
installBrowserUserAgent()
902904

903905
// The scheme the offline page and server picker load from must be declared
904906
// before the app is ready; the per-session handlers attach later.
Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
import { app } from 'electron'
22
import { describe, expect, it, vi } from 'vitest'
3-
import { browserUserAgent, stockChromeUserAgent } from '@/main/browser-agent/user-agent'
3+
import { installBrowserUserAgent, stockChromeUserAgent } from '@/main/user-agent'
44

55
vi.mock('electron', () => import('@/test/electron-mock'))
66

77
const ELECTRON_DEFAULT =
88
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Sim/1.0.0 Chrome/140.0.7339.207 Electron/43.1.1 Safari/537.36'
9+
const STOCK_CHROME =
10+
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36'
911

1012
describe('stockChromeUserAgent', () => {
1113
it('drops the application and Electron tokens a browser allowlist rejects', () => {
@@ -15,9 +17,7 @@ describe('stockChromeUserAgent', () => {
1517
})
1618

1719
it('reproduces the desktop string Chrome sends under user-agent reduction', () => {
18-
expect(stockChromeUserAgent(ELECTRON_DEFAULT)).toBe(
19-
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36'
20-
)
20+
expect(stockChromeUserAgent(ELECTRON_DEFAULT)).toBe(STOCK_CHROME)
2121
})
2222

2323
it('keeps the platform token of the machine it is running on', () => {
@@ -32,12 +32,13 @@ describe('stockChromeUserAgent', () => {
3232
})
3333
})
3434

35-
describe('browserUserAgent', () => {
36-
it('derives from the string Electron would otherwise have sent', () => {
35+
describe('installBrowserUserAgent', () => {
36+
it('idempotently makes stock Chrome the process-wide fallback every request path uses', () => {
3737
app.userAgentFallback = ELECTRON_DEFAULT
3838

39-
expect(browserUserAgent()).toBe(
40-
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36'
41-
)
39+
installBrowserUserAgent()
40+
installBrowserUserAgent()
41+
42+
expect(app.userAgentFallback).toBe(STOCK_CHROME)
4243
})
4344
})

apps/desktop/src/main/browser-agent/user-agent.ts renamed to apps/desktop/src/main/user-agent.ts

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/**
2-
* The user agent the browser resource presents to sites.
2+
* The user agent the whole desktop process presents: the embedded browser and
3+
* the app's own windows alike (desktop identity travels in `X-Sim-Client-Info`).
34
*
45
* Electron's default string carries two tokens no browser sends —
56
* `Sim/<version>` and `Electron/<version>`. Chromium's own token sits right
@@ -38,9 +39,11 @@ export function stockChromeUserAgent(defaultUserAgent: string): string {
3839
}
3940

4041
/**
41-
* Derived from the string Electron would otherwise have sent, so the reported
42-
* Chromium version tracks whatever Chromium the app actually ships.
42+
* Sets the stock Chrome identity as `app.userAgentFallback` before any session
43+
* exists. Session and per-tab overrides miss some request paths (a cross-origin
44+
* challenge frame still sends the process default), and a site that sees two
45+
* user agents in one challenge rejects it as a spoof. Idempotent.
4346
*/
44-
export function browserUserAgent(): string {
45-
return stockChromeUserAgent(app.userAgentFallback)
47+
export function installBrowserUserAgent(): void {
48+
app.userAgentFallback = stockChromeUserAgent(app.userAgentFallback)
4649
}

‎apps/docs/content/docs/platform/self-hosting/background-jobs.mdx‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ Point cron at an **internal** address where possible (the in-cluster Service, or
4747
| Time pause/resume | `/api/resume/poll` | `*/1 * * * *` | Workflows paused on a timer |
4848
| Outbox processing | `/api/webhooks/outbox/process` | `*/1 * * * *` | Transactional-outbox retries for billing, membership, enterprise issuance, and workflow-deployment side effects |
4949
| Workspace file search dispatch | `/api/cron/workspace-file-search-dispatch` | `*/1 * * * *` | Dispatches indexing work for workspace file search |
50+
| Knowledge projection | `/api/cron/knowledge-projection` | `*/1 * * * *` | Brings knowledge base search up to date with document, permission, and chunk changes |
5051
| Connector sync | `/api/knowledge/connectors/sync` | `*/5 * * * *` | Knowledge base connector syncs |
5152
| Connector member sync | `/api/knowledge/connectors/member-sync` | `*/5 * * * *` | Per-member access sync for permission-aware connectors |
5253
| Connector directory sync | `/api/knowledge/connectors/directory-sync` | `*/5 * * * *` | Refreshes the directory groups administrator-mode connectors mirror, so a membership change takes effect without waiting for a content sync |
Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
/**
2+
* @vitest-environment node
3+
*/
4+
import { createMockRequest } from '@sim/testing'
5+
import { beforeEach, describe, expect, it, vi } from 'vitest'
6+
7+
const mocks = vi.hoisted(() => ({
8+
enqueueSweep: vi.fn(),
9+
verifyCronAuth: vi.fn(),
10+
}))
11+
12+
vi.mock('@/lib/auth/internal', () => ({ verifyCronAuth: mocks.verifyCronAuth }))
13+
vi.mock('@/lib/knowledge/projection/enqueue', () => ({
14+
enqueueKnowledgeProjectionSweep: mocks.enqueueSweep,
15+
}))
16+
17+
import { GET } from '@/app/api/cron/knowledge-projection/route'
18+
19+
function request() {
20+
return createMockRequest(
21+
'GET',
22+
undefined,
23+
{},
24+
'http://localhost:3000/api/cron/knowledge-projection'
25+
)
26+
}
27+
28+
describe('knowledge projection sweep route', () => {
29+
beforeEach(() => {
30+
vi.clearAllMocks()
31+
mocks.verifyCronAuth.mockReturnValue(null)
32+
})
33+
34+
it('returns as soon as Trigger.dev accepts the pass', async () => {
35+
mocks.enqueueSweep.mockResolvedValue({
36+
triggered: true,
37+
backend: 'trigger-dev',
38+
jobId: 'run-1',
39+
})
40+
41+
const response = await GET(request())
42+
43+
expect(response.status).toBe(202)
44+
await expect(response.json()).resolves.toEqual({
45+
success: true,
46+
triggered: true,
47+
backend: 'trigger-dev',
48+
jobId: 'run-1',
49+
})
50+
})
51+
52+
it('answers 200 without a pass when the projector has nothing to do', async () => {
53+
mocks.enqueueSweep.mockResolvedValue({ triggered: false, backend: null, jobId: null })
54+
55+
const response = await GET(request())
56+
57+
expect(response.status).toBe(200)
58+
await expect(response.json()).resolves.toEqual({
59+
success: true,
60+
triggered: false,
61+
backend: null,
62+
jobId: null,
63+
})
64+
})
65+
66+
it('returns the cron auth refusal without enqueueing', async () => {
67+
mocks.verifyCronAuth.mockReturnValue(new Response(null, { status: 401 }))
68+
69+
const response = await GET(request())
70+
71+
expect(response.status).toBe(401)
72+
expect(mocks.enqueueSweep).not.toHaveBeenCalled()
73+
})
74+
75+
it('fails closed when Trigger.dev does not accept the pass', async () => {
76+
mocks.enqueueSweep.mockRejectedValue(new Error('trigger unavailable'))
77+
78+
const response = await GET(request())
79+
80+
expect(response.status).toBe(500)
81+
await expect(response.json()).resolves.toEqual({
82+
success: false,
83+
error: 'Sweep enqueue failed',
84+
})
85+
})
86+
})
Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
import { createLogger } from '@sim/logger'
2+
import { getErrorMessage } from '@sim/utils/errors'
3+
import { type NextRequest, NextResponse } from 'next/server'
4+
import { verifyCronAuth } from '@/lib/auth/internal'
5+
import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
6+
import { enqueueKnowledgeProjectionSweep } from '@/lib/knowledge/projection/enqueue'
7+
8+
const logger = createLogger('KnowledgeProjectionSweepRoute')
9+
10+
export const dynamic = 'force-dynamic'
11+
export const maxDuration = 60
12+
13+
/**
14+
* The knowledge projector's periodic sweep: enqueues one pass per window while there is work, and
15+
* returns once Trigger.dev accepts it. Writers ask for passes as they commit; this converges
16+
* whatever those requests missed.
17+
*/
18+
export const GET = withRouteHandler(async (request: NextRequest) => {
19+
const authError = verifyCronAuth(request, 'Knowledge projection sweep')
20+
if (authError) return authError
21+
22+
try {
23+
const result = await enqueueKnowledgeProjectionSweep()
24+
return NextResponse.json({ success: true, ...result }, { status: result.triggered ? 202 : 200 })
25+
} catch (error) {
26+
logger.error('Knowledge projection sweep enqueue failed', { error: getErrorMessage(error) })
27+
return NextResponse.json({ success: false, error: 'Sweep enqueue failed' }, { status: 500 })
28+
}
29+
})

0 commit comments

Comments
 (0)