From e19eebf4512880df9bb8050da98ae89fd7f455bb Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 08:41:09 +0530 Subject: [PATCH 01/24] feat(browserstack-service): route wdio jasmine to the CLI flow (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../browserstack-service/src/cli/cliUtils.ts | 2 +- .../frameworks/wdioJasmineTestFramework.ts | 16 ++++++ .../browserstack-service/src/cli/index.ts | 7 ++- .../tests/cli/wdioJasmineGate.test.ts | 57 +++++++++++++++++++ 4 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts create mode 100644 packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts diff --git a/packages/browserstack-service/src/cli/cliUtils.ts b/packages/browserstack-service/src/cli/cliUtils.ts index 3c6e5c05..6fbe10bf 100644 --- a/packages/browserstack-service/src/cli/cliUtils.ts +++ b/packages/browserstack-service/src/cli/cliUtils.ts @@ -47,7 +47,7 @@ const CLI_DOWNLOAD_TMP_SUFFIX = '.zip' export class CLIUtils { static automationFrameworkDetail = {} static testFrameworkDetail = {} - static CLISupportedFrameworks = ['mocha', 'cucumber'] + static CLISupportedFrameworks = ['mocha', 'cucumber', 'jasmine'] static isDevelopmentEnv() { return process.env.BROWSERSTACK_CLI_ENV === 'development' diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts new file mode 100644 index 00000000..68524b06 --- /dev/null +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -0,0 +1,16 @@ +import TestFramework from './testFramework.js' +import { BStackLogger as logger } from '../cliLogger.js' + +/** + * CLI test framework for `framework: 'jasmine'` under WebdriverIO. + * + * Extends the BASE TestFramework, never WdioMochaTestFramework: jasmine's identity (fullName), + * hook taxonomy (`hookName`) and skip/exclude states differ from mocha's, and inheriting mocha's + * event model would report jasmine specs through the wrong runner's semantics. + */ +export default class WdioJasmineTestFramework extends TestFramework { + constructor(testFrameworks: string[], testFrameworkVersions: Record, binSessionId: string) { + super(testFrameworks, testFrameworkVersions, binSessionId) + logger.debug('WdioJasmineTestFramework: constructed') + } +} diff --git a/packages/browserstack-service/src/cli/index.ts b/packages/browserstack-service/src/cli/index.ts index 444ec6d7..735ccc6d 100644 --- a/packages/browserstack-service/src/cli/index.ts +++ b/packages/browserstack-service/src/cli/index.ts @@ -17,6 +17,7 @@ import type { Options } from '@wdio/types' import TestOpsConfig from '../testOps/testOpsConfig.js' import WdioMochaTestFramework from './frameworks/wdioMochaTestFramework.js' import WdioCucumberTestFramework from './frameworks/wdioCucumberTestFramework.js' +import WdioJasmineTestFramework from './frameworks/wdioJasmineTestFramework.js' import WdioAutomationFramework from './frameworks/wdioAutomationFramework.js' import WebdriverIOModule from './modules/webdriverIOModule.js' import AccessibilityModule from './modules/accessibilityModule.js' @@ -48,7 +49,7 @@ export class BrowserstackCLI { modulesLoaded = false binSessionId: string | null = null modules: Record = {} - testFramework: WdioMochaTestFramework|WdioCucumberTestFramework|null = null + testFramework: WdioMochaTestFramework|WdioCucumberTestFramework|WdioJasmineTestFramework|null = null cliParams: Record | null = null automationFramework: WdioAutomationFramework|null = null SDK_CLI_BIN_PATH: string | null = null @@ -555,6 +556,10 @@ export class BrowserstackCLI { this.testFramework = new WdioCucumberTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string) return } + if (testFrameworkDetail.name.toLowerCase() === 'webdriverio-jasmine') { + this.testFramework = new WdioJasmineTestFramework([testFrameworkDetail.name], testFrameworkDetail.version, this.binSessionId as string) + return + } // An unmatched name leaves testFramework null, and every CLI event then no-ops with no // error of any kind. Name it so the silence is diagnosable. this.logger.error(`setupTestFramework: no CLI test framework registered for name=${testFrameworkDetail.name}; test events will not be tracked`) diff --git a/packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts new file mode 100644 index 00000000..424744d9 --- /dev/null +++ b/packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts @@ -0,0 +1,57 @@ +import { describe, expect, it, vi, afterEach } from 'vitest' +import * as bstackLogger from '../../src/bstackLogger.js' + +import { BrowserstackCLI } from '../../src/cli/index.js' +import { CLIUtils } from '../../src/cli/cliUtils.js' +import TestFramework from '../../src/cli/frameworks/testFramework.js' +import WdioMochaTestFramework from '../../src/cli/frameworks/wdioMochaTestFramework.js' +import WdioCucumberTestFramework from '../../src/cli/frameworks/wdioCucumberTestFramework.js' +import WdioJasmineTestFramework from '../../src/cli/frameworks/wdioJasmineTestFramework.js' + +vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) + +describe('CLIUtils.checkCLISupportedFrameworks', () => { + it.each(['mocha', 'cucumber', 'jasmine'])('routes %s to the CLI flow', (framework) => { + expect(CLIUtils.checkCLISupportedFrameworks(framework)).toBe(true) + }) + + it.each([undefined, 'WebdriverIO-jasmine', 'Jasmine', 'unknown'])('keeps %s on the legacy flow', (framework) => { + expect(CLIUtils.checkCLISupportedFrameworks(framework)).toBe(false) + }) +}) + +describe('BrowserstackCLI.setupTestFramework', () => { + const cli = BrowserstackCLI.getInstance() + + afterEach(() => { + vi.restoreAllMocks() + cli.testFramework = null + }) + + const setup = (name: string) => { + vi.spyOn(CLIUtils, 'getTestFrameworkDetail').mockReturnValue({ name, version: { [name]: '9.0.0' } }) + cli.setupTestFramework() + return cli.getTestFramework() + } + + it('constructs the jasmine framework for WebdriverIO-jasmine', () => { + const framework = setup('WebdriverIO-jasmine') + expect(framework).toBeInstanceOf(WdioJasmineTestFramework) + expect(framework).not.toBeInstanceOf(WdioMochaTestFramework) + expect(framework).toBeInstanceOf(TestFramework) + expect(framework!.getTestFrameworks()).toEqual(['WebdriverIO-jasmine']) + expect(framework!.getTestFrameworksVersions()).toEqual({ 'WebdriverIO-jasmine': '9.0.0' }) + }) + + it('still constructs the mocha framework for WebdriverIO-mocha', () => { + expect(setup('WebdriverIO-mocha')).toBeInstanceOf(WdioMochaTestFramework) + }) + + it('still constructs the cucumber framework for WebdriverIO-cucumber', () => { + expect(setup('WebdriverIO-cucumber')).toBeInstanceOf(WdioCucumberTestFramework) + }) + + it('leaves an unknown name unregistered', () => { + expect(setup('WebdriverIO-unknown')).toBeNull() + }) +}) From 3ae9e25ce118bf0f5daa2432cd24c45268a109b2 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 09:54:51 +0530 Subject: [PATCH 02/24] feat(browserstack-service): add WdioJasmineTestFramework and reporter CLI feed (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../frameworks/wdioJasmineTestFramework.ts | 363 +++++++++++++++++- packages/browserstack-service/src/reporter.ts | 52 +++ .../cli/wdioJasmineTestFramework.test.ts | 295 ++++++++++++++ .../tests/reporter.jasmineCli.test.ts | 188 +++++++++ 4 files changed, 896 insertions(+), 2 deletions(-) create mode 100644 packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts create mode 100644 packages/browserstack-service/tests/reporter.jasmineCli.test.ts diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts index 68524b06..45dde1af 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -1,16 +1,375 @@ +import path from 'node:path' +import util from 'node:util' +import { v4 as uuidv4 } from 'uuid' +import type { Frameworks } from '@wdio/types' +import type { HookStats, TestStats } from '@wdio/reporter' + import TestFramework from './testFramework.js' +import { TestFrameworkState } from '../states/testFrameworkState.js' +import { HookState } from '../states/hookState.js' +import TestFrameworkInstance from '../instances/testFrameworkInstance.js' +import TrackedInstance from '../instances/trackedInstance.js' +import { CLIUtils } from '../cliUtils.js' +import { TestFrameworkConstants } from './constants/testFrameworkConstants.js' import { BStackLogger as logger } from '../cliLogger.js' +import { getHookType, getTestTags, removeAnsiColors } from '../../util.js' +import { shouldProcessEventForTesthub } from '../../testHub/utils.js' +import { TEST_ANALYTICS_ID } from '../../constants.js' + +/** What the reporter knows at each event and the framework cannot see: its suite stack and suite file. */ +export interface JasmineSuiteContext { + scopes: string[] + suiteFile?: string +} + +interface FailureFields { + failure?: Array<{ backtrace: string[] }> + failureReason?: string + failureType?: string | null +} /** * CLI test framework for `framework: 'jasmine'` under WebdriverIO. * * Extends the BASE TestFramework, never WdioMochaTestFramework: jasmine's identity (fullName), - * hook taxonomy (`hookName`) and skip/exclude states differ from mocha's, and inheriting mocha's - * event model would report jasmine specs through the wrong runner's semantics. + * hook taxonomy and skip/exclude states differ from mocha's. + * + * Test and hook states come from the WDIO reporter (`reporter.ts`), the only jasmine source that + * sees pending, focused-out and beforeAll-failed specs. Reporter hooks are not awaited by WDIO, so + * every reporter event is applied and dispatched through one ordered queue. */ export default class WdioJasmineTestFramework extends TestFramework { + // Same data keys WdioMochaTestFramework uses: TestHubModule's LOG arm reads them through + // WdioMochaTestFramework.getLogEntries/clearLogs. + static KEY_HOOK_LAST_STARTED = 'test_hook_last_started' + static KEY_HOOK_LAST_FINISHED = 'test_hook_last_finished' + static KEY_TEST_RETRIES = 'test_retries' + static KEY_TEST_DURATION = 'test_duration' + static KEY_HOOK_IDENTIFIER = 'hook_identifier' + static KEY_HOOK_SCOPE = 'hook_scope' + static KEY_HOOK_SCOPES = 'hook_scopes' + static KEY_HOOK_DURATION = 'hook_duration' + static KEY_HOOK_FAILURE = 'hook_failure' + static KEY_HOOK_FAILURE_TYPE = 'hook_failure_type' + static KEY_HOOK_FAILURE_REASON = 'hook_failure_reason' + + static #pendingEvents = 0 + + #specInstances = new Map() + // beforeAll/afterAll never overlap, so a hook's uid (its title) is unique while it is open. + #hookInstances = new Map() + #openHook: TestFrameworkInstance | null = null + #lastSpec: TestFrameworkInstance | null = null + #queue: Promise = Promise.resolve() + constructor(testFrameworks: string[], testFrameworkVersions: Record, binSessionId: string) { super(testFrameworks, testFrameworkVersions, binSessionId) logger.debug('WdioJasmineTestFramework: constructed') } + + /** True when no reporter event is still waiting to be dispatched (the reporter's `isSynchronised`). */ + static isIdle() { + return WdioJasmineTestFramework.#pendingEvents === 0 + } + + /** + * Service hooks (beforeTest/afterTest and friends) also call this. Their TEST/INIT_TEST/LOG_REPORT + * states would duplicate what the reporter already dispatched, so they are absorbed; waiting for the + * queue first leaves the tracked instance pointing at the spec the service is running. + */ + async trackEvent(testFrameworkState: State, hookState: State, args: Record = {}) { + if (testFrameworkState === TestFrameworkState.LOG) { + this.onReporterLog(args.logEntry as Record) + return + } + await this.#queue + logger.debug(`WdioJasmineTestFramework: absorbed service event ${testFrameworkState}/${hookState}`) + } + + onReporterTestStart(testStats: TestStats, context: JasmineSuiteContext) { + try { + logger.debug(`WdioJasmineTestFramework: test start uid=${testStats.uid}`) + const instance = this.#createInstance() + const scopes = [...context.scopes] + const fullTitle = testStats.fullTitle + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_UUID]: uuidv4(), + [TestFrameworkConstants.KEY_TEST_RESULT]: TestFrameworkConstants.DEFAULT_TEST_RESULT, + [TestFrameworkConstants.KEY_TEST_ID]: fullTitle, + [TestFrameworkConstants.KEY_TEST_NAME]: testStats.title, + [TestFrameworkConstants.KEY_TEST_SCOPE]: fullTitle, + [TestFrameworkConstants.KEY_TEST_SCOPES]: scopes, + [TestFrameworkConstants.KEY_TEST_TAGS]: getTestTags(testStats as unknown as Frameworks.Test, scopes), + ...this.#filePaths(context.suiteFile), + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: this.#iso(testStats.start), + [WdioJasmineTestFramework.KEY_TEST_DURATION]: testStats._duration, + [WdioJasmineTestFramework.KEY_TEST_RETRIES]: { limit: testStats.retries || 0, attempts: testStats.retries || 0 }, + }) + this.#specInstances.set(testStats.uid, instance) + // Registered synchronously: the service's beforeTest for this spec may read it before the queue runs. + TestFramework.setTrackedInstance(instance.getContext(), instance) + + const args = { test: this.#specArg(testStats, context), suiteTitle: this.#suiteTitle(testStats) } + this.#enqueue('TEST/PRE', async () => { + this.#lastSpec = instance + process.env[TEST_ANALYTICS_ID] = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) + await this.#emit(instance, TestFrameworkState.TEST, HookState.PRE, args) + }) + return TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) as string + } catch (error) { + logger.error(`WdioJasmineTestFramework: test start failed: ${util.format(error)}`) + } + } + + onReporterTestEnd(testStats: TestStats, context: JasmineSuiteContext) { + try { + const instance = this.#specInstances.get(testStats.uid) + if (!instance) { + logger.debug(`WdioJasmineTestFramework: no instance for test end uid=${testStats.uid}`) + return + } + this.#specInstances.delete(testStats.uid) + + const state = testStats.state + const error = testStats.error + let result: string = state + let failure: FailureFields = {} + if (state === 'failed') { + result = (error && error.message && error.message.includes('sync skip; aborting execution')) ? 'ignore' : 'failed' + if (error) { + failure = this.#failureFields(error) + } + } + const entries: Record = { + [TestFrameworkConstants.KEY_TEST_RESULT]: result, + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: this.#iso(testStats.start), + [TestFrameworkConstants.KEY_TEST_ENDED_AT]: this.#iso(testStats.end), + [WdioJasmineTestFramework.KEY_TEST_DURATION]: testStats._duration, + [TestFrameworkConstants.KEY_TEST_RESULT_AT]: new Date().toISOString(), + } + if (failure.failure) { + entries[TestFrameworkConstants.KEY_TEST_FAILURE] = failure.failure + entries[TestFrameworkConstants.KEY_TEST_FAILURE_REASON] = failure.failureReason + entries[TestFrameworkConstants.KEY_TEST_FAILURE_TYPE] = failure.failureType + } + const testResult = { + passed: state === 'passed', + skipped: state === 'skipped', + error, + duration: testStats._duration, + retries: { limit: testStats.retries || 0, attempts: testStats.retries || 0 }, + } + const args = { test: this.#specArg(testStats, context), result: testResult, suiteTitle: this.#suiteTitle(testStats) } + this.#enqueue('TEST/POST', async () => { + instance.updateMultipleEntries(entries) + await this.#emit(instance, TestFrameworkState.TEST, HookState.POST, args) + }) + } catch (error) { + logger.error(`WdioJasmineTestFramework: test end failed: ${util.format(error)}`) + } + } + + onReporterHookStart(hookStats: HookStats, context: JasmineSuiteContext) { + try { + const title = hookStats.title + const key = getHookType(String(title ?? '').toLowerCase()) + const hookFrameworkState = WdioJasmineTestFramework.#hookState(key) + if (!hookFrameworkState || context.scopes.length === 0) { + logger.debug(`WdioJasmineTestFramework: hook start not reported title=${title} scopes=${context.scopes.length}`) + return + } + const scopes = [...context.scopes] + const instance = this.#createInstance() + const hookId = uuidv4() + const hook: Record = { + key, + [TestFrameworkConstants.KEY_HOOK_ID]: hookId, + [TestFrameworkConstants.KEY_HOOK_RESULT]: TestFrameworkConstants.DEFAULT_HOOK_RESULT, + [TestFrameworkConstants.KEY_EVENT_STARTED_AT]: this.#iso(hookStats.start), + [TestFrameworkConstants.KEY_HOOK_LOGS]: [], + [TestFrameworkConstants.KEY_HOOK_NAME]: title, + [WdioJasmineTestFramework.KEY_HOOK_IDENTIFIER]: `${title} for ${scopes[scopes.length - 1]}`, + [WdioJasmineTestFramework.KEY_HOOK_SCOPE]: `${scopes[0]} - ${title}`, + [WdioJasmineTestFramework.KEY_HOOK_SCOPES]: scopes, + [WdioJasmineTestFramework.KEY_HOOK_DURATION]: hookStats._duration, + ...this.#filePaths(context.suiteFile), + } + instance.updateData(TestFrameworkConstants.KEY_HOOK_ID, hookId) + this.#hookInstances.set(hookStats.uid, instance) + + const emitHook = shouldProcessEventForTesthub('HookRunStarted') + this.#enqueue(`${key}/PRE`, async () => { + this.#openHook = instance + const hooksStarted = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_STARTED) as Map + hooksStarted.set(key, [hook]) + instance.updateData(WdioJasmineTestFramework.KEY_HOOK_LAST_STARTED, key) + if (emitHook) { + await this.#emit(instance, hookFrameworkState, HookState.PRE, { hook: hookStats }) + } + }) + } catch (error) { + logger.error(`WdioJasmineTestFramework: hook start failed: ${util.format(error)}`) + } + } + + onReporterHookEnd(hookStats: HookStats) { + try { + const instance = this.#hookInstances.get(hookStats.uid) + if (!instance) { + return + } + this.#hookInstances.delete(hookStats.uid) + const key = getHookType(String(hookStats.title ?? '').toLowerCase()) + const hookFrameworkState = WdioJasmineTestFramework.#hookState(key)! + + const state = hookStats.state + const error = hookStats.error + let result = state as string | undefined + let failure: FailureFields = {} + if (state === 'failed') { + result = (error && error.message && error.message.includes('sync skip; aborting execution')) ? 'ignore' : 'failed' + if (error) { + failure = this.#failureFields(error) + } + } + const endedAt = this.#iso(hookStats.end) + const duration = hookStats._duration + const emitHook = shouldProcessEventForTesthub('HookRunFinished') + this.#enqueue(`${key}/POST`, async () => { + if (this.#openHook === instance) { + this.#openHook = null + } + const hooksStarted = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_STARTED) as Map[]> + const hooksFinished = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_FINISHED) as Map[]> + const hook = hooksStarted.get(key)?.pop() + if (!hook) { + return + } + if (result) { + hook[TestFrameworkConstants.KEY_HOOK_RESULT] = result + } + hook[TestFrameworkConstants.KEY_EVENT_ENDED_AT] = endedAt + hook[WdioJasmineTestFramework.KEY_HOOK_DURATION] = duration + if (failure.failure) { + hook[WdioJasmineTestFramework.KEY_HOOK_FAILURE] = failure.failure + hook[WdioJasmineTestFramework.KEY_HOOK_FAILURE_REASON] = failure.failureReason + hook[WdioJasmineTestFramework.KEY_HOOK_FAILURE_TYPE] = failure.failureType + } + hooksFinished.set(key, [hook]) + instance.updateData(WdioJasmineTestFramework.KEY_HOOK_LAST_FINISHED, key) + if (emitHook) { + await this.#emit(instance, hookFrameworkState, HookState.POST, { hook: hookStats, result: { passed: result === 'passed', error } }) + } + }) + } catch (error) { + logger.error(`WdioJasmineTestFramework: hook end failed: ${util.format(error)}`) + } + } + + /** Console logs and screenshots: an open beforeAll/afterAll wins, else the last-started spec, even after it ended. */ + onReporterLog(logEntry: Record | undefined) { + try { + if (!logEntry || !shouldProcessEventForTesthub('LogCreated')) { + return + } + this.#enqueue('LOG/POST', async () => { + const instance = this.#openHook ?? this.#lastSpec + if (!instance) { + return + } + const logRecord: Record = { + kind: logEntry.kind ?? TestFrameworkConstants.KIND_LOG, + message: Buffer.from(String(logEntry.message ?? '')), + level: logEntry.level, + timestamp: logEntry.timestamp, + } + if (instance === this.#openHook) { + logRecord[TestFrameworkConstants.KEY_HOOK_ID] = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOK_ID) + } + const entries = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_LOGS) as unknown[] + entries.push(logRecord) + await this.#emit(instance, TestFrameworkState.LOG, HookState.POST, { logEntry }) + }) + } catch (error) { + logger.error(`WdioJasmineTestFramework: log failed: ${util.format(error)}`) + } + } + + #createInstance() { + const trackedContext = TrackedInstance.createContext(CLIUtils.getCurrentInstanceName()) + const instance = new TestFrameworkInstance( + trackedContext, + this.getTestFrameworks(), + this.getTestFrameworksVersions(), + TestFrameworkState.NONE, + HookState.NONE + ) + const frameworkName = this.getTestFrameworks()[0] + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME]: frameworkName, + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_VERSION]: this.getTestFrameworksVersions()[frameworkName], + [TestFrameworkConstants.KEY_TEST_LOGS]: [], + [TestFrameworkConstants.KEY_HOOKS_STARTED]: new Map(), + [TestFrameworkConstants.KEY_HOOKS_FINISHED]: new Map(), + }) + return instance + } + + async #emit(instance: TestFrameworkInstance, testFrameworkState: State, hookState: State, args: Record) { + TestFramework.setTrackedInstance(instance.getContext(), instance) + this.updateInstanceState(instance, testFrameworkState, hookState) + args.instance = instance + await this.runHooks(instance, testFrameworkState, hookState, args) + } + + #enqueue(label: string, step: () => Promise) { + WdioJasmineTestFramework.#pendingEvents++ + this.#queue = this.#queue.then(step).catch((error) => { + logger.error(`WdioJasmineTestFramework: ${label} failed: ${util.format(error)}`) + }).finally(() => { + WdioJasmineTestFramework.#pendingEvents-- + }) + } + + /** The binary re-bases `test_file_path` itself, so it must be absolute. */ + #filePaths(suiteFile: string | undefined) { + const absolute = suiteFile ? path.resolve(suiteFile) : undefined + return { + [TestFrameworkConstants.KEY_TEST_FILE_PATH]: absolute, + [TestFrameworkConstants.KEY_TEST_LOCATION]: absolute ? path.relative(process.cwd(), absolute) : undefined, + } + } + + #failureFields(error: Error): FailureFields { + return { + failure: [{ backtrace: [removeAnsiColors(error.message), removeAnsiColors(error.stack || '')] }], + failureReason: removeAnsiColors(error.message), + failureType: error.message === null ? null : error.message.toString().match(/AssertionError/) ? 'AssertionError' : 'UnhandledError', + } + } + + /** SpecResult-shaped, like the service hooks' `test`: jasmine has no `title`. */ + #specArg(testStats: TestStats, context: JasmineSuiteContext) { + return { description: testStats.title, fullName: testStats.fullTitle, file: context.suiteFile } as unknown as Frameworks.Test + } + + /** The describe chain, derived the way `service.beforeTest` does for jasmine. */ + #suiteTitle(testStats: TestStats) { + const fullName = testStats.fullTitle || '' + return fullName.slice(0, fullName.indexOf(testStats.title || '') - 1) + } + + #iso(date: Date | undefined) { + return date ? date.toISOString() : undefined + } + + static #hookState(key: string) { + if (key === 'BEFORE_ALL') { + return TestFrameworkState.BEFORE_ALL + } + if (key === 'AFTER_ALL') { + return TestFrameworkState.AFTER_ALL + } + return null + } } diff --git a/packages/browserstack-service/src/reporter.ts b/packages/browserstack-service/src/reporter.ts index 68d77738..3e684124 100644 --- a/packages/browserstack-service/src/reporter.ts +++ b/packages/browserstack-service/src/reporter.ts @@ -5,6 +5,8 @@ import WDIOReporter from '@wdio/reporter' import type { Options, Frameworks } from '@wdio/types' import { BrowserstackCLI } from './cli/index.js' import { reportSkippedTest, resolveSpecFile } from './cli/skipReporter.js' +import WdioJasmineTestFramework from './cli/frameworks/wdioJasmineTestFramework.js' +import type { JasmineSuiteContext } from './cli/frameworks/wdioJasmineTestFramework.js' import * as url from 'node:url' import { v4 as uuidv4 } from 'uuid' @@ -72,6 +74,10 @@ class _TestReporter extends WDIOReporter { } public async appendTestItemLog(stdLog: StdLog) { + if (this.isCliJasmine()) { + this.cliJasmineFramework()?.onReporterLog(stdLog as unknown as Record) + return + } if (this._currentHook.uuid && !this._currentHook.finished) { stdLog.hook_run_uuid = this._currentHook.uuid } else if (_TestReporter.currentTest.uuid) { @@ -134,6 +140,29 @@ class _TestReporter extends WDIOReporter { this._suites.pop() } + /** + * WDIO waits for reporters to synchronise before the worker exits; the CLI jasmine feed dispatches + * asynchronously, so hold the worker until it has drained. Always true on every other path. + */ + get isSynchronised() { + return WdioJasmineTestFramework.isIdle() + } + + /** On the CLI flow jasmine's test/hook/log events feed the framework tracker, never the legacy Listener. */ + isCliJasmine() { + return this._config?.framework === 'jasmine' && BrowserstackCLI.getInstance().isRunning() + } + + cliJasmineFramework() { + const framework = BrowserstackCLI.getInstance().getTestFramework() + return framework instanceof WdioJasmineTestFramework ? framework : null + } + + jasmineSuiteContext(): JasmineSuiteContext { + const suiteFile = this._suiteName || (this.specs?.length > 0 ? this.specs[this.specs.length - 1]?.replace('file:', '') : undefined) + return { scopes: this._suites.map(s => s.title), suiteFile } + } + needToSendData(testType?: string, event?: string) { if (!this._observability) {return false} @@ -158,6 +187,10 @@ class _TestReporter extends WDIOReporter { } testStats.end ||= new Date() + if (this.isCliJasmine()) { + this.cliJasmineFramework()?.onReporterTestEnd(testStats, this.jasmineSuiteContext()) + return + } this.listener.testFinished(await this.getRunData(testStats, 'TestRunFinished')) } @@ -168,6 +201,14 @@ class _TestReporter extends WDIOReporter { if (testStats.fullTitle === '') { return } + if (this.isCliJasmine()) { + const cliUuid = this.cliJasmineFramework()?.onReporterTestStart(testStats, this.jasmineSuiteContext()) + if (cliUuid) { + _TestReporter.currentTest.uuid = cliUuid + _TestReporter._tests[testStats.fullTitle] = { uuid: cliUuid } + } + return + } const uuid = uuidv4() _TestReporter.currentTest.uuid = uuid @@ -181,6 +222,10 @@ class _TestReporter extends WDIOReporter { if (!this.needToSendData('hook', 'start')) { return } + if (this.isCliJasmine()) { + this.cliJasmineFramework()?.onReporterHookStart(hookStats, this.jasmineSuiteContext()) + return + } const identifier = this.getHookIdentifier(hookStats) const hookId = uuidv4() @@ -196,6 +241,13 @@ class _TestReporter extends WDIOReporter { if (!this.needToSendData('hook', 'end')) { return } + if (this.isCliJasmine()) { + if (!hookStats.state && !hookStats.error) { + hookStats.state = 'passed' + } + this.cliJasmineFramework()?.onReporterHookEnd(hookStats) + return + } const identifier = this.getHookIdentifier(hookStats) if (_TestReporter._tests[identifier]) { _TestReporter._tests[identifier].finishedAt = (new Date()).toISOString() diff --git a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts new file mode 100644 index 00000000..8eec475f --- /dev/null +++ b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts @@ -0,0 +1,295 @@ +import path from 'node:path' +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' + +import * as bstackLogger from '../../src/bstackLogger.js' +import WdioJasmineTestFramework from '../../src/cli/frameworks/wdioJasmineTestFramework.js' +import TestFramework from '../../src/cli/frameworks/testFramework.js' +import { TestFrameworkState } from '../../src/cli/states/testFrameworkState.js' +import { HookState } from '../../src/cli/states/hookState.js' +import type TestFrameworkInstance from '../../src/cli/instances/testFrameworkInstance.js' + +vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) + +const SUITE_FILE = path.join(process.cwd(), 'test/p2/nested.spec.js') +const BEFORE_ALL_REASON = 'Not run because a beforeAll function failed. The beforeAll failure will be reported on the suite that caused it.' + +interface Dispatch { + state: State + hook: State + data: Record + args: Record +} + +const snapshot = (instance: TestFrameworkInstance) => + JSON.parse(JSON.stringify(Object.fromEntries(instance.getAllData()), (_k, v) => v instanceof Map ? Object.fromEntries(v) : v)) + +const testStats = (overrides: Record = {}) => ({ + type: 'test', + uid: 'outer passing test0', + title: 'outer passing test', + fullTitle: 'Nested outer outer passing test', + start: new Date('2026-09-25T15:34:12.259Z'), + end: undefined as Date | undefined, + _duration: 0, + retries: 0, + state: 'pending', + ...overrides, +}) + +const hookStats = (title: string, overrides: Record = {}) => ({ + type: 'hook', + uid: title, + title, + start: new Date('2026-09-25T15:34:08.170Z'), + end: undefined as Date | undefined, + _duration: 0, + ...overrides, +}) + +const context = (scopes = ['Nested outer']) => ({ scopes, suiteFile: SUITE_FILE }) + +describe('WdioJasmineTestFramework', () => { + let framework: WdioJasmineTestFramework + let dispatches: Dispatch[] + + const drain = () => framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, {}) + + beforeEach(() => { + process.env.BROWSERSTACK_OBSERVABILITY = 'true' + framework = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin-session') + dispatches = [] + vi.spyOn(framework, 'runHooks').mockImplementation(async (instance, state, hook, args) => { + dispatches.push({ state, hook, data: snapshot(instance), args: args as Record }) + }) + }) + + afterEach(() => { + vi.restoreAllMocks() + delete process.env.BROWSERSTACK_OBSERVABILITY + }) + + it('sends legacy identity for a spec, char-for-char with the CP0 TestRun', async () => { + framework.onReporterTestStart(testStats() as any, context()) + await drain() + + expect(dispatches).toHaveLength(1) + const [{ state, hook, data, args }] = dispatches + expect([state, hook]).toEqual([TestFrameworkState.TEST, HookState.PRE]) + expect(data.test_name).toBe('outer passing test') + expect(data.test_id).toBe('Nested outer outer passing test') + expect(data.test_scope).toBe('Nested outer outer passing test') + expect(data.test_scopes).toEqual(['Nested outer']) + expect(data.test_tags).toEqual([]) + expect(data.test_file_path).toBe(SUITE_FILE) + expect(data.test_location).toBe('test/p2/nested.spec.js') + expect(data.test_framework_name).toBe('WebdriverIO-jasmine') + expect(data.test_retries).toEqual({ limit: 0, attempts: 0 }) + expect(data.test_started_at).toBe('2026-09-25T15:34:12.259Z') + expect(data.test_duration).toBe(0) + expect(data.test_result).toBe('pending') + expect('test_code' in data).toBe(false) + expect(args.suiteTitle).toBe('Nested outer') + expect(args.test).toEqual({ description: 'outer passing test', fullName: 'Nested outer outer passing test', file: SUITE_FILE }) + }) + + it('keeps @tags from describes and the spec, sigil included', async () => { + framework.onReporterTestStart(testStats({ title: 'tagged @smoke', fullTitle: 'Outer @regression tagged @smoke' }) as any, context(['Outer @regression'])) + await drain() + expect(dispatches[0].data.test_tags).toEqual(['@regression', '@smoke']) + }) + + it('finishes a passed spec on the same uuid with reporter timing and the result timestamp', async () => { + const stats = testStats() + framework.onReporterTestStart(stats as any, context()) + framework.onReporterTestEnd({ ...stats, state: 'passed', end: new Date('2026-09-25T15:34:16.645Z'), _duration: 4386 } as any, context()) + await drain() + + expect(dispatches.map(d => [d.state, d.hook])).toEqual([[TestFrameworkState.TEST, HookState.PRE], [TestFrameworkState.TEST, HookState.POST]]) + const [start, finish] = dispatches + expect(finish.data.test_uuid).toBe(start.data.test_uuid) + expect(finish.data.test_result).toBe('passed') + expect(finish.data.test_started_at).toBe('2026-09-25T15:34:12.259Z') + expect(finish.data.test_ended_at).toBe('2026-09-25T15:34:16.645Z') + expect(finish.data.test_duration).toBe(4386) + expect(typeof finish.data.test_result_at).toBe('string') + expect('test_failure' in finish.data).toBe(false) + expect(finish.args.result).toMatchObject({ passed: true, skipped: false }) + }) + + it('reports a pending spec as Start + Finish(skipped)', async () => { + framework.onReporterTestStart(testStats({ uid: 'xit0', title: 'xit skipped test', fullTitle: 'Pending suite xit skipped test' }) as any, context(['Pending suite'])) + // @wdio/reporter replaces the TestStats object on test:pending; only the uid carries over + framework.onReporterTestEnd(testStats({ uid: 'xit0', title: 'xit skipped test', fullTitle: 'Pending suite xit skipped test', state: 'skipped', end: new Date() }) as any, context(['Pending suite'])) + await drain() + + expect(dispatches).toHaveLength(2) + expect(dispatches[1].data.test_result).toBe('skipped') + expect(dispatches[1].data.test_duration).toBe(0) + expect(dispatches[1].args.result).toMatchObject({ passed: false, skipped: true }) + }) + + it('keeps jasmine\'s exact reason on a beforeAll-failed child', async () => { + const error = { message: BEFORE_ALL_REASON, stack: '' } + framework.onReporterTestStart(testStats({ uid: 'child0' }) as any, context()) + framework.onReporterTestEnd(testStats({ uid: 'child0', state: 'failed', error, end: new Date() }) as any, context()) + await drain() + + const finish = dispatches[1].data + expect(finish.test_result).toBe('failed') + expect(finish.test_failure_reason).toBe(BEFORE_ALL_REASON) + expect(finish.test_failure_type).toBe('UnhandledError') + expect(finish.test_failure).toEqual([{ backtrace: [BEFORE_ALL_REASON, ''] }]) + }) + + it('maps an AssertionError message to failure_type AssertionError', async () => { + framework.onReporterTestStart(testStats() as any, context()) + framework.onReporterTestEnd(testStats({ state: 'failed', error: { message: 'AssertionError: nope', stack: 'at x' } }) as any, context()) + await drain() + expect(dispatches[1].data.test_failure_type).toBe('AssertionError') + }) + + it('gives overlapping specs distinct instances and closes each on its own uuid', async () => { + framework.onReporterTestStart(testStats({ uid: 'a' }) as any, context()) + framework.onReporterTestStart(testStats({ uid: 'b', title: 'b', fullTitle: 'Nested outer b' }) as any, context()) + framework.onReporterTestEnd(testStats({ uid: 'a', state: 'passed', end: new Date() }) as any, context()) + framework.onReporterTestEnd(testStats({ uid: 'b', title: 'b', fullTitle: 'Nested outer b', state: 'passed', end: new Date() }) as any, context()) + await drain() + + const uuidOf = (i: number) => dispatches[i].data.test_uuid + expect(uuidOf(0)).not.toBe(uuidOf(1)) + expect(uuidOf(2)).toBe(uuidOf(0)) + expect(uuidOf(3)).toBe(uuidOf(1)) + expect(dispatches[2].data.test_id).toBe('Nested outer outer passing test') + }) + + it('reports beforeAll/afterAll with legacy hook identity and no test linkage', async () => { + const before = hookStats('"before all" hook') + framework.onReporterHookStart(before as any, context()) + framework.onReporterHookEnd({ ...before, state: 'passed', end: new Date('2026-09-25T15:34:12.257Z'), _duration: 4087 } as any) + const after = hookStats('"after all" hook') + framework.onReporterHookStart(after as any, context(['Nested outer', 'Nested middle'])) + framework.onReporterHookEnd({ ...after, state: 'passed', end: new Date() } as any) + await drain() + + expect(dispatches.map(d => [d.state, d.hook])).toEqual([ + [TestFrameworkState.BEFORE_ALL, HookState.PRE], + [TestFrameworkState.BEFORE_ALL, HookState.POST], + [TestFrameworkState.AFTER_ALL, HookState.PRE], + [TestFrameworkState.AFTER_ALL, HookState.POST], + ]) + const started = (dispatches[0].data.test_hooks_started as Record[]>).BEFORE_ALL[0] + expect(started).toMatchObject({ + key: 'BEFORE_ALL', + hook_name: '"before all" hook', + hook_identifier: '"before all" hook for Nested outer', + hook_scope: 'Nested outer - "before all" hook', + hook_scopes: ['Nested outer'], + hook_result: 'pending', + event_started_at: '2026-09-25T15:34:08.170Z', + test_file_path: SUITE_FILE, + }) + const finished = (dispatches[1].data.test_hooks_finished as Record[]>).BEFORE_ALL[0] + expect(finished).toMatchObject({ hook_result: 'passed', event_ended_at: '2026-09-25T15:34:12.257Z', hook_duration: 4087 }) + expect(finished.hook_id).toBe(started.hook_id) + expect('hook_failure' in finished).toBe(false) + expect('test_uuid' in dispatches[0].data).toBe(false) + + const afterStarted = (dispatches[2].data.test_hooks_started as Record[]>).AFTER_ALL[0] + expect(afterStarted.hook_identifier).toBe('"after all" hook for Nested middle') + expect(afterStarted.hook_scope).toBe('Nested outer - "after all" hook') + }) + + it('carries failure fields on a failed beforeAll', async () => { + const before = hookStats('"before all" hook') + framework.onReporterHookStart(before as any, context()) + framework.onReporterHookEnd({ ...before, state: 'failed', error: { message: 'boom', stack: 'Error: boom' }, end: new Date() } as any) + await drain() + const finished = (dispatches[1].data.test_hooks_finished as Record[]>).BEFORE_ALL[0] + expect(finished).toMatchObject({ + hook_result: 'failed', + hook_failure: [{ backtrace: ['boom', 'Error: boom'] }], + hook_failure_reason: 'boom', + hook_failure_type: 'UnhandledError', + }) + }) + + it('never reports each-hooks, or a hook outside any describe', async () => { + framework.onReporterHookStart(hookStats('"before each" hook') as any, context()) + framework.onReporterHookStart(hookStats('"before all" hook') as any, context([])) + await drain() + expect(dispatches).toHaveLength(0) + }) + + it('suppresses hook and log families in an accessibility-only run, like the legacy Listener', async () => { + delete process.env.BROWSERSTACK_OBSERVABILITY + process.env.BROWSERSTACK_ACCESSIBILITY = 'true' + try { + const before = hookStats('"before all" hook') + framework.onReporterHookStart(before as any, context()) + framework.onReporterHookEnd({ ...before, state: 'passed', end: new Date() } as any) + framework.onReporterTestStart(testStats() as any, context()) + framework.onReporterLog({ level: 'INFO', message: 'hi', timestamp: 't', kind: 'TEST_LOG' }) + await drain() + expect(dispatches.map(d => d.state)).toEqual([TestFrameworkState.TEST]) + } finally { + delete process.env.BROWSERSTACK_ACCESSIBILITY + } + }) + + it('absorbs service-hook states instead of re-dispatching them, leaving the spec tracked', async () => { + framework.onReporterTestStart(testStats() as any, context()) + await framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test: {} }) + const tracked = TestFramework.getTrackedInstance() + expect(TestFramework.getState(tracked, 'test_uuid')).toBe(dispatches[0].data.test_uuid) + + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { test: {} }) + await framework.trackEvent(TestFrameworkState.LOG_REPORT, HookState.POST, { test: {}, result: {} }) + await framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { test: {}, result: {} }) + expect(dispatches).toHaveLength(1) + }) + + it('attributes logs to an open all-hook, else to the last-started spec', async () => { + framework.onReporterTestStart(testStats() as any, context()) + framework.onReporterLog({ level: 'INFO', message: 'in test', timestamp: 't1', kind: 'TEST_LOG' }) + const after = hookStats('"after all" hook') + framework.onReporterHookStart(after as any, context()) + framework.onReporterLog({ level: 'INFO', message: 'in hook', timestamp: 't2', kind: 'TEST_LOG' }) + framework.onReporterHookEnd({ ...after, state: 'passed', end: new Date() } as any) + framework.onReporterLog({ level: 'INFO', message: 'after hook', timestamp: 't3', kind: 'TEST_LOG' }) + await drain() + + const logs = dispatches.filter(d => d.state === TestFrameworkState.LOG) + expect(logs).toHaveLength(3) + const specUuid = dispatches[0].data.test_uuid + expect(logs[0].data.test_uuid).toBe(specUuid) + expect((logs[0].data.test_logs as Record[]).at(-1)).not.toHaveProperty('hook_id') + const hookLog = (logs[1].data.test_logs as Record[]).at(-1)! + expect(hookLog.hook_id).toBe((dispatches.find(d => d.state === TestFrameworkState.AFTER_ALL)!.data.test_hooks_started as Record[]>).AFTER_ALL[0].hook_id) + expect(logs[2].data.test_uuid).toBe(specUuid) + }) + + it('keeps a screenshot entry\'s kind on the log path', async () => { + framework.onReporterTestStart(testStats() as any, context()) + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: 't', level: 'INFO' } }) + await drain() + const log = dispatches.find(d => d.state === TestFrameworkState.LOG)! + expect((log.data.test_logs as Record[])[0].kind).toBe('TEST_SCREENSHOT') + }) + + it('is not idle until queued events are dispatched', async () => { + framework.onReporterTestStart(testStats() as any, context()) + expect(WdioJasmineTestFramework.isIdle()).toBe(false) + await drain() + expect(WdioJasmineTestFramework.isIdle()).toBe(true) + }) + + it('logs and continues when an observer throws', async () => { + vi.mocked(framework.runHooks).mockRejectedValueOnce(new Error('observer blew up')) + const stats = testStats() + framework.onReporterTestStart(stats as any, context()) + framework.onReporterTestEnd({ ...stats, state: 'passed', end: new Date() } as any, context()) + await drain() + expect(dispatches.map(d => d.hook)).toEqual([HookState.POST]) + expect(WdioJasmineTestFramework.isIdle()).toBe(true) + }) +}) diff --git a/packages/browserstack-service/tests/reporter.jasmineCli.test.ts b/packages/browserstack-service/tests/reporter.jasmineCli.test.ts new file mode 100644 index 00000000..537c8b55 --- /dev/null +++ b/packages/browserstack-service/tests/reporter.jasmineCli.test.ts @@ -0,0 +1,188 @@ +import path from 'node:path' +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' + +import TestReporter from '../src/reporter.js' +import { BrowserstackCLI } from '../src/cli/index.js' +import WdioJasmineTestFramework from '../src/cli/frameworks/wdioJasmineTestFramework.js' +import * as utils from '../src/util.js' +import * as bstackLogger from '../src/bstackLogger.js' + +vi.mock('uuid', () => ({ v4: () => 'legacy-uuid' })) +vi.mock('@wdio/reporter', () => import(path.join(process.cwd(), '__mocks__', '@wdio/reporter'))) +vi.mock('@wdio/logger', () => import(path.join(process.cwd(), '__mocks__', '@wdio/logger'))) + +vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) + +const runnerConfig = { + type: 'runner', + cid: '0-0', + capabilities: { browserName: 'chrome', browserVersion: '151' }, + config: { framework: 'jasmine', hostname: 'hub.browserstack.com' }, + specs: ['/work/test/p2/nested.spec.js'], + sessionId: 'sessionId' +} + +const suite = { title: 'Nested outer', file: '/work/test/p2/nested.spec.js' } +const testStats = () => ({ + type: 'test', + uid: 'outer passing test0', + title: 'outer passing test', + fullTitle: 'Nested outer outer passing test', + start: new Date('2026-09-25T15:34:12.259Z'), + _duration: 0, + retries: 0, + state: 'pending', +}) +const hookStats = () => ({ type: 'hook', uid: 'h', title: '"before all" hook', start: new Date(), _duration: 0 }) +const logEntry = () => ({ timestamp: new Date().toISOString(), level: 'INFO', message: 'console line', kind: 'TEST_LOG' as const, http_response: {} }) + +describe('reporter jasmine arm', () => { + let reporter: TestReporter + let listener: Record> + let framework: WdioJasmineTestFramework + + const setCli = (running: boolean, testFramework: unknown) => { + vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => running, + getTestFramework: () => testFramework, + } as unknown as BrowserstackCLI) + } + + beforeEach(async () => { + vi.spyOn(utils, 'getCloudProvider').mockReturnValue('browserstack') + vi.spyOn(utils, 'getGitMetaData').mockResolvedValue(undefined as any) + framework = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin') + for (const m of ['onReporterTestStart', 'onReporterTestEnd', 'onReporterHookStart', 'onReporterHookEnd', 'onReporterLog'] as const) { + vi.spyOn(framework, m) + } + vi.mocked(framework.onReporterTestStart).mockReturnValue('cli-uuid') + vi.mocked(framework.onReporterTestEnd).mockReturnValue(undefined) + vi.mocked(framework.onReporterHookStart).mockReturnValue(undefined) + vi.mocked(framework.onReporterHookEnd).mockReturnValue(undefined) + vi.mocked(framework.onReporterLog).mockReturnValue(undefined) + + reporter = new TestReporter({}) + await reporter.onRunnerStart(runnerConfig as any) + reporter.onSuiteStart(suite as any) + listener = { + testStarted: vi.spyOn(reporter['listener'], 'testStarted').mockImplementation(() => {}), + testFinished: vi.spyOn(reporter['listener'], 'testFinished').mockImplementation(() => {}), + hookStarted: vi.spyOn(reporter['listener'], 'hookStarted').mockImplementation(() => {}), + hookFinished: vi.spyOn(reporter['listener'], 'hookFinished').mockImplementation(() => {}), + logCreated: vi.spyOn(reporter['listener'], 'logCreated').mockImplementation(() => {}), + } + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + describe('on the CLI flow', () => { + beforeEach(() => setCli(true, framework)) + + it('feeds every test and hook event to the framework and enqueues nothing on the legacy Listener', async () => { + const stats = testStats() + await reporter.onHookStart(hookStats() as any) + await reporter.onHookEnd(hookStats() as any) + await reporter.onTestStart(stats as any) + await reporter.onTestEnd({ ...stats, state: 'passed' } as any) + await reporter.appendTestItemLog(logEntry()) + + expect(framework.onReporterHookStart).toHaveBeenCalledTimes(1) + expect(framework.onReporterHookEnd).toHaveBeenCalledTimes(1) + expect(framework.onReporterTestStart).toHaveBeenCalledTimes(1) + expect(framework.onReporterTestEnd).toHaveBeenCalledTimes(1) + expect(framework.onReporterLog).toHaveBeenCalledTimes(1) + for (const spy of Object.values(listener)) { + expect(spy).not.toHaveBeenCalled() + } + }) + + it('passes the suite stack and suite file', async () => { + await reporter.onTestStart(testStats() as any) + expect(framework.onReporterTestStart).toHaveBeenCalledWith( + expect.objectContaining({ fullTitle: 'Nested outer outer passing test' }), + { scopes: ['Nested outer'], suiteFile: '/work/test/p2/nested.spec.js' } + ) + }) + + it('records the CLI uuid for the spec so command-result lookups resolve to the wire uuid', async () => { + await reporter.onTestStart(testStats() as any) + expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: 'cli-uuid' }) + }) + + it('keeps legacy\'s end stamp and forced hook pass on the WDIO stats objects', async () => { + const stats = testStats() as Record + await reporter.onTestEnd(stats as any) + expect(stats.end).toBeInstanceOf(Date) + const hook = hookStats() as Record + await reporter.onHookEnd(hook as any) + expect(hook.state).toBe('passed') + }) + + it('drops ', async () => { + await reporter.onTestStart({ ...testStats(), fullTitle: '' } as any) + await reporter.onTestEnd({ ...testStats(), fullTitle: '' } as any) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(framework.onReporterTestEnd).not.toHaveBeenCalled() + }) + + it('sends nothing when Test Observability is opted out', async () => { + reporter['_observability'] = false + await reporter.onTestStart(testStats() as any) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(listener.testStarted).not.toHaveBeenCalled() + }) + + it('never falls back to the legacy Listener when the tracker is missing', async () => { + setCli(true, null) + await reporter.onTestStart(testStats() as any) + await reporter.appendTestItemLog(logEntry()) + expect(listener.testStarted).not.toHaveBeenCalled() + expect(listener.logCreated).not.toHaveBeenCalled() + }) + + it('reports unsynchronised while the framework has queued events', () => { + const spy = vi.spyOn(WdioJasmineTestFramework, 'isIdle').mockReturnValue(false) + expect(reporter.isSynchronised).toBe(false) + spy.mockReturnValue(true) + expect(reporter.isSynchronised).toBe(true) + }) + }) + + describe('on the legacy flow', () => { + beforeEach(() => setCli(false, null)) + + it('enqueues on the legacy Listener and never touches the framework', async () => { + const stats = testStats() + await reporter.onHookStart(hookStats() as any) + await reporter.onHookEnd(hookStats() as any) + await reporter.onTestStart(stats as any) + await reporter.onTestEnd({ ...stats, state: 'passed' } as any) + reporter['_currentHook'] = {} + await reporter.appendTestItemLog(logEntry()) + + expect(listener.hookStarted).toHaveBeenCalledTimes(1) + expect(listener.hookFinished).toHaveBeenCalledTimes(1) + expect(listener.testStarted).toHaveBeenCalledTimes(1) + expect(listener.testFinished).toHaveBeenCalledTimes(1) + expect(listener.logCreated).toHaveBeenCalledTimes(1) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(framework.onReporterLog).not.toHaveBeenCalled() + expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: 'legacy-uuid' }) + }) + }) + + describe('mocha on the CLI flow', () => { + beforeEach(() => setCli(true, framework)) + + it('leaves the mocha arm unchanged: no start/end publishing from the reporter', async () => { + reporter['_config']!.framework = 'mocha' + await reporter.onTestStart(testStats() as any) + await reporter.onTestEnd({ ...testStats(), state: 'passed' } as any) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(listener.testStarted).not.toHaveBeenCalled() + expect(listener.testFinished).not.toHaveBeenCalled() + }) + }) +}) From 362ee0e34e3c5fa4b6a6cbe3da954f5c16c85ba0 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 13:43:42 +0530 Subject: [PATCH 03/24] fix(browserstack-service): route jasmine reporter events to TestHub only (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../frameworks/wdioJasmineTestFramework.ts | 113 +++++++++++++++--- .../src/cli/modules/testHubModule.ts | 7 ++ packages/browserstack-service/src/reporter.ts | 10 +- .../modules/testHubModule.skipMarker.test.ts | 57 +++++++++ .../cli/wdioJasmineTestFramework.test.ts | 109 +++++++++++++---- 5 files changed, 255 insertions(+), 41 deletions(-) create mode 100644 packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts index 45dde1af..fe717388 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -15,6 +15,7 @@ import { BStackLogger as logger } from '../cliLogger.js' import { getHookType, getTestTags, removeAnsiColors } from '../../util.js' import { shouldProcessEventForTesthub } from '../../testHub/utils.js' import { TEST_ANALYTICS_ID } from '../../constants.js' +import type TestHubModule from '../modules/testHubModule.js' /** What the reporter knows at each event and the framework cannot see: its suite stack and suite file. */ export interface JasmineSuiteContext { @@ -34,9 +35,12 @@ interface FailureFields { * Extends the BASE TestFramework, never WdioMochaTestFramework: jasmine's identity (fullName), * hook taxonomy and skip/exclude states differ from mocha's. * - * Test and hook states come from the WDIO reporter (`reporter.ts`), the only jasmine source that - * sees pending, focused-out and beforeAll-failed specs. Reporter hooks are not awaited by WDIO, so - * every reporter event is applied and dispatched through one ordered queue. + * Two event sources, as on the legacy flow: + * - the WDIO reporter (`reporter.ts`) feeds Test Observability only — it is the one jasmine source that + * sees pending, focused-out and beforeAll-failed specs. Its hooks are not awaited by WDIO, so its events + * are applied through one ordered queue and sent straight to TestHubModule, never to the other modules; + * - the service's awaited beforeTest/afterTest drive the product modules (a11y, Automate, Percy) through + * the observers, for executed specs only, with TestHub skipped. */ export default class WdioJasmineTestFramework extends TestFramework { // Same data keys WdioMochaTestFramework uses: TestHubModule's LOG arm reads them through @@ -56,6 +60,11 @@ export default class WdioJasmineTestFramework extends TestFramework { static #pendingEvents = 0 #specInstances = new Map() + // Open specs by fullName: how the service's hooks find the instance the reporter minted. + #specsByFullName = new Map() + #serviceOnly = new Set() + #suiteTitles = new Map() + #testHub: TestHubModule | null = null // beforeAll/afterAll never overlap, so a hook's uid (its title) is unique while it is open. #hookInstances = new Map() #openHook: TestFrameworkInstance | null = null @@ -72,18 +81,47 @@ export default class WdioJasmineTestFramework extends TestFramework { return WdioJasmineTestFramework.#pendingEvents === 0 } + setTestHubModule(testHub: TestHubModule | null | undefined) { + this.#testHub = testHub ?? null + } + /** - * Service hooks (beforeTest/afterTest and friends) also call this. Their TEST/INIT_TEST/LOG_REPORT - * states would duplicate what the reporter already dispatched, so they are absorbed; waiting for the - * queue first leaves the tracked instance pointing at the spec the service is running. + * The service's hooks call this. INIT_TEST pins the spec's instance and uuid; TEST PRE/POST reach the + * product modules. LOG_REPORT carries nothing jasmine needs: the result comes from the reporter. */ async trackEvent(testFrameworkState: State, hookState: State, args: Record = {}) { if (testFrameworkState === TestFrameworkState.LOG) { this.onReporterLog(args.logEntry as Record) return } - await this.#queue - logger.debug(`WdioJasmineTestFramework: absorbed service event ${testFrameworkState}/${hookState}`) + try { + await this.#queue + if (testFrameworkState !== TestFrameworkState.INIT_TEST && testFrameworkState !== TestFrameworkState.TEST) { + return + } + const test = (args.test ?? {}) as { fullName?: string, description?: string, file?: string } + const instance = this.#serviceInstance(test) + TestFramework.setTrackedInstance(instance.getContext(), instance) + process.env[TEST_ANALYTICS_ID] = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) + if (testFrameworkState === TestFrameworkState.INIT_TEST) { + return + } + + const moduleArgs: Record = { ...args } + if (hookState === HookState.PRE) { + this.#suiteTitles.set(instance, args.suiteTitle) + } else { + // afterTest hands over the raw `Jasmine__TopLevel__Suite`; the modules need the describe chain beforeTest derived + moduleArgs.suiteTitle = this.#suiteTitles.get(instance) ?? args.suiteTitle + this.#suiteTitles.delete(instance) + if (this.#serviceOnly.delete(instance) && test.fullName && this.#specsByFullName.get(test.fullName) === instance) { + this.#specsByFullName.delete(test.fullName) + } + } + await this.#toModules(instance, testFrameworkState, hookState, moduleArgs) + } catch (error) { + logger.error(`WdioJasmineTestFramework: service event ${testFrameworkState}/${hookState} failed: ${util.format(error)}`) + } } onReporterTestStart(testStats: TestStats, context: JasmineSuiteContext) { @@ -106,14 +144,14 @@ export default class WdioJasmineTestFramework extends TestFramework { [WdioJasmineTestFramework.KEY_TEST_RETRIES]: { limit: testStats.retries || 0, attempts: testStats.retries || 0 }, }) this.#specInstances.set(testStats.uid, instance) + this.#specsByFullName.set(fullTitle, instance) // Registered synchronously: the service's beforeTest for this spec may read it before the queue runs. TestFramework.setTrackedInstance(instance.getContext(), instance) const args = { test: this.#specArg(testStats, context), suiteTitle: this.#suiteTitle(testStats) } this.#enqueue('TEST/PRE', async () => { this.#lastSpec = instance - process.env[TEST_ANALYTICS_ID] = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) - await this.#emit(instance, TestFrameworkState.TEST, HookState.PRE, args) + await this.#toTestHub(instance, TestFrameworkState.TEST, HookState.PRE, args) }) return TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) as string } catch (error) { @@ -129,6 +167,9 @@ export default class WdioJasmineTestFramework extends TestFramework { return } this.#specInstances.delete(testStats.uid) + if (this.#specsByFullName.get(testStats.fullTitle) === instance) { + this.#specsByFullName.delete(testStats.fullTitle) + } const state = testStats.state const error = testStats.error @@ -162,7 +203,7 @@ export default class WdioJasmineTestFramework extends TestFramework { const args = { test: this.#specArg(testStats, context), result: testResult, suiteTitle: this.#suiteTitle(testStats) } this.#enqueue('TEST/POST', async () => { instance.updateMultipleEntries(entries) - await this.#emit(instance, TestFrameworkState.TEST, HookState.POST, args) + await this.#toTestHub(instance, TestFrameworkState.TEST, HookState.POST, args) }) } catch (error) { logger.error(`WdioJasmineTestFramework: test end failed: ${util.format(error)}`) @@ -204,7 +245,7 @@ export default class WdioJasmineTestFramework extends TestFramework { hooksStarted.set(key, [hook]) instance.updateData(WdioJasmineTestFramework.KEY_HOOK_LAST_STARTED, key) if (emitHook) { - await this.#emit(instance, hookFrameworkState, HookState.PRE, { hook: hookStats }) + await this.#toTestHub(instance, hookFrameworkState, HookState.PRE, { hook: hookStats }) } }) } catch (error) { @@ -258,7 +299,7 @@ export default class WdioJasmineTestFramework extends TestFramework { hooksFinished.set(key, [hook]) instance.updateData(WdioJasmineTestFramework.KEY_HOOK_LAST_FINISHED, key) if (emitHook) { - await this.#emit(instance, hookFrameworkState, HookState.POST, { hook: hookStats, result: { passed: result === 'passed', error } }) + await this.#toTestHub(instance, hookFrameworkState, HookState.POST, { hook: hookStats, result: { passed: result === 'passed', error } }) } }) } catch (error) { @@ -286,9 +327,11 @@ export default class WdioJasmineTestFramework extends TestFramework { if (instance === this.#openHook) { logRecord[TestFrameworkConstants.KEY_HOOK_ID] = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOK_ID) } - const entries = TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_LOGS) as unknown[] - entries.push(logRecord) - await this.#emit(instance, TestFrameworkState.LOG, HookState.POST, { logEntry }) + // Sent in the instance's own state, not LOG: the binary keys an entry `hook_run_uuid` only when + // its state is a hook state, so an open all-hook's logs land on the hook. + if (this.#testHub) { + await this.#testHub.sendLogCreatedEvent({ instance, logEntries: [logRecord] }) + } }) } catch (error) { logger.error(`WdioJasmineTestFramework: log failed: ${util.format(error)}`) @@ -315,13 +358,47 @@ export default class WdioJasmineTestFramework extends TestFramework { return instance } - async #emit(instance: TestFrameworkInstance, testFrameworkState: State, hookState: State, args: Record) { - TestFramework.setTrackedInstance(instance.getContext(), instance) + async #toTestHub(instance: TestFrameworkInstance, testFrameworkState: State, hookState: State, args: Record) { + const testHub = this.#testHub + if (!testHub) { + logger.debug(`WdioJasmineTestFramework: TestHub module not loaded; ${testFrameworkState}/${hookState} not reported`) + return + } this.updateInstanceState(instance, testFrameworkState, hookState) args.instance = instance + if (testFrameworkState === TestFrameworkState.TEST && hookState === HookState.PRE) { + testHub.onBeforeTest(args) + } + await testHub.sendTestFrameworkEvent(args) + } + + async #toModules(instance: TestFrameworkInstance, testFrameworkState: State, hookState: State, args: Record) { + args.instance = instance + args.skipTestHub = true await this.runHooks(instance, testFrameworkState, hookState, args) } + /** The reporter's instance for this spec; minted here only when the reporter is not feeding (TO opted out). */ + #serviceInstance(test: { fullName?: string, description?: string, file?: string }) { + const existing = test.fullName ? this.#specsByFullName.get(test.fullName) : undefined + if (existing) { + return existing + } + const instance = this.#createInstance() + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_UUID]: uuidv4(), + [TestFrameworkConstants.KEY_TEST_RESULT]: TestFrameworkConstants.DEFAULT_TEST_RESULT, + [TestFrameworkConstants.KEY_TEST_ID]: test.fullName, + [TestFrameworkConstants.KEY_TEST_NAME]: test.description, + ...this.#filePaths(test.file), + }) + this.#serviceOnly.add(instance) + if (test.fullName) { + this.#specsByFullName.set(test.fullName, instance) + } + return instance + } + #enqueue(label: string, step: () => Promise) { WdioJasmineTestFramework.#pendingEvents++ this.#queue = this.#queue.then(step).catch((error) => { diff --git a/packages/browserstack-service/src/cli/modules/testHubModule.ts b/packages/browserstack-service/src/cli/modules/testHubModule.ts index 5026bdaa..1830be1e 100644 --- a/packages/browserstack-service/src/cli/modules/testHubModule.ts +++ b/packages/browserstack-service/src/cli/modules/testHubModule.ts @@ -82,6 +82,10 @@ export default class TestHubModule extends BaseModule { } onBeforeTest(args: Record) { + // Jasmine reports to TestHub from its reporter feed; its module-only dispatches carry this marker + if (args.skipTestHub === true) { + return + } this.logger.debug('onBeforeTest: Called after test hook from cli configured module!!!') const autoInstance = AutomationFramework.getTrackedInstance() as AutomationFrameworkInstance const instances = [autoInstance] @@ -90,6 +94,9 @@ export default class TestHubModule extends BaseModule { } onAllTestEvents(args: Record) { + if (args.skipTestHub === true) { + return + } this.logger.debug('onAllTestEvents: Called after all test events from cli configured module!!!') const instance = args.instance as TestFrameworkInstance const testState = instance.getCurrentTestState() diff --git a/packages/browserstack-service/src/reporter.ts b/packages/browserstack-service/src/reporter.ts index 3e684124..b593e5c8 100644 --- a/packages/browserstack-service/src/reporter.ts +++ b/packages/browserstack-service/src/reporter.ts @@ -7,6 +7,7 @@ import { BrowserstackCLI } from './cli/index.js' import { reportSkippedTest, resolveSpecFile } from './cli/skipReporter.js' import WdioJasmineTestFramework from './cli/frameworks/wdioJasmineTestFramework.js' import type { JasmineSuiteContext } from './cli/frameworks/wdioJasmineTestFramework.js' +import type TestHubModule from './cli/modules/testHubModule.js' import * as url from 'node:url' import { v4 as uuidv4 } from 'uuid' @@ -154,8 +155,13 @@ class _TestReporter extends WDIOReporter { } cliJasmineFramework() { - const framework = BrowserstackCLI.getInstance().getTestFramework() - return framework instanceof WdioJasmineTestFramework ? framework : null + const cli = BrowserstackCLI.getInstance() + const framework = cli.getTestFramework() + if (!(framework instanceof WdioJasmineTestFramework)) { + return null + } + framework.setTestHubModule(cli.modules?.TestHubModule as TestHubModule | undefined) + return framework } jasmineSuiteContext(): JasmineSuiteContext { diff --git a/packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts b/packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts new file mode 100644 index 00000000..3dbba978 --- /dev/null +++ b/packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts @@ -0,0 +1,57 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' + +import * as bstackLogger from '../../../src/bstackLogger.js' +import TestHubModule from '../../../src/cli/modules/testHubModule.js' +import TestFrameworkInstance from '../../../src/cli/instances/testFrameworkInstance.js' +import TrackedInstance from '../../../src/cli/instances/trackedInstance.js' +import { TestFrameworkState } from '../../../src/cli/states/testFrameworkState.js' +import { HookState } from '../../../src/cli/states/hookState.js' +import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/testFrameworkConstants.js' + +vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) + +describe('TestHubModule skipTestHub marker', () => { + let module: TestHubModule + let sendTestFrameworkEvent: ReturnType + let sendTestSessionEvent: ReturnType + + const instanceIn = (state: State, hook: State, framework = 'WebdriverIO-cucumber') => { + const instance = new TestFrameworkInstance(TrackedInstance.createContext('t'), [framework], { [framework]: '9' }, state, hook) + instance.updateMultipleEntries({ + [TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME]: framework, + [TestFrameworkConstants.KEY_TEST_UUID]: 'uuid-1', + [TestFrameworkConstants.KEY_TEST_RESULT_AT]: 'now', + }) + return instance + } + + beforeEach(() => { + module = new TestHubModule({}) + sendTestFrameworkEvent = vi.spyOn(module, 'sendTestFrameworkEvent').mockResolvedValue(true) + sendTestSessionEvent = vi.spyOn(module, 'sendTestSessionEvent').mockResolvedValue(undefined) + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + it('handles test and hook events that carry no marker, as before', () => { + module.onBeforeTest({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE) }) + module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE) }) + module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.BEFORE_ALL, HookState.POST) }) + expect(sendTestSessionEvent).toHaveBeenCalledTimes(1) + expect(sendTestFrameworkEvent).toHaveBeenCalledTimes(2) + }) + + it('ignores events marked skipTestHub', () => { + module.onBeforeTest({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE), skipTestHub: true }) + module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.TEST, HookState.POST), skipTestHub: true }) + expect(sendTestSessionEvent).not.toHaveBeenCalled() + expect(sendTestFrameworkEvent).not.toHaveBeenCalled() + }) + + it('only honours the literal true', () => { + module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE), skipTestHub: 'true' }) + expect(sendTestFrameworkEvent).toHaveBeenCalledTimes(1) + }) +}) diff --git a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts index 8eec475f..2a459047 100644 --- a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts @@ -7,6 +7,7 @@ import TestFramework from '../../src/cli/frameworks/testFramework.js' import { TestFrameworkState } from '../../src/cli/states/testFrameworkState.js' import { HookState } from '../../src/cli/states/hookState.js' import type TestFrameworkInstance from '../../src/cli/instances/testFrameworkInstance.js' +import TestHubModule from '../../src/cli/modules/testHubModule.js' vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) @@ -50,16 +51,37 @@ const context = (scopes = ['Nested outer']) => ({ scopes, suiteFile: SUITE_FILE describe('WdioJasmineTestFramework', () => { let framework: WdioJasmineTestFramework + // what reached TestHub (reporter path) and what reached the module observers (service path) let dispatches: Dispatch[] + let moduleDispatches: Dispatch[] + let logSends: Array<{ state: string, entries: Record[], data: Record }> + let sessionEvents: number + let testHub: Record> - const drain = () => framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, {}) + const drain = () => framework.trackEvent(TestFrameworkState.LOG_REPORT, HookState.POST, {}) beforeEach(() => { process.env.BROWSERSTACK_OBSERVABILITY = 'true' framework = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin-session') dispatches = [] + moduleDispatches = [] + logSends = [] + sessionEvents = 0 + testHub = { + onBeforeTest: vi.fn(() => { sessionEvents++ }), + sendTestFrameworkEvent: vi.fn(async (args: Record) => { + const instance = args.instance as TestFrameworkInstance + dispatches.push({ state: instance.getCurrentTestState(), hook: instance.getCurrentHookState(), data: snapshot(instance), args }) + return true + }), + sendLogCreatedEvent: vi.fn(async (args: Record) => { + const instance = args.instance as TestFrameworkInstance + logSends.push({ state: instance.getCurrentTestState().toString(), entries: args.logEntries as Record[], data: snapshot(instance) }) + }), + } + framework.setTestHubModule(testHub as unknown as TestHubModule) vi.spyOn(framework, 'runHooks').mockImplementation(async (instance, state, hook, args) => { - dispatches.push({ state, hook, data: snapshot(instance), args: args as Record }) + moduleDispatches.push({ state, hook, data: snapshot(instance), args: args as Record }) }) }) @@ -231,24 +253,64 @@ describe('WdioJasmineTestFramework', () => { framework.onReporterLog({ level: 'INFO', message: 'hi', timestamp: 't', kind: 'TEST_LOG' }) await drain() expect(dispatches.map(d => d.state)).toEqual([TestFrameworkState.TEST]) + expect(logSends).toHaveLength(0) } finally { delete process.env.BROWSERSTACK_ACCESSIBILITY } }) - it('absorbs service-hook states instead of re-dispatching them, leaving the spec tracked', async () => { + it('sends reporter events to TestHub only, never through the module observers', async () => { + const stats = testStats() + framework.onReporterTestStart(stats as any, context()) + framework.onReporterTestEnd({ ...stats, state: 'passed', end: new Date() } as any, context()) + await drain() + expect(testHub.sendTestFrameworkEvent).toHaveBeenCalledTimes(2) + expect(sessionEvents).toBe(1) + expect(moduleDispatches).toHaveLength(0) + }) + + it('drives the modules from the service hooks on the reporter\'s instance, with TestHub skipped', async () => { + const spec = { description: 'outer passing test', fullName: 'Nested outer outer passing test' } framework.onReporterTestStart(testStats() as any, context()) - await framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test: {} }) - const tracked = TestFramework.getTrackedInstance() - expect(TestFramework.getState(tracked, 'test_uuid')).toBe(dispatches[0].data.test_uuid) + await framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test: spec }) + const uuid = dispatches[0].data.test_uuid + expect(TestFramework.getState(TestFramework.getTrackedInstance(), 'test_uuid')).toBe(uuid) + expect(process.env.TEST_ANALYTICS_ID).toBe(uuid) + + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { test: spec, suiteTitle: 'Nested outer' }) + await framework.trackEvent(TestFrameworkState.LOG_REPORT, HookState.POST, { test: spec, result: {} }) + await framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { test: spec, result: { passed: true }, suiteTitle: 'Jasmine__TopLevel__Suite' }) + + expect(moduleDispatches.map(d => [d.state, d.hook])).toEqual([[TestFrameworkState.TEST, HookState.PRE], [TestFrameworkState.TEST, HookState.POST]]) + for (const d of moduleDispatches) { + expect(d.args.skipTestHub).toBe(true) + expect(d.data.test_uuid).toBe(uuid) + } + expect(moduleDispatches[1].args.suiteTitle).toBe('Nested outer') + expect(moduleDispatches[1].args.result).toEqual({ passed: true }) + expect(testHub.sendTestFrameworkEvent).toHaveBeenCalledTimes(1) + }) - await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { test: {} }) - await framework.trackEvent(TestFrameworkState.LOG_REPORT, HookState.POST, { test: {}, result: {} }) - await framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { test: {}, result: {} }) - expect(dispatches).toHaveLength(1) + it('never shows a spec the reporter saw but the service did not (pending, excluded, beforeAll-failed) to the modules', async () => { + framework.onReporterTestStart(testStats({ uid: 'x' }) as any, context()) + framework.onReporterTestEnd(testStats({ uid: 'x', state: 'skipped', end: new Date() }) as any, context()) + await drain() + expect(dispatches).toHaveLength(2) + expect(moduleDispatches).toHaveLength(0) + }) + + it('still drives the modules when Test Observability is opted out and the reporter feeds nothing', async () => { + const spec = { description: 'a', fullName: 'Suite a', file: SUITE_FILE } + await framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test: spec }) + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { test: spec, suiteTitle: 'Suite' }) + await framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { test: spec, result: { passed: true } }) + expect(moduleDispatches).toHaveLength(2) + expect(moduleDispatches[0].data.test_uuid).toBe(moduleDispatches[1].data.test_uuid) + expect(process.env.TEST_ANALYTICS_ID).toBe(moduleDispatches[0].data.test_uuid) + expect(testHub.sendTestFrameworkEvent).not.toHaveBeenCalled() }) - it('attributes logs to an open all-hook, else to the last-started spec', async () => { + it('sends an open all-hook\'s logs in the hook state, else the last-started spec\'s in the test state', async () => { framework.onReporterTestStart(testStats() as any, context()) framework.onReporterLog({ level: 'INFO', message: 'in test', timestamp: 't1', kind: 'TEST_LOG' }) const after = hookStats('"after all" hook') @@ -258,22 +320,20 @@ describe('WdioJasmineTestFramework', () => { framework.onReporterLog({ level: 'INFO', message: 'after hook', timestamp: 't3', kind: 'TEST_LOG' }) await drain() - const logs = dispatches.filter(d => d.state === TestFrameworkState.LOG) - expect(logs).toHaveLength(3) + expect(logSends.map(l => l.state)).toEqual(['TestFrameworkState.TEST', 'TestFrameworkState.AFTER_ALL', 'TestFrameworkState.TEST']) const specUuid = dispatches[0].data.test_uuid - expect(logs[0].data.test_uuid).toBe(specUuid) - expect((logs[0].data.test_logs as Record[]).at(-1)).not.toHaveProperty('hook_id') - const hookLog = (logs[1].data.test_logs as Record[]).at(-1)! - expect(hookLog.hook_id).toBe((dispatches.find(d => d.state === TestFrameworkState.AFTER_ALL)!.data.test_hooks_started as Record[]>).AFTER_ALL[0].hook_id) - expect(logs[2].data.test_uuid).toBe(specUuid) + expect(logSends[0].data.test_uuid).toBe(specUuid) + expect(logSends[0].entries[0]).not.toHaveProperty('hook_id') + const hookId = (dispatches.find(d => d.state === TestFrameworkState.AFTER_ALL)!.data.test_hooks_started as Record[]>).AFTER_ALL[0].hook_id + expect(logSends[1].entries[0].hook_id).toBe(hookId) + expect(logSends[2].data.test_uuid).toBe(specUuid) }) it('keeps a screenshot entry\'s kind on the log path', async () => { framework.onReporterTestStart(testStats() as any, context()) await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: 't', level: 'INFO' } }) await drain() - const log = dispatches.find(d => d.state === TestFrameworkState.LOG)! - expect((log.data.test_logs as Record[])[0].kind).toBe('TEST_SCREENSHOT') + expect(logSends[0].entries[0].kind).toBe('TEST_SCREENSHOT') }) it('is not idle until queued events are dispatched', async () => { @@ -284,7 +344,7 @@ describe('WdioJasmineTestFramework', () => { }) it('logs and continues when an observer throws', async () => { - vi.mocked(framework.runHooks).mockRejectedValueOnce(new Error('observer blew up')) + testHub.sendTestFrameworkEvent.mockRejectedValueOnce(new Error('send blew up')) const stats = testStats() framework.onReporterTestStart(stats as any, context()) framework.onReporterTestEnd({ ...stats, state: 'passed', end: new Date() } as any, context()) @@ -292,4 +352,11 @@ describe('WdioJasmineTestFramework', () => { expect(dispatches.map(d => d.hook)).toEqual([HookState.POST]) expect(WdioJasmineTestFramework.isIdle()).toBe(true) }) + + // The class calls these TestHubModule methods directly; renaming or removing one must fail here. + it('relies on TestHubModule handlers that exist', () => { + for (const method of ['onBeforeTest', 'sendTestFrameworkEvent', 'sendLogCreatedEvent']) { + expect(typeof (TestHubModule.prototype as unknown as Record)[method]).toBe('function') + } + }) }) From 1a7d7c73286c1c12898dd5943a02a72b27c0489d Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 14:50:51 +0530 Subject: [PATCH 04/24] feat(browserstack-service): jasmine shared dispatch gates and HTTP logs on CLI (SDK-7772) - service hooks classify jasmine hooks by the WDIO hookName instead of the mocha title; the mocha skip cascade stays mocha-only - beforeTest annotates each jasmine spec with `Test: ` on the CLI flow - jasmine registers the command listener on the CLI flow; browserCommand sends HTTP command logs over gRPC (LogEntry kind HTTP) to the spec it names, and jasmine screenshots name their spec the same way - the jasmine class sends a session event for every beforeAll/afterAll start so hook runs carry their platform and Automate session, as on the legacy flow Co-Authored-By: Claude Opus 5.5 --- .../frameworks/wdioJasmineTestFramework.ts | 27 +- .../src/insights-handler.ts | 28 +- packages/browserstack-service/src/service.ts | 46 +++- .../cli/wdioJasmineTestFramework.test.ts | 45 +++- .../tests/service.jasmineCliGates.test.ts | 254 ++++++++++++++++++ 5 files changed, 378 insertions(+), 22 deletions(-) create mode 100644 packages/browserstack-service/tests/service.jasmineCliGates.test.ts diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts index fe717388..649e8366 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -58,10 +58,18 @@ export default class WdioJasmineTestFramework extends TestFramework { static KEY_HOOK_FAILURE_REASON = 'hook_failure_reason' static #pendingEvents = 0 + static #hookTypes = new Map([ + ['beforeAll', 'BEFORE_ALL'], + ['afterAll', 'AFTER_ALL'], + ['beforeEach', 'BEFORE_EACH'], + ['afterEach', 'AFTER_EACH'], + ]) #specInstances = new Map() // Open specs by fullName: how the service's hooks find the instance the reporter minted. #specsByFullName = new Map() + // Every spec of this worker by uuid: legacy attaches HTTP logs and screenshots to the last started spec, even after it ended. + #specsByUuid = new Map() #serviceOnly = new Set() #suiteTitles = new Map() #testHub: TestHubModule | null = null @@ -81,6 +89,11 @@ export default class WdioJasmineTestFramework extends TestFramework { return WdioJasmineTestFramework.#pendingEvents === 0 } + /** The WDIO `hookName` a jasmine hook runs under, as the hook-type key `getHookType` returns for mocha titles. */ + static hookTypeFromName(hookName: string | undefined) { + return WdioJasmineTestFramework.#hookTypes.get(hookName ?? '') ?? 'unknown' + } + setTestHubModule(testHub: TestHubModule | null | undefined) { this.#testHub = testHub ?? null } @@ -145,6 +158,7 @@ export default class WdioJasmineTestFramework extends TestFramework { }) this.#specInstances.set(testStats.uid, instance) this.#specsByFullName.set(fullTitle, instance) + this.#specsByUuid.set(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) as string, instance) // Registered synchronously: the service's beforeTest for this spec may read it before the queue runs. TestFramework.setTrackedInstance(instance.getContext(), instance) @@ -236,6 +250,8 @@ export default class WdioJasmineTestFramework extends TestFramework { ...this.#filePaths(context.suiteFile), } instance.updateData(TestFrameworkConstants.KEY_HOOK_ID, hookId) + // The session event is keyed by KEY_TEST_UUID: it links the hook run to its Automate session and platform + instance.updateData(TestFrameworkConstants.KEY_TEST_UUID, hookId) this.#hookInstances.set(hookStats.uid, instance) const emitHook = shouldProcessEventForTesthub('HookRunStarted') @@ -307,14 +323,18 @@ export default class WdioJasmineTestFramework extends TestFramework { } } - /** Console logs and screenshots: an open beforeAll/afterAll wins, else the last-started spec, even after it ended. */ + /** + * Console logs: an open beforeAll/afterAll wins, else the last-started spec, even after it ended. + * HTTP command logs and screenshots name their spec (`test_run_uuid`), as legacy did; unknown uuids are dropped. + */ onReporterLog(logEntry: Record | undefined) { try { if (!logEntry || !shouldProcessEventForTesthub('LogCreated')) { return } + const targetUuid = logEntry.test_run_uuid as string | undefined this.#enqueue('LOG/POST', async () => { - const instance = this.#openHook ?? this.#lastSpec + const instance = targetUuid ? this.#specsByUuid.get(targetUuid) : (this.#openHook ?? this.#lastSpec) if (!instance) { return } @@ -366,7 +386,8 @@ export default class WdioJasmineTestFramework extends TestFramework { } this.updateInstanceState(instance, testFrameworkState, hookState) args.instance = instance - if (testFrameworkState === TestFrameworkState.TEST && hookState === HookState.PRE) { + // Legacy sent a run's platform and session with its start event, hooks included (parity #19, #23) + if (hookState === HookState.PRE) { testHub.onBeforeTest(args) } await testHub.sendTestFrameworkEvent(args) diff --git a/packages/browserstack-service/src/insights-handler.ts b/packages/browserstack-service/src/insights-handler.ts index 3542882b..1dc36338 100644 --- a/packages/browserstack-service/src/insights-handler.ts +++ b/packages/browserstack-service/src/insights-handler.ts @@ -812,7 +812,8 @@ class _InsightsHandler { logEntry: { kind: TestFrameworkConstants.KIND_SCREENSHOT, message: result.value, - timestamp: new Date().toISOString() + timestamp: new Date().toISOString(), + ...(this._framework === 'jasmine' ? { test_run_uuid: testMeta.uuid } : {}) } }) : this.listener.onScreenshot([{ @@ -829,16 +830,29 @@ class _InsightsHandler { } // log http request + const httpResponse = { + path: requestData.endpoint, + method: requestData.method, + body, + response: result + } + // Only jasmine registers `command` on the CLI flow, and the legacy listener is inert there + if (BrowserstackCLI.getInstance().isRunning()) { + await BrowserstackCLI.getInstance().getTestFramework()?.trackEvent(TestFrameworkState.LOG, HookState.POST, { + logEntry: { + kind: 'HTTP', + message: JSON.stringify(httpResponse), + timestamp: new Date().toISOString(), + test_run_uuid: testMeta.uuid + } + }) + return + } this.listener.logCreated([{ test_run_uuid: testMeta.uuid, timestamp: new Date().toISOString(), kind: 'HTTP', - http_response: { - path: requestData.endpoint, - method: requestData.method, - body, - response: result - } + http_response: httpResponse }] ) } diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index e262f5ca..104a1a5d 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -45,6 +45,7 @@ import { HookState } from './cli/states/hookState.js' import { AutomationFrameworkConstants } from './cli/frameworks/constants/automationFrameworkConstants.js' import TestFramework from './cli/frameworks/testFramework.js' import WdioCucumberTestFramework from './cli/frameworks/wdioCucumberTestFramework.js' +import WdioJasmineTestFramework from './cli/frameworks/wdioJasmineTestFramework.js' import { TestFrameworkState } from './cli/states/testFrameworkState.js' import { TestFrameworkConstants } from './cli/frameworks/constants/testFrameworkConstants.js' import AccessibilityModule from './cli/modules/accessibilityModule.js' @@ -330,12 +331,22 @@ export default class BrowserstackService implements Services.ServiceInstance { * `browserCommand` is the only producer of TEST_SCREENSHOT logs — the binary * has no screenshot producer of its own — so the result event has to be * registered on this path too, or a screenshot taken mid-test never reaches - * Observability (SDK-4177). The `command` (beforeCommand) event is - * deliberately NOT registered: it only fills the map that browserCommand's - * HTTP-log half reads, and that half emits on the JS listener pipeline the - * binary owns here. Leaving it unregistered keeps the screenshot upload — - * which rides its own JWT-authenticated endpoint — as the single effect. + * Observability (SDK-4177). The `command` (beforeCommand) event only fills + * the map browserCommand's HTTP-log half reads, so it is registered for + * jasmine alone, whose HTTP command logs go over gRPC; mocha and cucumber + * never reported them on this path. */ + if (this._config.framework === 'jasmine') { + this._browser.on('command', (command) => { + if (shouldProcessEventForTesthub('')) { + this._insightsHandler?.browserCommand( + 'client:beforeCommand', + Object.assign(command, { sessionId }), + this._currentTest + ) + } + }) + } this._browser.on('result', (result) => { if (shouldProcessEventForTesthub('')) { this._insightsHandler?.browserCommand( @@ -450,7 +461,7 @@ export default class BrowserstackService implements Services.ServiceInstance { } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'beforeHook' }) - async beforeHook (test: Frameworks.Test|CucumberHook, context: unknown) { + async beforeHook (test: Frameworks.Test|CucumberHook, context: unknown, hookName?: string) { if (this._config.framework !== 'cucumber') { this._currentTest = test as Frameworks.Test // not update currentTest when this is called for cucumber step } @@ -481,7 +492,7 @@ export default class BrowserstackService implements Services.ServiceInstance { await framework.trackEvent(hookFrameworkState, HookState.PRE, { test }) } } else if (framework) { - const hookFrameworkState = TestFrameworkState[getHookType((test as Frameworks.Test).title) as keyof typeof TestFrameworkState] + const hookFrameworkState = TestFrameworkState[this._cliHookType(test as Frameworks.Test, hookName) as keyof typeof TestFrameworkState] if (hookFrameworkState) { await framework.trackEvent(hookFrameworkState, HookState.PRE, { test }) } @@ -496,7 +507,7 @@ export default class BrowserstackService implements Services.ServiceInstance { } @PerformanceTester.Measure(PERFORMANCE_SDK_EVENTS.EVENTS.SDK_HOOK, { hookType: 'afterHook' }) - async afterHook(test: Frameworks.Test | CucumberHook, context: unknown, result: Frameworks.TestResult) { + async afterHook(test: Frameworks.Test | CucumberHook, context: unknown, result: Frameworks.TestResult, hookName?: string) { // The Mocha hook window is closed — clear the tracker (see beforeHook). if (this._config.framework === 'mocha') { setCurrentMochaHookWindow(null) @@ -535,16 +546,16 @@ export default class BrowserstackService implements Services.ServiceInstance { return } if (framework) { - const hookFrameworkState = TestFrameworkState[getHookType((test as Frameworks.Test).title) as keyof typeof TestFrameworkState] + const hookType = this._cliHookType(test as Frameworks.Test, hookName) + const hookFrameworkState = TestFrameworkState[hookType as keyof typeof TestFrameworkState] if (hookFrameworkState) { await framework.trackEvent(hookFrameworkState, HookState.POST, { test, result }) } // a failed (or skipping) before/each hook silently drops the suite's remaining // tests in mocha — report them as skipped so they surface on the dashboard and // attribute their Automate session (port of the legacy insights-handler cascade) - const hookType = getHookType((test as Frameworks.Test).title) const suite = (test as Frameworks.Test).ctx?.test?.parent - if (result && !result.passed && ['BEFORE_ALL', 'BEFORE_EACH', 'AFTER_EACH'].includes(hookType) && suite) { + if (this._config.framework === 'mocha' && result && !result.passed && ['BEFORE_ALL', 'BEFORE_EACH', 'AFTER_EACH'].includes(hookType) && suite) { await reportSuiteSkipped(framework, suite) } } @@ -555,6 +566,16 @@ export default class BrowserstackService implements Services.ServiceInstance { await this._accessibilityHandler?.afterHook() } + /** + * Jasmine's hook argument is a copy of the last started spec, never the hook itself, so only + * the `hookName` WDIO passes says which hook is running. + */ + private _cliHookType(test: Frameworks.Test, hookName?: string) { + return this._config.framework === 'jasmine' + ? WdioJasmineTestFramework.hookTypeFromName(hookName) + : getHookType(test.title) + } + /** * Cucumber abandons the whole feature when a `BeforeAll` throws, so every scenario in it * (Rule-nested included) is reported SKIPPED rather than vanishing. Ports @@ -623,6 +644,9 @@ export default class BrowserstackService implements Services.ServiceInstance { // skip reporter must never re-report it from onTestSkip markTestStarted(getUniqueIdentifier(test, this._config.framework)) this._insightsHandler?.setTestData(test, uuid) + if (this._config.framework === 'jasmine') { + await this._setAnnotation(`Test: ${test.fullName ?? test.title}`) + } await BrowserstackCLI.getInstance().getTestFramework()!.trackEvent(TestFrameworkState.TEST, HookState.PRE, { test, suiteTitle }) return } diff --git a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts index 2a459047..cfce5f23 100644 --- a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts @@ -214,7 +214,11 @@ describe('WdioJasmineTestFramework', () => { expect(finished).toMatchObject({ hook_result: 'passed', event_ended_at: '2026-09-25T15:34:12.257Z', hook_duration: 4087 }) expect(finished.hook_id).toBe(started.hook_id) expect('hook_failure' in finished).toBe(false) - expect('test_uuid' in dispatches[0].data).toBe(false) + // the session event is keyed by the hook's own uuid, so the hook run gets its platform and session (#23); + // the binary builds the HookRun from hook_id and never links it to a test (#25) + expect(dispatches[0].data.test_uuid).toBe(started.hook_id) + expect(testHub.onBeforeTest).toHaveBeenCalledTimes(2) + expect(testHub.onBeforeTest.mock.calls.map(([args]) => (args as Record).instance)).toEqual([dispatches[0].args.instance, dispatches[2].args.instance]) const afterStarted = (dispatches[2].data.test_hooks_started as Record[]>).AFTER_ALL[0] expect(afterStarted.hook_identifier).toBe('"after all" hook for Nested middle') @@ -336,6 +340,45 @@ describe('WdioJasmineTestFramework', () => { expect(logSends[0].entries[0].kind).toBe('TEST_SCREENSHOT') }) + it('sends an HTTP command log to the spec it names, even inside an all-hook or after the spec ended', async () => { + const first = testStats() + const uuid = framework.onReporterTestStart(first as any, context()) + framework.onReporterTestEnd({ ...first, state: 'passed', end: new Date() } as any, context()) + const after = hookStats('"after all" hook') + framework.onReporterHookStart(after as any, context()) + const message = JSON.stringify({ path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } }) + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'HTTP', message, timestamp: 't', test_run_uuid: uuid } }) + await drain() + + expect(logSends).toHaveLength(1) + expect(logSends[0].state).toBe('TestFrameworkState.TEST') + expect(logSends[0].data.test_uuid).toBe(uuid) + expect(logSends[0].entries[0]).toMatchObject({ kind: 'HTTP', timestamp: 't' }) + expect(logSends[0].entries[0]).not.toHaveProperty('hook_id') + expect(Buffer.from(logSends[0].entries[0].message as Uint8Array).toString()).toBe(message) + }) + + it('drops a named-spec log whose uuid this worker never minted', async () => { + framework.onReporterTestStart(testStats() as any, context()) + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'HTTP', message: '{}', timestamp: 't', test_run_uuid: 'not-ours' } }) + await drain() + expect(logSends).toHaveLength(0) + }) + + it('maps the WDIO hookName to the hook-type key, and nothing else', () => { + expect(['beforeAll', 'afterAll', 'beforeEach', 'afterEach'].map(n => WdioJasmineTestFramework.hookTypeFromName(n))) + .toEqual(['BEFORE_ALL', 'AFTER_ALL', 'BEFORE_EACH', 'AFTER_EACH']) + expect(WdioJasmineTestFramework.hookTypeFromName(undefined)).toBe('unknown') + expect(WdioJasmineTestFramework.hookTypeFromName('constructor')).toBe('unknown') + }) + + it('ignores the service\'s hook events: hooks reach TestHub from the reporter only', async () => { + await framework.trackEvent(TestFrameworkState.BEFORE_EACH, HookState.PRE, { test: { fullName: 'Nested outer outer passing test' } }) + await framework.trackEvent(TestFrameworkState.BEFORE_ALL, HookState.POST, { test: {}, result: { passed: false } }) + expect(dispatches).toHaveLength(0) + expect(moduleDispatches).toHaveLength(0) + }) + it('is not idle until queued events are dispatched', async () => { framework.onReporterTestStart(testStats() as any, context()) expect(WdioJasmineTestFramework.isIdle()).toBe(false) diff --git a/packages/browserstack-service/tests/service.jasmineCliGates.test.ts b/packages/browserstack-service/tests/service.jasmineCliGates.test.ts new file mode 100644 index 00000000..3a891c8f --- /dev/null +++ b/packages/browserstack-service/tests/service.jasmineCliGates.test.ts @@ -0,0 +1,254 @@ +import path from 'node:path' + +import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' + +import BrowserstackService from '../src/service.js' +import InsightsHandler from '../src/insights-handler.js' +import * as utils from '../src/util.js' +import * as skipReporter from '../src/cli/skipReporter.js' +import { BrowserstackCLI } from '../src/cli/index.js' +import * as bstackLogger from '../src/bstackLogger.js' +import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' +import { HookState } from '../src/cli/states/hookState.js' +import { TESTOPS_SCREENSHOT_ENV } from '../src/constants.js' +import TestFramework from '../src/cli/frameworks/testFramework.js' + +vi.mock('fetch') +vi.mock('@wdio/logger', () => import(path.join(process.cwd(), '__mocks__', '@wdio/logger'))) +vi.mock('uuid', () => ({ v4: () => '123456789' })) +vi.mock('../src/data-store.js', () => ({ saveWorkerData: vi.fn() })) +vi.mock('../src/instrumentation/performance/performance-tester.js', () => ({ + default: { + start: vi.fn(), + end: vi.fn(), + startMonitoring: vi.fn(), + measureWrapper: vi.fn().mockImplementation((_name: string, fn: Function) => fn), + Measure: vi.fn().mockImplementation(() => (_target: any, _propertyKey: string, descriptor: PropertyDescriptor) => descriptor), + browser: undefined, + scenarioThatRan: [], + } +})) +vi.mock('../src/cli/index.js', () => ({ + BrowserstackCLI: { + getInstance: () => ({ + isRunning: () => false, + getTestFramework: () => null, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + }) + } +})) + +vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) + +// What WDIO hands a jasmine hook: a copy of the last started spec (no `title`), then the hookName. +const lastSpec = { description: 'outer passing test', fullName: 'Nested outer outer passing test', file: '/p/nested.spec.js' } as any +const mochaHook = (title: string) => ({ title, ctx: { test: { parent: { title: 'suite', tests: [], suites: [] } } } }) as any + +let getInstanceSpy: ReturnType | undefined + +const cliWith = (framework: unknown) => { + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => framework, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as any) +} + +const makeService = (framework: string) => new BrowserstackService({} as any, [] as any, { user: 'foo', key: 'bar', framework } as any) + +afterEach(() => { + getInstanceSpy?.mockRestore() + getInstanceSpy = undefined +}) + +describe('service hooks on the CLI flow (S4)', () => { + it('classifies jasmine hooks by hookName, without touching the missing title', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const getHookTypeSpy = vi.spyOn(utils, 'getHookType') + const service = makeService('jasmine') + + for (const hookName of ['beforeAll', 'beforeEach']) { + await service.beforeHook(lastSpec, {}, hookName) + } + for (const hookName of ['afterEach', 'afterAll']) { + await service.afterHook(lastSpec, {}, { passed: true } as any, hookName) + } + + expect(getHookTypeSpy).not.toHaveBeenCalled() + expect(trackEvent.mock.calls.map(([state, hook]) => [state, hook])).toEqual([ + [TestFrameworkState.BEFORE_ALL, HookState.PRE], + [TestFrameworkState.BEFORE_EACH, HookState.PRE], + [TestFrameworkState.AFTER_EACH, HookState.POST], + [TestFrameworkState.AFTER_ALL, HookState.POST], + ]) + getHookTypeSpy.mockRestore() + }) + + it('sends nothing for a jasmine hook without a hookName, and does not throw', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const service = makeService('jasmine') + + await expect(service.beforeHook({} as any, {})).resolves.toBeUndefined() + await expect(service.afterHook({} as any, {}, { passed: true } as any)).resolves.toBeUndefined() + expect(trackEvent).not.toHaveBeenCalled() + }) + + it('never runs the mocha skip cascade for a failed jasmine hook, but still records the hook failure', async () => { + cliWith({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + const cascade = vi.spyOn(skipReporter, 'reportSuiteSkipped').mockResolvedValue(undefined as any) + const service = makeService('jasmine') + + await service.afterHook({ ...lastSpec, ctx: { test: { parent: {} } } }, {}, { passed: false, error: new Error('beforeAll failed') } as any, 'beforeAll') + + expect(cascade).not.toHaveBeenCalled() + expect(service['_hookFailReasons']).toEqual(['beforeAll failed']) + cascade.mockRestore() + }) + + it('keeps mocha on its title classification and skip cascade', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const cascade = vi.spyOn(skipReporter, 'reportSuiteSkipped').mockResolvedValue(undefined as any) + const service = makeService('mocha') + + await service.beforeHook(mochaHook('"before each" hook for "t"'), {}, 'beforeEach') + await service.afterHook(mochaHook('"before all" hook for "t"'), {}, { passed: false, error: new Error('x') } as any, 'afterAll') + + expect(trackEvent.mock.calls.map(([state, hook]) => [state, hook])).toEqual([ + [TestFrameworkState.BEFORE_EACH, HookState.PRE], + [TestFrameworkState.BEFORE_ALL, HookState.POST], + ]) + expect(cascade).toHaveBeenCalledTimes(1) + cascade.mockRestore() + }) +}) + +describe('service.beforeTest on the CLI flow (S5)', () => { + const runBeforeTest = async (framework: string, test: Record) => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const service = makeService(framework) + const annotate = vi.spyOn(service as any, '_setAnnotation').mockResolvedValue(undefined) + const getState = vi.spyOn(TestFramework, 'getState').mockReturnValue('spec-uuid') + await service.beforeTest(test as any) + getState.mockRestore() + return { annotate, trackEvent } + } + + it('annotates each jasmine spec with its full name, before the modules see TEST/PRE', async () => { + const { annotate, trackEvent } = await runBeforeTest('jasmine', lastSpec) + expect(annotate).toHaveBeenCalledWith('Test: Nested outer outer passing test') + const testPre = trackEvent.mock.calls.findIndex(([state]) => state === TestFrameworkState.TEST) + expect(annotate.mock.invocationCallOrder[0]).toBeLessThan(trackEvent.mock.invocationCallOrder[testPre]) + }) + + it('does not annotate mocha tests on the CLI flow', async () => { + const { annotate } = await runBeforeTest('mocha', { title: 't', parent: 'suite' }) + expect(annotate).not.toHaveBeenCalled() + }) +}) + +describe('service.before on the CLI flow (S6)', () => { + const registeredEvents = async (framework: string) => { + process.env.BROWSERSTACK_OBSERVABILITY = 'true' + cliWith(null) + const browser = { on: vi.fn(), sessionId: 's1', capabilities: {}, config: {}, execute: vi.fn(), executeScript: vi.fn() } as any + const service = new BrowserstackService({} as any, [{}] as any, { user: 'foo', key: 'bar', framework, capabilities: {} } as any) + await service.before(service['_config'] as any, [], browser) + delete process.env.BROWSERSTACK_OBSERVABILITY + return vi.mocked(browser.on).mock.calls.map(([event]: [string]) => event) + } + + it('registers command and result for jasmine', async () => { + const events = await registeredEvents('jasmine') + expect(events).toContain('command') + expect(events).toContain('result') + }) + + it.each(['mocha', 'cucumber'])('keeps %s on result only', async (framework) => { + const events = await registeredEvents(framework) + expect(events).toContain('result') + expect(events).not.toContain('command') + }) +}) + +describe('insights-handler.browserCommand on the CLI flow (S6)', () => { + const browser = { on: vi.fn(), sessionId: 's', capabilities: {}, config: {}, execute: vi.fn() } as any + const command = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/title', body: {} } + const result = { ...command, result: { value: 'StackDemo' } } + + const handlerFor = (framework: string) => { + const handler = new InsightsHandler(browser, framework) + handler['getIdentifier'] = vi.fn().mockReturnValue('Nested outer outer passing test') + handler['_tests'] = { 'Nested outer outer passing test': { uuid: 'spec-uuid' } } + return handler + } + + beforeEach(() => { + delete process.env[TESTOPS_SCREENSHOT_ENV] + }) + + it('sends the HTTP log over gRPC to the named spec, never to the legacy listener', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const handler = handlerFor('jasmine') + const logCreated = vi.spyOn(handler['listener'], 'logCreated').mockImplementation(() => {}) + + await handler.browserCommand('client:beforeCommand', { ...command } as any, lastSpec) + await handler.browserCommand('client:afterCommand', { ...result } as any, lastSpec) + + expect(logCreated).not.toHaveBeenCalled() + expect(trackEvent).toHaveBeenCalledTimes(1) + const [state, hook, { logEntry }] = trackEvent.mock.calls[0] + expect([state, hook]).toEqual([TestFrameworkState.LOG, HookState.POST]) + expect(logEntry.kind).toBe('HTTP') + expect(logEntry.test_run_uuid).toBe('spec-uuid') + expect(JSON.parse(logEntry.message)).toEqual({ path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } }) + }) + + it('keeps the legacy HTTP log shape on the listener when the CLI is not running', async () => { + const handler = handlerFor('jasmine') + const logCreated = vi.spyOn(handler['listener'], 'logCreated').mockImplementation(() => {}) + + await handler.browserCommand('client:beforeCommand', { ...command } as any, lastSpec) + await handler.browserCommand('client:afterCommand', { ...result } as any, lastSpec) + + expect(logCreated).toHaveBeenCalledWith([{ + test_run_uuid: 'spec-uuid', + timestamp: expect.any(String), + kind: 'HTTP', + http_response: { path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } } + }]) + }) + + it('names the spec on a jasmine screenshot, and leaves the mocha screenshot entry unchanged', async () => { + process.env[TESTOPS_SCREENSHOT_ENV] = 'true' + const screenshot = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/screenshot', result: { value: 'b64' } } + + const jasmineTrack = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent: jasmineTrack }) + await handlerFor('jasmine').browserCommand('client:afterCommand', { ...screenshot } as any, lastSpec) + getInstanceSpy!.mockRestore() + + const mochaTrack = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent: mochaTrack }) + await handlerFor('mocha').browserCommand('client:afterCommand', { ...screenshot } as any, { title: 't' } as any) + + expect(jasmineTrack.mock.calls[0][2].logEntry).toEqual({ kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: expect.any(String), test_run_uuid: 'spec-uuid' }) + expect(Object.keys(mochaTrack.mock.calls[0][2].logEntry)).toEqual(['kind', 'message', 'timestamp']) + }) + + it('drops commands with no spec yet, as legacy did before the first spec', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const handler = handlerFor('jasmine') + handler['getIdentifier'] = vi.fn().mockReturnValue(undefined) + + await handler.browserCommand('client:beforeCommand', { ...command } as any, {} as any) + await handler.browserCommand('client:afterCommand', { ...result } as any, {} as any) + + expect(trackEvent).not.toHaveBeenCalled() + }) +}) From 6b687de5fda0130235a1087e39faf9d20c2a96a9 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 17:05:33 +0530 Subject: [PATCH 05/24] feat(browserstack-service): jasmine Automate session verdict on CLI (SDK-7772) - WdioJasmineTestFramework keeps the legacy service.after() inputs: specs ran, the runner's failed-spec count, and the test and hook failure reasons of the live session in the order they happened - AutomateModule marks a jasmine worker's live session with that verdict, in the legacy body shape {status, name, reason}; a session no spec registered is still marked and a reloaded session keeps the mark onReload sent - ignoreHooksStatus is read from the worker's service options, because the binary config echo carries testObservabilityOptions empty - other frameworks keep the existing per-test aggregation Co-Authored-By: Claude Opus 5.5 --- .../frameworks/wdioJasmineTestFramework.ts | 104 ++++++++++++++++++ .../src/cli/modules/automateModule.ts | 52 ++++++++- .../tests/cli/modules/automateModule.test.ts | 102 +++++++++++++++++ .../cli/wdioJasmineTestFramework.test.ts | 83 ++++++++++++++ 4 files changed, 338 insertions(+), 3 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts index 649e8366..7b946760 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -5,6 +5,8 @@ import type { Frameworks } from '@wdio/types' import type { HookStats, TestStats } from '@wdio/reporter' import TestFramework from './testFramework.js' +import AutomationFramework from './automationFramework.js' +import { AutomationFrameworkConstants } from './constants/automationFrameworkConstants.js' import { TestFrameworkState } from '../states/testFrameworkState.js' import { HookState } from '../states/hookState.js' import TestFrameworkInstance from '../instances/testFrameworkInstance.js' @@ -23,6 +25,17 @@ export interface JasmineSuiteContext { suiteFile?: string } +/** The Automate verdict legacy `service.after()` marks a session with. */ +export interface SessionVerdict { + status: 'passed' | 'failed' + reason?: string +} + +interface SessionFailure { + reason: string + fromHook: boolean +} + interface FailureFields { failure?: Array<{ backtrace: string[] }> failureReason?: string @@ -58,6 +71,12 @@ export default class WdioJasmineTestFramework extends TestFramework { static KEY_HOOK_FAILURE_REASON = 'hook_failure_reason' static #pendingEvents = 0 + static #serviceHookStates = new Set([ + TestFrameworkState.BEFORE_ALL, + TestFrameworkState.BEFORE_EACH, + TestFrameworkState.AFTER_EACH, + TestFrameworkState.AFTER_ALL, + ]) static #hookTypes = new Map([ ['beforeAll', 'BEFORE_ALL'], ['afterAll', 'AFTER_ALL'], @@ -78,9 +97,17 @@ export default class WdioJasmineTestFramework extends TestFramework { #openHook: TestFrameworkInstance | null = null #lastSpec: TestFrameworkInstance | null = null #queue: Promise = Promise.resolve() + // Legacy `service.ts` session verdict inputs: `_specsRan` and the runner result are per worker, + // the reason lists per session (`onReload` resets them). + #specsRan = false + #runnerFailedCount = 0 + // In the order they happened, as legacy pushed them + #sessionFailures = new Map() + static #current: WdioJasmineTestFramework | null = null constructor(testFrameworks: string[], testFrameworkVersions: Record, binSessionId: string) { super(testFrameworks, testFrameworkVersions, binSessionId) + WdioJasmineTestFramework.#current = this logger.debug('WdioJasmineTestFramework: constructed') } @@ -109,6 +136,12 @@ export default class WdioJasmineTestFramework extends TestFramework { } try { await this.#queue + if (WdioJasmineTestFramework.#serviceHookStates.has(testFrameworkState)) { + if (hookState === HookState.POST) { + this.#recordHookResult(args.result as Frameworks.TestResult | undefined) + } + return + } if (testFrameworkState !== TestFrameworkState.INIT_TEST && testFrameworkState !== TestFrameworkState.TEST) { return } @@ -124,6 +157,7 @@ export default class WdioJasmineTestFramework extends TestFramework { if (hookState === HookState.PRE) { this.#suiteTitles.set(instance, args.suiteTitle) } else { + this.#recordTestResult(args.result as Frameworks.TestResult | undefined) // afterTest hands over the raw `Jasmine__TopLevel__Suite`; the modules need the describe chain beforeTest derived moduleArgs.suiteTitle = this.#suiteTitles.get(instance) ?? args.suiteTitle this.#suiteTitles.delete(instance) @@ -187,6 +221,9 @@ export default class WdioJasmineTestFramework extends TestFramework { const state = testStats.state const error = testStats.error + if (state === 'failed') { + this.#runnerFailedCount++ + } let result: string = state let failure: FailureFields = {} if (state === 'failed') { @@ -358,6 +395,73 @@ export default class WdioJasmineTestFramework extends TestFramework { } } + /** + * The verdict legacy `service.after()` would mark the live session with: passed only when the runner + * reported no failed spec, a spec ran, and no test or hook failed, with `ignoreHooksStatus` handled as + * coded there. `null` for any other session: legacy marked a reloaded session once, in `onReload`. + * `undefined` when this worker does not run jasmine on the CLI flow. + */ + static sessionVerdict(sessionId: string, ignoreHooksStatus: boolean): SessionVerdict | null | undefined { + const current = WdioJasmineTestFramework.#current + return current ? current.#verdict(sessionId, ignoreHooksStatus) : undefined + } + + #verdict(sessionId: string, ignoreHooksStatus: boolean): SessionVerdict | null { + if (!sessionId || sessionId !== WdioJasmineTestFramework.#liveSessionId()) { + return null + } + const failures = this.#sessionFailures.get(sessionId) ?? [] + const pureTestFailReasons = failures.filter(f => !f.fromHook).map(f => f.reason) + const hookFailReasons = failures.filter(f => f.fromHook).map(f => f.reason) + const failReasons = ignoreHooksStatus ? pureTestFailReasons : failures.map(f => f.reason) + const joined = (reasons: string[]) => reasons.length > 0 ? reasons.join('\n') : undefined + + if (this.#runnerFailedCount === 0 && this.#specsRan) { + const reasons = ignoreHooksStatus ? pureTestFailReasons : failReasons + return reasons.length > 0 ? { status: 'failed', reason: joined(reasons) } : { status: 'passed' } + } + if (ignoreHooksStatus && this.#specsRan) { + const hasOnlyHookFailures = failReasons.length === 0 && hookFailReasons.length > 0 + if (hasOnlyHookFailures && pureTestFailReasons.length === 0) { + return { status: 'passed' } + } + return { status: 'failed', reason: joined(pureTestFailReasons) } + } + return { + status: 'failed', + reason: ignoreHooksStatus && pureTestFailReasons.length > 0 ? joined(pureTestFailReasons) : joined(failReasons), + } + } + + /** Legacy `service.afterTest()`: the spec ran; a failure counts toward the session. */ + #recordTestResult(result: Frameworks.TestResult | undefined) { + this.#specsRan = true + if (!result || result.passed || result.skipped) { + return + } + this.#recordFailure((result.error && result.error.message) || 'Unknown Error', false) + } + + /** Legacy `service.afterHook()`, any of the four hook types. */ + #recordHookResult(result: Frameworks.TestResult | undefined) { + if (!result || result.passed) { + return + } + this.#recordFailure((result.error && result.error.message) || 'Hook failed', true) + } + + #recordFailure(reason: string, fromHook: boolean) { + const sessionId = WdioJasmineTestFramework.#liveSessionId() + const failures = this.#sessionFailures.get(sessionId) ?? [] + failures.push({ reason, fromHook }) + this.#sessionFailures.set(sessionId, failures) + } + + static #liveSessionId(): string { + const autoInstance = AutomationFramework.getTrackedInstance() + return autoInstance ? String(AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) || '') : '' + } + #createInstance() { const trackedContext = TrackedInstance.createContext(CLIUtils.getCurrentInstanceName()) const instance = new TestFrameworkInstance( diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index d01d3080..dbde960e 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -1,4 +1,6 @@ import BaseModule from './baseModule.js' +import WdioJasmineTestFramework from '../frameworks/wdioJasmineTestFramework.js' +import { BrowserstackCLI } from '../index.js' import { BStackLogger } from '../cliLogger.js' import TestFramework from '../frameworks/testFramework.js' import { TestFrameworkState } from '../states/testFrameworkState.js' @@ -348,6 +350,16 @@ export default class AutomateModule extends BaseModule { const accessKey = this.config.accessKey as string const testContextOptions = this.config.testContextOptions as TestContextOptions + // The binary's config echo carries `testObservabilityOptions` empty, so read the worker's own service options + const serviceOptions = BrowserstackCLI.getInstance().options as { testObservabilityOptions?: { ignoreHooksStatus?: boolean } } + const ignoreHooksStatus = serviceOptions?.testObservabilityOptions?.ignoreHooksStatus === true + const liveSessionId = this.liveSessionId() + if (WdioJasmineTestFramework.sessionVerdict(liveSessionId, ignoreHooksStatus) !== undefined) { + await this.markJasmineSessions(liveSessionId, ignoreHooksStatus) + this.sessionMap.clear() + return + } + for (const [sessionId, sessionData] of this.sessionMap.entries()) { try { const failedTests = Array.from(sessionData.testResults.values()).filter(test => test.status === 'failed') @@ -389,6 +401,39 @@ export default class AutomateModule extends BaseModule { this.sessionMap.clear() } + private liveSessionId(): string { + const autoInstance = AutomationFramework.getTrackedInstance() + return autoInstance ? String(AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) || '') : '' + } + + /** + * Jasmine: legacy `service.after()` marked only the live session, with the worker's status, the last + * name, and the test and hook failure reasons. A session nothing registered (a beforeAll failed before + * any spec ran) is still marked, and a reloaded one is left to the mark `onReload` already sent. + */ + private async markJasmineSessions(liveSessionId: string, ignoreHooksStatus: boolean) { + const testContextOptions = this.config.testContextOptions as TestContextOptions + const auth = { user: this.config.userName as string, key: this.config.accessKey as string } + const sessionIds = new Set(this.sessionMap.keys()) + if (liveSessionId && isBrowserstackSession(AutomationFramework.getDriver(AutomationFramework.getTrackedInstance()) as WebdriverIO.Browser)) { + sessionIds.add(liveSessionId) + } + + for (const sessionId of sessionIds) { + try { + await this.flushSessionName(sessionId) + const verdict = WdioJasmineTestFramework.sessionVerdict(sessionId, ignoreHooksStatus) + if (!verdict || testContextOptions.skipSessionStatus) { + continue + } + const name = testContextOptions.skipSessionName ? undefined : this.sessionMap.get(sessionId)?.lastTestName || undefined + await this.markSessionStatus(sessionId, verdict.status, verdict.reason, auth, name) + } catch (error) { + this.logger.error(`Failed to process session ${sessionId}: ${error}`) + } + } + } + // An App Automate session is identified by the service-level app / skipAppOverride flag, // OR an app supplied only via the appium:app / appium:options.app capability — which the // service-level config (this.config) does not carry. Mirrors accessibilityModule.isAppAutomateSession. @@ -474,10 +519,10 @@ export default class AutomateModule extends BaseModule { )(sessionId, sessionName, config) } - async markSessionStatus(sessionId: string, sessionStatus: 'passed' | 'failed', sessionErrorMessage: string | undefined, config: { user: string; key: string; }): Promise { + async markSessionStatus(sessionId: string, sessionStatus: 'passed' | 'failed', sessionErrorMessage: string | undefined, config: { user: string; key: string; }, sessionName?: string): Promise { return await PerformanceTester.measureWrapper( PERFORMANCE_SDK_EVENTS.AUTOMATE_EVENTS.SESSION_STATUS, - async (sessionId: string, sessionStatus: 'passed' | 'failed', sessionErrorMessage: string | undefined, config: { user: string; key: string; }) => { + async (sessionId: string, sessionStatus: 'passed' | 'failed', sessionErrorMessage: string | undefined, config: { user: string; key: string; }, sessionName?: string) => { try { const auth = Buffer.from(`${config.user}:${config.key}`).toString('base64') const { url: sessionStatusApiUrl, method, product } = this.resolveSessionApi(sessionId) @@ -485,6 +530,7 @@ export default class AutomateModule extends BaseModule { const body = { status: sessionStatus, + ...(sessionName ? { name: sessionName } : {}), ...(sessionErrorMessage ? { reason: sessionErrorMessage } : {}) } @@ -504,7 +550,7 @@ export default class AutomateModule extends BaseModule { this.logger.error(`Failed to update session status on BrowserStack: ${err}`) } } - )(sessionId, sessionStatus, sessionErrorMessage, config) + )(sessionId, sessionStatus, sessionErrorMessage, config, sessionName) } } diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index b0e27c76..098b946a 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -9,6 +9,8 @@ import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/te import { isBrowserstackSession } from '../../../src/util.js' import PerformanceTester from '../../../src/instrumentation/performance/performance-tester.js' import { _fetch as fetch } from '../../../src/fetchWrapper.js' +import WdioJasmineTestFramework from '../../../src/cli/frameworks/wdioJasmineTestFramework.js' +import { BrowserstackCLI } from '../../../src/cli/index.js' import type { Options } from '@wdio/types' // Mock dependencies @@ -21,6 +23,19 @@ vi.mock('../../../src/cli/frameworks/testFramework.js', () => ({ } })) +vi.mock('../../../src/cli/index.js', () => ({ + BrowserstackCLI: { + getInstance: vi.fn(() => ({ options: {} })) + } +})) + +// undefined = this worker is not jasmine on the CLI flow, so the existing aggregation applies +vi.mock('../../../src/cli/frameworks/wdioJasmineTestFramework.js', () => ({ + default: { + sessionVerdict: vi.fn(() => undefined) + } +})) + vi.mock('../../../src/cli/frameworks/automationFramework.js', () => ({ default: { getTrackedInstance: vi.fn(), @@ -1005,3 +1020,90 @@ describe('AutomateModule preferScenarioName', () => { expect(namesPUT()).toContain('Can log in') }) }) + +describe('AutomateModule — jasmine session verdict', () => { + let automateModule: AutomateModule + const putBodies = () => vi.mocked(fetch).mock.calls.map(([url, opts]) => [String(url).split('/sessions/')[1], JSON.parse((opts as { body: string }).body)]) + + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as any) + vi.mocked(AutomationFramework.getDriver).mockReturnValue({ sessionId: 'live' } as any) + vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => key === 'framework_session_id' ? 'live' : {}) + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(fetch).mockResolvedValue({ json: async () => ({}) } as any) + automateModule = new AutomateModule({} as Options.Testrunner) + automateModule.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false }, + userName: 'u', + accessKey: 'k' + } as any + }) + + afterEach(() => { + vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReset().mockReturnValue(undefined) + }) + + it('marks the live session with the framework verdict, the last name and the joined reasons', async () => { + vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockImplementation((id) => id === 'live' + ? { status: 'failed', reason: 'beforeAll hook failure\nafterEach hook failure' } + : null) + ;(automateModule as any).sessionMap.set('live', { lastTestName: 'Hookfail afterEach suite', appliedName: 'Hookfail afterEach suite', testResults: new Map(), scenariosRan: 0 }) + + await automateModule.onAfterExecute() + + expect(putBodies()).toEqual([ + ['live.json', { status: 'failed', name: 'Hookfail afterEach suite', reason: 'beforeAll hook failure\nafterEach hook failure' }] + ]) + }) + + it('marks a live session no spec registered, without a name', async () => { + vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReturnValue({ status: 'failed', reason: 'beforeAll hook failure' }) + + await automateModule.onAfterExecute() + + expect(putBodies()).toEqual([['live.json', { status: 'failed', reason: 'beforeAll hook failure' }]]) + }) + + it('leaves a reloaded session to its onReload mark, but still names it', async () => { + vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockImplementation((id) => id === 'live' ? { status: 'passed' } : null) + ;(automateModule as any).sessionMap.set('old', { lastTestName: 'Suite A', testResults: new Map(), scenariosRan: 0 }) + + await automateModule.onAfterExecute() + + expect(putBodies()).toEqual([ + ['old.json', { name: 'Suite A' }], + ['live.json', { status: 'passed' }], + ]) + }) + + it('passes ignoreHooksStatus from the worker\'s service options to the verdict', async () => { + vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReturnValue({ status: 'passed' }) + vi.mocked(BrowserstackCLI.getInstance).mockReturnValueOnce({ options: { testObservabilityOptions: { ignoreHooksStatus: true } } } as any) + + await automateModule.onAfterExecute() + + expect(WdioJasmineTestFramework.sessionVerdict).toHaveBeenCalledWith('live', true) + }) + + it('honours skipSessionStatus and skipSessionName', async () => { + vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReturnValue({ status: 'failed', reason: 'x' }) + ;(automateModule as any).sessionMap.set('live', { lastTestName: 'Suite', testResults: new Map(), scenariosRan: 0 }) + ;(automateModule.config as any).testContextOptions = { skipSessionName: true, skipSessionStatus: true } + + await automateModule.onAfterExecute() + + expect(fetch).not.toHaveBeenCalled() + }) + + it('keeps the existing per-test aggregation when the worker is not jasmine', async () => { + (automateModule as any).sessionMap.set('live', { + lastTestName: 'Suite', appliedName: 'Suite', scenariosRan: 0, + testResults: new Map([['t', { testName: 'Suite', status: 'failed', reason: 'boom' }]]) + }) + + await automateModule.onAfterExecute() + + expect(putBodies()).toEqual([['live.json', { status: 'failed', reason: 'boom' }]]) + }) +}) diff --git a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts index cfce5f23..dd1e3b9b 100644 --- a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts @@ -8,6 +8,7 @@ import { TestFrameworkState } from '../../src/cli/states/testFrameworkState.js' import { HookState } from '../../src/cli/states/hookState.js' import type TestFrameworkInstance from '../../src/cli/instances/testFrameworkInstance.js' import TestHubModule from '../../src/cli/modules/testHubModule.js' +import AutomationFramework from '../../src/cli/frameworks/automationFramework.js' vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) @@ -396,6 +397,88 @@ describe('WdioJasmineTestFramework', () => { expect(WdioJasmineTestFramework.isIdle()).toBe(true) }) + describe('session verdict (legacy service.after())', () => { + let liveSession = 'live' + const spec = (title: string, overrides: Record = {}) => testStats({ uid: title, title, fullTitle: `Suite ${title}`, ...overrides }) + const reporterEnd = (stats: ReturnType, state: string) => + framework.onReporterTestEnd({ ...stats, state, end: new Date() } as any, context()) + const serviceTest = (title: string, result: Record) => + framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { test: { fullName: `Suite ${title}`, description: title }, result }) + const serviceHook = (state: State, result: Record) => framework.trackEvent(state, HookState.POST, { test: {}, result }) + const fail = (message: string) => ({ passed: false, error: new Error(message) }) + + beforeEach(() => { + liveSession = 'live' + vi.spyOn(AutomationFramework, 'getTrackedInstance').mockReturnValue({} as any) + vi.spyOn(AutomationFramework, 'getState').mockImplementation(() => liveSession) + }) + + it('reproduces the CP0 hookfail verdict: failed, every hook failure in the order it happened', async () => { + await serviceHook(TestFrameworkState.BEFORE_ALL, fail('beforeAll hook failure')) + for (const title of ['beforeAll child one', 'beforeAll child two', 'beforeEach child one', 'beforeEach child two']) { + if (title.startsWith('beforeEach')) { + await serviceHook(TestFrameworkState.BEFORE_EACH, fail('beforeEach hook failure')) + } + const stats = spec(title) + framework.onReporterTestStart(stats as any, context()) + reporterEnd(stats, 'failed') + } + for (const title of ['afterEach child one', 'afterEach child two']) { + const stats = spec(title) + framework.onReporterTestStart(stats as any, context()) + await serviceTest(title, { passed: true }) + await serviceHook(TestFrameworkState.AFTER_EACH, fail('afterEach hook failure')) + reporterEnd(stats, 'failed') + } + await drain() + + expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ + status: 'failed', + reason: 'beforeAll hook failure\nbeforeEach hook failure\nbeforeEach hook failure\nafterEach hook failure\nafterEach hook failure' + }) + }) + + it('fails a session where no spec ran, even with no failure recorded', async () => { + expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'failed', reason: undefined }) + }) + + it('passes when specs ran and nothing failed, with no reason', async () => { + const stats = spec('ok') + framework.onReporterTestStart(stats as any, context()) + await serviceTest('ok', { passed: true }) + reporterEnd(stats, 'passed') + await drain() + expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'passed' }) + }) + + it('does not count a pending() spec as a failure', async () => { + await serviceTest('pending', { passed: false, skipped: true }) + expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'passed' }) + }) + + it('under ignoreHooksStatus passes a run whose only failures are hooks, and keeps test failures', async () => { + await serviceTest('ok', { passed: true }) + await serviceHook(TestFrameworkState.BEFORE_EACH, fail('beforeEach hook failure')) + const child = spec('child') + framework.onReporterTestStart(child as any, context()) + reporterEnd(child, 'failed') + await drain() + expect(WdioJasmineTestFramework.sessionVerdict('live', true)).toEqual({ status: 'passed' }) + expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'failed', reason: 'beforeEach hook failure' }) + + await serviceTest('bad', fail('boom')) + expect(WdioJasmineTestFramework.sessionVerdict('live', true)).toEqual({ status: 'failed', reason: 'boom' }) + }) + + it('keeps failures per session and gives no verdict for a session that is no longer live', async () => { + await serviceTest('first', fail('before reload')) + liveSession = 'reloaded' + await serviceTest('second', { passed: true }) + expect(WdioJasmineTestFramework.sessionVerdict('reloaded', false)).toEqual({ status: 'passed' }) + expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toBeNull() + }) + }) + // The class calls these TestHubModule methods directly; renaming or removing one must fail here. it('relies on TestHubModule handlers that exist', () => { for (const method of ['onBeforeTest', 'sendTestFrameworkEvent', 'sendLogCreatedEvent']) { From 4c3c6ea56cf222f9d2f299d1b2da3e4a5d649c1a Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 22:21:04 +0530 Subject: [PATCH 06/24] fix(browserstack-service): strip SDK-only session-name options from capabilities (port of 3cdbfec, SDK-7772) Port of 3cdbfec from SDK-7606/wdio-cucumber-platformisation-v8. sessionNamePrependTopLevelSuiteTitle and sessionNameOmitTestTitle reach the outgoing capabilities on the CLI flow and the hub rejects the session ("additional properties [...] outside of the schema"). Both are read from testContextOptions, so stripping them from the capabilities does not disable either option. Co-Authored-By: Claude Opus 5.5 --- packages/browserstack-service/src/constants.ts | 2 +- packages/browserstack-service/tests/skipAppOverride.test.ts | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/browserstack-service/src/constants.ts b/packages/browserstack-service/src/constants.ts index ae000252..1132ba04 100644 --- a/packages/browserstack-service/src/constants.ts +++ b/packages/browserstack-service/src/constants.ts @@ -42,7 +42,7 @@ export const DEFAULT_WAIT_TIMEOUT_FOR_PENDING_UPLOADS = 5000 // 5s export const DEFAULT_WAIT_INTERVAL_FOR_PENDING_UPLOADS = 100 // 100ms export const BSTACK_SERVICE_VERSION = bstackServiceVersion -export const NOT_ALLOWED_KEYS_IN_CAPS = ['includeTagsInTestingScope', 'excludeTagsInTestingScope', 'testManagementOptions', 'skipAppOverride', 'preferScenarioName'] +export const NOT_ALLOWED_KEYS_IN_CAPS = ['includeTagsInTestingScope', 'excludeTagsInTestingScope', 'testManagementOptions', 'skipAppOverride', 'preferScenarioName', 'sessionNamePrependTopLevelSuiteTitle', 'sessionNameOmitTestTitle'] export const BROWSERSTACK_TEST_PLAN_ID = 'BROWSERSTACK_TEST_PLAN_ID' export const LOGS_FILE = 'logs/bstack-wdio-service.log' diff --git a/packages/browserstack-service/tests/skipAppOverride.test.ts b/packages/browserstack-service/tests/skipAppOverride.test.ts index 5b175848..f65e36e8 100644 --- a/packages/browserstack-service/tests/skipAppOverride.test.ts +++ b/packages/browserstack-service/tests/skipAppOverride.test.ts @@ -89,7 +89,9 @@ describe('NOT_ALLOWED_KEYS_IN_CAPS cloud-leak strip', () => { expect(NOT_ALLOWED_KEYS_IN_CAPS).toContain('skipAppOverride') }) - it('includes preferScenarioName so it is never forwarded to bstack:options', () => { + it('includes the SDK-only options the binary round-trips into caps on the CLI flow', () => { expect(NOT_ALLOWED_KEYS_IN_CAPS).toContain('preferScenarioName') + expect(NOT_ALLOWED_KEYS_IN_CAPS).toContain('sessionNamePrependTopLevelSuiteTitle') + expect(NOT_ALLOWED_KEYS_IN_CAPS).toContain('sessionNameOmitTestTitle') }) }) From 4343a4c55c7993186ee8c214751ee6ba4f3ccb44 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 22:21:46 +0530 Subject: [PATCH 07/24] fix(browserstack-service): apply jasmine sessionNameFormat on CLI (SDK-7772) The binary config echo cannot carry the sessionNameFormat function, so it arrives empty and the session keeps its default name. For jasmine the AutomateModule now takes the function from the worker's service options, called as legacy did (no test title). Other frameworks are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../src/cli/modules/automateModule.ts | 26 ++++++++++-- .../tests/cli/modules/automateModule.test.ts | 42 +++++++++++++++++++ 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index dbde960e..c6554f50 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -90,9 +90,10 @@ export default class AutomateModule extends BaseModule { } let name = suiteTitle - if (testContextOptions.sessionNameFormat) { + const sessionNameFormat = this.sessionNameFormatFor(instace, testContextOptions) + if (sessionNameFormat) { const caps = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_CAPABILITIES) - name = testContextOptions.sessionNameFormat( + name = sessionNameFormat( this.browserStackConfig, caps, suiteTitle, @@ -183,9 +184,10 @@ export default class AutomateModule extends BaseModule { } let name = suiteTitle - if (testContextOptions.sessionNameFormat) { + const sessionNameFormat = this.sessionNameFormatFor(instace, testContextOptions) + if (sessionNameFormat) { const caps = AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_CAPABILITIES) - name = testContextOptions.sessionNameFormat( + name = sessionNameFormat( this.browserStackConfig, caps, suiteTitle, @@ -326,6 +328,22 @@ export default class AutomateModule extends BaseModule { } } + /** + * The binary's config echo cannot carry a function, so `sessionNameFormat` arrives empty. Jasmine + * takes it from the worker's own service options, as legacy did. + */ + private sessionNameFormatFor(instance: TestFrameworkInstance, testContextOptions: TestContextOptions) { + if (testContextOptions.sessionNameFormat) { + return testContextOptions.sessionNameFormat + } + const frameworkName = String(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '') + if (!frameworkName.toLowerCase().includes('jasmine')) { + return undefined + } + const format = (BrowserstackCLI.getInstance().options as { sessionNameFormat?: unknown })?.sessionNameFormat + return typeof format === 'function' ? format as TestContextOptions['sessionNameFormat'] : undefined + } + private isCucumberInstance(instance: TestFrameworkInstance): boolean { const frameworkName = String(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '') return frameworkName.toLowerCase().includes('cucumber') diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index 098b946a..4d1fbf45 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -1107,3 +1107,45 @@ describe('AutomateModule — jasmine session verdict', () => { expect(putBodies()).toEqual([['live.json', { status: 'failed', reason: 'boom' }]]) }) }) + +describe('AutomateModule — jasmine sessionNameFormat', () => { + let automateModule: AutomateModule + const format = vi.fn((_config: unknown, _caps: unknown, suiteTitle: string, testTitle?: string) => `fmt[${suiteTitle}][${String(testTitle)}]`) + const names = () => vi.mocked(fetch).mock.calls.map(([, opts]) => JSON.parse((opts as { body: string }).body).name) + + const runBeforeTest = (frameworkName: string, test: Record) => { + vi.mocked(TestFramework.getState).mockImplementation((_i, key) => key === TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME ? frameworkName : undefined) + return automateModule.onBeforeTest({ instance: {}, test, suiteTitle: 'Nested outer' }) + } + + beforeEach(() => { + vi.clearAllMocks() + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as any) + vi.mocked(AutomationFramework.getDriver).mockReturnValue({ sessionId: 's1' } as any) + vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => key === 'framework_session_id' ? 's1' : {}) + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(fetch).mockResolvedValue({ json: async () => ({}) } as any) + vi.mocked(BrowserstackCLI.getInstance).mockReturnValue({ options: { sessionNameFormat: format } } as any) + automateModule = new AutomateModule({} as Options.Testrunner) + automateModule.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false, sessionNameFormat: '' }, + userName: 'u', + accessKey: 'k' + } as any + }) + + afterEach(() => { + vi.mocked(BrowserstackCLI.getInstance).mockReset().mockReturnValue({ options: {} } as any) + }) + + it('applies the service-option format for jasmine, with no test title (legacy call shape)', async () => { + await runBeforeTest('WebdriverIO-jasmine', { description: 'outer passing test', fullName: 'Nested outer outer passing test' }) + expect(names()).toEqual(['fmt[Nested outer][undefined]']) + }) + + it('leaves mocha on the binary config (format absent there)', async () => { + await runBeforeTest('WebdriverIO-mocha', { title: 't', parent: 'Suite' }) + expect(format).not.toHaveBeenCalled() + expect(names()).toEqual(['Suite - t']) + }) +}) From f97f8d9f74dd3bb210270f4b342b03843e2a85c4 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sat, 26 Sep 2026 23:14:02 +0530 Subject: [PATCH 08/24] feat(browserstack-service): jasmine accessibility gate on CLI (SDK-7772) - no scan window before the first jasmine spec, as the classic handler (PRE_TEST_SCAN_FRAMEWORKS excludes jasmine); mocha/cucumber keep it - the include/exclude filter and the test name read the jasmine spec description when the test has no title Co-Authored-By: Claude Opus 5.5 --- .../src/cli/modules/accessibilityModule.ts | 16 +++++-- .../cli/modules/accessibilityModule.test.ts | 44 +++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts index edcb1b3b..d8cf2a26 100644 --- a/packages/browserstack-service/src/cli/modules/accessibilityModule.ts +++ b/packages/browserstack-service/src/cli/modules/accessibilityModule.ts @@ -278,7 +278,7 @@ export default class AccessibilityModule extends BaseModule { // on suite and test titles, and in this window neither exists yet. onBeforeTest // re-computes the per-test gate, tags included, so this only affects the window. const preTestSessionId = this.currentSessionId() - if (this.autoScanning && preTestSessionId !== undefined && preTestSessionId !== null) { + if (this.autoScanning && this.supportsPreTestWindow() && preTestSessionId !== undefined && preTestSessionId !== null) { this.accessibilityMap.set(preTestSessionId, true) this.logger.debug('Accessibility scan gate opened ahead of the first test') } @@ -338,9 +338,11 @@ export default class AccessibilityModule extends BaseModule { try { this.logger.debug('Accessibility before test hook. Starting accessibility scan for this test case.') const suiteTitle = (typeof args.suiteTitle === 'string' ? args.suiteTitle : '') || '' - const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string } : {}) || {} + const test = (args.test && typeof args.test === 'object' ? args.test as { title?: string, description?: string } : {}) || {} + // jasmine specs carry their name in `description` (`title` is unset), as on the classic handler + const testTitle = test.title ?? test.description - this.currentTestName = test.title || null + this.currentTestName = testTitle || null this.testContextSeen = true const autoInstance: AutomationFrameworkInstance = AutomationFramework.getTrackedInstance() const testInstance: TestFrameworkInstance = TestFramework.getTrackedInstance() @@ -353,7 +355,7 @@ export default class AccessibilityModule extends BaseModule { // only ever populated on the cucumber path, so mocha and jasmine keep the exact 3-arg // behaviour — both extra args arrive undefined/false and the tag branch is not taken. const world = args.world as { [key: string]: unknown } | undefined - const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, test.title || '', accessibilityOptions as Record | undefined, world, Boolean(world)) && this.accessibility + const shouldScanTest = this.autoScanning && shouldScanTestForAccessibility(suiteTitle, testTitle || '', accessibilityOptions as Record | undefined, world, Boolean(world)) && this.accessibility this.accessibilityMap.set(sessionId, shouldScanTest) @@ -537,6 +539,12 @@ export default class AccessibilityModule extends BaseModule { return false } + // Legacy jasmine opened no scan window before its first spec (classic PRE_TEST_SCAN_FRAMEWORKS) + private supportsPreTestWindow(): boolean { + const frameworks = BrowserstackCLI.getInstance().getTestFramework()?.getTestFrameworks() ?? [] + return !frameworks.some((name) => String(name).toLowerCase().includes('jasmine')) + } + // See the classic handler: one definition, used at every scan site. private get hasNoParent(): boolean { return !this.currentHookRunUuid && !this.testContextSeen diff --git a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts index d4114756..5349c5e9 100644 --- a/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/accessibilityModule.test.ts @@ -59,6 +59,7 @@ import AutomationFramework from '../../../src/cli/frameworks/automationFramework import { AutomationFrameworkState } from '../../../src/cli/states/automationFrameworkState.js' import { HookState } from '../../../src/cli/states/hookState.js' import { TestFrameworkState } from '../../../src/cli/states/testFrameworkState.js' +import { BrowserstackCLI } from '../../../src/cli/index.js' describe('AccessibilityModule', () => { let accessibilityModule: AccessibilityModule @@ -194,6 +195,30 @@ describe('AccessibilityModule', () => { expect(accessibilityModule.accessibilityMap.get('session-w')).toBe(true) }) + it('opens no window for jasmine, whose classic flow scanned nothing before the first spec', async () => { + withA11yCaps() + const cliSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + getTestFramework: () => ({ getTestFrameworks: () => ['WebdriverIO-jasmine'] }) + } as any) + + await accessibilityModule.onBeforeExecute() + + expect(accessibilityModule.accessibilityMap.has('session-w')).toBe(false) + cliSpy.mockRestore() + }) + + it('still opens the window for mocha', async () => { + withA11yCaps() + const cliSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + getTestFramework: () => ({ getTestFrameworks: () => ['WebdriverIO-mocha'] }) + } as any) + + await accessibilityModule.onBeforeExecute() + + expect(accessibilityModule.accessibilityMap.get('session-w')).toBe(true) + cliSpy.mockRestore() + }) + it('respects autoScanning — the one validation the window still owns', async () => { withA11yCaps() accessibilityModule.autoScanning = false @@ -276,6 +301,25 @@ describe('AccessibilityModule', () => { }) }) + describe('onBeforeTest — jasmine spec name', () => { + it('filters on the spec description when the test has no title, and names the test with it', async () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + vi.mocked(AutomationFramework.getState).mockImplementation(() => 'session-w') + await accessibilityModule.onBeforeTest({ + suiteTitle: 'Nested outer', + test: { description: 'outer passing test', fullName: 'Nested outer outer passing test' } + }) + expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Nested outer', 'outer passing test', {}, undefined, false) + }) + + it('keeps the title when there is one (mocha)', async () => { + vi.mocked(shouldScanTestForAccessibility).mockReturnValue(true) + vi.mocked(AutomationFramework.getState).mockImplementation(() => 'session-w') + await accessibilityModule.onBeforeTest({ suiteTitle: 'Suite', test: { title: 't', description: 'ignored' } }) + expect(shouldScanTestForAccessibility).toHaveBeenCalledWith('Suite', 't', {}, undefined, false) + }) + }) + describe('onBeforeExecute', () => { it('should patch browser methods when automation instance exists', async () => { vi.mocked(AutomationFramework.getState).mockImplementation((instance, key) => { From ddc719db6158846ed35790ccd86aaf705694b9d1 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Sun, 27 Sep 2026 00:24:34 +0530 Subject: [PATCH 09/24] fix(browserstack-service): name the current jasmine test for Percy on CLI (SDK-7772) PercySDK attaches testCase from TestReporter.currentTest.name, which legacy sets while building the TestRunStarted event. The jasmine CLI branch of onTestStart returns before that, so CLI snapshots carried no testCase. It now sets the name next to the CLI uuid. Mocha is unchanged. Co-Authored-By: Claude Opus 5.5 --- packages/browserstack-service/src/reporter.ts | 1 + .../browserstack-service/tests/reporter.jasmineCli.test.ts | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/packages/browserstack-service/src/reporter.ts b/packages/browserstack-service/src/reporter.ts index b593e5c8..c4d839e3 100644 --- a/packages/browserstack-service/src/reporter.ts +++ b/packages/browserstack-service/src/reporter.ts @@ -211,6 +211,7 @@ class _TestReporter extends WDIOReporter { const cliUuid = this.cliJasmineFramework()?.onReporterTestStart(testStats, this.jasmineSuiteContext()) if (cliUuid) { _TestReporter.currentTest.uuid = cliUuid + _TestReporter.currentTest.name = testStats.title _TestReporter._tests[testStats.fullTitle] = { uuid: cliUuid } } return diff --git a/packages/browserstack-service/tests/reporter.jasmineCli.test.ts b/packages/browserstack-service/tests/reporter.jasmineCli.test.ts index 537c8b55..76ea1a68 100644 --- a/packages/browserstack-service/tests/reporter.jasmineCli.test.ts +++ b/packages/browserstack-service/tests/reporter.jasmineCli.test.ts @@ -111,6 +111,11 @@ describe('reporter jasmine arm', () => { expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: 'cli-uuid' }) }) + it('names the current test for Percy\'s testCase, as legacy getRunData did', async () => { + await reporter.onTestStart(testStats() as any) + expect((TestReporter as any).currentTest).toMatchObject({ uuid: 'cli-uuid', name: 'outer passing test' }) + }) + it('keeps legacy\'s end stamp and forced hook pass on the WDIO stats objects', async () => { const stats = testStats() as Record await reporter.onTestEnd(stats as any) From d54cfc1e55b2834c18d8f48efd15f1363e8368bc Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Mon, 28 Sep 2026 12:19:16 +0530 Subject: [PATCH 10/24] fix(browserstack-service): forward the CLI-arg test plan id to the binary (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../browserstack-service/src/cli/cliUtils.ts | 8 +-- .../tests/cli/cliUtils.test.ts | 50 +++++++++++++++++++ 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/packages/browserstack-service/src/cli/cliUtils.ts b/packages/browserstack-service/src/cli/cliUtils.ts index 6fbe10bf..6ad4a907 100644 --- a/packages/browserstack-service/src/cli/cliUtils.ts +++ b/packages/browserstack-service/src/cli/cliUtils.ts @@ -23,6 +23,7 @@ import { isFalse, isTurboScale, shouldAddServiceVersion, + getTestPlanId, } from '../util.js' import PerformanceTester from '../instrumentation/performance/performance-tester.js' import { EVENTS as PerformanceEvents } from '../instrumentation/performance/constants.js' @@ -32,7 +33,6 @@ import type { Options, Capabilities } from '@wdio/types' import type { BrowserstackConfig, BrowserstackOptions, - TestManagementOptions, TestObservabilityOptions, } from '../types.js' import { TestFrameworkConstants } from './frameworks/constants/testFrameworkConstants.js' @@ -116,11 +116,7 @@ export class CLIUtils { ) const observabilityOptions: TestObservabilityOptions = options.testObservabilityOptions || {} - const testManagementOptions: TestManagementOptions = - options.testManagementOptions || {} - const testPlanId = typeof testManagementOptions.testPlanId === 'string' - ? testManagementOptions.testPlanId.trim() - : '' + const testPlanId = (getTestPlanId(options as BrowserstackConfig & Options.Testrunner) || '').trim() const binconfig: Record = { userName: observabilityOptions.user || config.user, accessKey: observabilityOptions.key || config.key, diff --git a/packages/browserstack-service/tests/cli/cliUtils.test.ts b/packages/browserstack-service/tests/cli/cliUtils.test.ts index aa82167c..5898bae7 100644 --- a/packages/browserstack-service/tests/cli/cliUtils.test.ts +++ b/packages/browserstack-service/tests/cli/cliUtils.test.ts @@ -226,6 +226,56 @@ describe('CLIUtils', () => { testPlanId: 'tm-plan-456' }) }) + + describe('test plan id from env and CLI args', () => { + const CLI_ARG = '--browserstack.testManagementOptions.testPlanId' + const capabilities = [{ browserName: 'chrome' }] + let originalArgv: string[] + + beforeEach(() => { + originalArgv = process.argv + delete process.env.BROWSERSTACK_TEST_PLAN_ID + }) + + afterEach(() => { + process.argv = originalArgv + delete process.env.BROWSERSTACK_TEST_PLAN_ID + }) + + const planOf = (options = createBrowserstackOptions()) => + JSON.parse(CLIUtils.getBinConfig(mockConfig, capabilities, options)).testManagementOptions + + it('reads BROWSERSTACK_TEST_PLAN_ID', () => { + process.env.BROWSERSTACK_TEST_PLAN_ID = 'env-plan-1' + expect(planOf()).toEqual({ testPlanId: 'env-plan-1' }) + }) + + it('reads the CLI arg followed by its value', () => { + process.argv = ['node', 'wdio', 'run', 'wdio.conf.js', CLI_ARG, 'TP-123'] + expect(planOf()).toEqual({ testPlanId: 'TP-123' }) + }) + + it('reads the CLI arg in --key=value form', () => { + process.argv = ['node', 'wdio', 'run', 'wdio.conf.js', `${CLI_ARG}=12345`] + expect(planOf()).toEqual({ testPlanId: '12345' }) + }) + + it('prefers the CLI arg over the service option', () => { + process.argv = ['node', 'wdio', 'run', 'wdio.conf.js', CLI_ARG, 'cli-plan'] + expect(planOf(createBrowserstackOptions({ testManagementOptions: { testPlanId: 'option-plan' } }))) + .toEqual({ testPlanId: 'cli-plan' }) + }) + + it('omits testManagementOptions when no source sets an id', () => { + process.argv = ['node', 'wdio', 'run', 'wdio.conf.js'] + expect(planOf()).toBeUndefined() + }) + + it('omits testManagementOptions when the env id is blank', () => { + process.env.BROWSERSTACK_TEST_PLAN_ID = ' ' + expect(planOf()).toBeUndefined() + }) + }) }) describe('getSdkVersion', () => { From bcea0e9701cf3fcf68e63a813e99efaee4b699bc Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Mon, 28 Sep 2026 13:29:01 +0530 Subject: [PATCH 11/24] fix(browserstack-service): stop forwarding the raw test plan id arg to the binary (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../src/cli/grpcClient.ts | 23 +++++++++++++- .../tests/cli/grpcClient.test.ts | 30 ++++++++++++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/packages/browserstack-service/src/cli/grpcClient.ts b/packages/browserstack-service/src/cli/grpcClient.ts index 2aabc1cd..909dd160 100644 --- a/packages/browserstack-service/src/cli/grpcClient.ts +++ b/packages/browserstack-service/src/cli/grpcClient.ts @@ -55,6 +55,27 @@ const SUMMARY_ANSI = { warn: { base: '\x1b[33m', emphasis: '\x1b[1;33m' } } +const TEST_PLAN_ID_ARG = '--browserstack.testManagementOptions.testPlanId' + +// The binary parses CLI args as `--key=value` only, so the space form reaches it as `true` and +// overrides the id getBinConfig already resolved; the id travels in the bin config instead. +export function stripTestPlanIdArg(args: string[]): string[] { + const result: string[] = [] + for (let i = 0; i < args.length; i++) { + if (args[i] === TEST_PLAN_ID_ARG) { + if (i + 1 < args.length && !args[i + 1].startsWith('-')) { + i++ + } + continue + } + if (args[i].startsWith(`${TEST_PLAN_ID_ARG}=`)) { + continue + } + result.push(args[i]) + } + return result +} + /** * GrpcClient - Singleton class for managing gRPC client connections * @@ -180,7 +201,7 @@ export class GrpcClient { sdkVersion: packageVersion, pathProject: process.cwd(), pathConfig: path.resolve(process.cwd(), 'browserstack.yml'), - cliArgs: process.argv.slice(2), + cliArgs: stripTestPlanIdArg(process.argv.slice(2)), frameworks: [automationFrameworkDetail.name, testFrameworkDetail.name], frameworkVersions, language: CLIUtils.getSdkLanguage(), diff --git a/packages/browserstack-service/tests/cli/grpcClient.test.ts b/packages/browserstack-service/tests/cli/grpcClient.test.ts index 94312c1c..0bf8b808 100644 --- a/packages/browserstack-service/tests/cli/grpcClient.test.ts +++ b/packages/browserstack-service/tests/cli/grpcClient.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' -import { GrpcClient } from '../../src/cli/grpcClient.js' +import { GrpcClient, stripTestPlanIdArg } from '../../src/cli/grpcClient.js' import { BStackLogger } from '../../src/cli/cliLogger.js' vi.mock('../../src/grpc/index.js', () => ({ @@ -241,3 +241,31 @@ describe('GrpcClient.logCreatedEvent', () => { }) }) }) + +describe('stripTestPlanIdArg', () => { + const ARG = '--browserstack.testManagementOptions.testPlanId' + + it('drops the arg and its value in the space form', () => { + expect(stripTestPlanIdArg(['run', 'wdio.conf.js', ARG, 'TP-123', '--spec', 'a.js'])) + .toEqual(['run', 'wdio.conf.js', '--spec', 'a.js']) + }) + + it('drops the arg in the = form', () => { + expect(stripTestPlanIdArg(['run', `${ARG}=TP-123`, 'wdio.conf.js'])) + .toEqual(['run', 'wdio.conf.js']) + }) + + it('leaves args untouched and in order when the arg is absent', () => { + const args = ['run', 'wdio.conf.js', '--browserstack.buildName', 'b1', '--logLevel=info'] + expect(stripTestPlanIdArg(args)).toEqual(args) + }) + + it('keeps a following flag when the arg has no value', () => { + expect(stripTestPlanIdArg(['run', ARG, '--spec', 'a.js'])) + .toEqual(['run', '--spec', 'a.js']) + }) + + it('handles the arg as the last token', () => { + expect(stripTestPlanIdArg(['run', 'wdio.conf.js', ARG])).toEqual(['run', 'wdio.conf.js']) + }) +}) From d8f56c135d734edd20eb05784eb074d1b2dc28bb Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Tue, 29 Sep 2026 20:21:52 +0530 Subject: [PATCH 12/24] test(cli): fold skipTestHub marker cases into the TestHubModule suite (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../modules/testHubModule.skipMarker.test.ts | 57 ------------------- .../tests/cli/modules/testHubModule.test.ts | 35 ++++++++++++ 2 files changed, 35 insertions(+), 57 deletions(-) delete mode 100644 packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts diff --git a/packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts b/packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts deleted file mode 100644 index 3dbba978..00000000 --- a/packages/browserstack-service/tests/cli/modules/testHubModule.skipMarker.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' - -import * as bstackLogger from '../../../src/bstackLogger.js' -import TestHubModule from '../../../src/cli/modules/testHubModule.js' -import TestFrameworkInstance from '../../../src/cli/instances/testFrameworkInstance.js' -import TrackedInstance from '../../../src/cli/instances/trackedInstance.js' -import { TestFrameworkState } from '../../../src/cli/states/testFrameworkState.js' -import { HookState } from '../../../src/cli/states/hookState.js' -import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/testFrameworkConstants.js' - -vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) - -describe('TestHubModule skipTestHub marker', () => { - let module: TestHubModule - let sendTestFrameworkEvent: ReturnType - let sendTestSessionEvent: ReturnType - - const instanceIn = (state: State, hook: State, framework = 'WebdriverIO-cucumber') => { - const instance = new TestFrameworkInstance(TrackedInstance.createContext('t'), [framework], { [framework]: '9' }, state, hook) - instance.updateMultipleEntries({ - [TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME]: framework, - [TestFrameworkConstants.KEY_TEST_UUID]: 'uuid-1', - [TestFrameworkConstants.KEY_TEST_RESULT_AT]: 'now', - }) - return instance - } - - beforeEach(() => { - module = new TestHubModule({}) - sendTestFrameworkEvent = vi.spyOn(module, 'sendTestFrameworkEvent').mockResolvedValue(true) - sendTestSessionEvent = vi.spyOn(module, 'sendTestSessionEvent').mockResolvedValue(undefined) - }) - - afterEach(() => { - vi.restoreAllMocks() - }) - - it('handles test and hook events that carry no marker, as before', () => { - module.onBeforeTest({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE) }) - module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE) }) - module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.BEFORE_ALL, HookState.POST) }) - expect(sendTestSessionEvent).toHaveBeenCalledTimes(1) - expect(sendTestFrameworkEvent).toHaveBeenCalledTimes(2) - }) - - it('ignores events marked skipTestHub', () => { - module.onBeforeTest({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE), skipTestHub: true }) - module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.TEST, HookState.POST), skipTestHub: true }) - expect(sendTestSessionEvent).not.toHaveBeenCalled() - expect(sendTestFrameworkEvent).not.toHaveBeenCalled() - }) - - it('only honours the literal true', () => { - module.onAllTestEvents({ instance: instanceIn(TestFrameworkState.TEST, HookState.PRE), skipTestHub: 'true' }) - expect(sendTestFrameworkEvent).toHaveBeenCalledTimes(1) - }) -}) diff --git a/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts b/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts index 47316f20..c1031d39 100644 --- a/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts @@ -152,6 +152,15 @@ describe('TestHubModule', () => { autoInstance: [mockAutomationInstance] }) }) + + it('should skip events marked skipTestHub', async () => { + const sendTestSessionEventSpy = vi.spyOn(testHubModule, 'sendTestSessionEvent').mockResolvedValue() + + await testHubModule.onBeforeTest({ test: { title: 'Test' } as Frameworks.Test, skipTestHub: true }) + + expect(AutomationFramework.getTrackedInstance).not.toHaveBeenCalled() + expect(sendTestSessionEventSpy).not.toHaveBeenCalled() + }) }) describe('onAllTestEvents', () => { @@ -225,6 +234,32 @@ describe('TestHubModule', () => { expect(sendTestFrameworkEventSpy).toHaveBeenCalledWith(mockArgs) }) + + it('should skip events marked skipTestHub', async () => { + const mockInstance = { + getCurrentTestState: vi.fn(() => TestFrameworkState.TEST), + getCurrentHookState: vi.fn(() => HookState.POST) + } + const sendTestFrameworkEventSpy = vi.spyOn(testHubModule, 'sendTestFrameworkEvent').mockResolvedValue() + + await testHubModule.onAllTestEvents({ instance: mockInstance, skipTestHub: true }) + + expect(mockInstance.getCurrentTestState).not.toHaveBeenCalled() + expect(sendTestFrameworkEventSpy).not.toHaveBeenCalled() + }) + + it('should only treat a literal true skipTestHub as the marker', async () => { + const mockInstance = { + getCurrentTestState: vi.fn(() => TestFrameworkState.TEST), + getCurrentHookState: vi.fn(() => HookState.PRE) + } + const sendTestFrameworkEventSpy = vi.spyOn(testHubModule, 'sendTestFrameworkEvent').mockResolvedValue() + const mockArgs = { instance: mockInstance, skipTestHub: 'true' } + + await testHubModule.onAllTestEvents(mockArgs) + + expect(sendTestFrameworkEventSpy).toHaveBeenCalledWith(mockArgs) + }) }) describe('sendTestFrameworkEvent', () => { From 977dbfef10c4c8c385115a024de6d89f4a86d7e9 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 30 Sep 2026 10:13:48 +0530 Subject: [PATCH 13/24] test: fold jasmine CLI cases into the existing cli, reporter, service and insights-handler suites (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../tests/cli/cliUtils.test.ts | 10 + .../tests/cli/index.test.ts | 38 +++ .../tests/cli/wdioJasmineGate.test.ts | 57 ---- .../tests/insights-handler.test.ts | 97 +++++++ .../tests/reporter.jasmineCli.test.ts | 193 ------------- .../tests/reporter.test.ts | 190 +++++++++++++ .../tests/service.jasmineCliGates.test.ts | 254 ------------------ .../tests/service.test.ts | 149 ++++++++++ 8 files changed, 484 insertions(+), 504 deletions(-) delete mode 100644 packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts delete mode 100644 packages/browserstack-service/tests/reporter.jasmineCli.test.ts delete mode 100644 packages/browserstack-service/tests/service.jasmineCliGates.test.ts diff --git a/packages/browserstack-service/tests/cli/cliUtils.test.ts b/packages/browserstack-service/tests/cli/cliUtils.test.ts index 5898bae7..99f23616 100644 --- a/packages/browserstack-service/tests/cli/cliUtils.test.ts +++ b/packages/browserstack-service/tests/cli/cliUtils.test.ts @@ -699,4 +699,14 @@ describe('CLIUtils', () => { expect(CLIUtils.getCliDir()).toBe('') }) }) + + describe('checkCLISupportedFrameworks', () => { + it.each(['mocha', 'cucumber', 'jasmine'])('routes %s to the CLI flow', (framework) => { + expect(CLIUtils.checkCLISupportedFrameworks(framework)).toBe(true) + }) + + it.each([undefined, 'WebdriverIO-jasmine', 'Jasmine', 'unknown'])('keeps %s on the legacy flow', (framework) => { + expect(CLIUtils.checkCLISupportedFrameworks(framework)).toBe(false) + }) + }) }) \ No newline at end of file diff --git a/packages/browserstack-service/tests/cli/index.test.ts b/packages/browserstack-service/tests/cli/index.test.ts index d4415b89..1e723f8b 100644 --- a/packages/browserstack-service/tests/cli/index.test.ts +++ b/packages/browserstack-service/tests/cli/index.test.ts @@ -3,6 +3,11 @@ import * as bstackLogger from '../../src/bstackLogger.js' import { BStackLogger } from '../../src/cli/cliLogger.js' import { BrowserstackCLI } from '../../src/cli/index.js' +import { CLIUtils } from '../../src/cli/cliUtils.js' +import TestFramework from '../../src/cli/frameworks/testFramework.js' +import WdioMochaTestFramework from '../../src/cli/frameworks/wdioMochaTestFramework.js' +import WdioCucumberTestFramework from '../../src/cli/frameworks/wdioCucumberTestFramework.js' +import WdioJasmineTestFramework from '../../src/cli/frameworks/wdioJasmineTestFramework.js' const bstackLoggerSpy = vi.spyOn(bstackLogger.BStackLogger, 'logToFile') bstackLoggerSpy.mockImplementation(() => {}) @@ -119,4 +124,37 @@ describe('BrowserstackCLI bootstrap error surfacing', () => { expect(instance.config.apis).toEqual({ automate: { api: 'https://flat-only.example' } }) }) }) + + describe('setupTestFramework', () => { + afterEach(() => { + instance.testFramework = null + }) + + const setup = (name: string) => { + vi.spyOn(CLIUtils, 'getTestFrameworkDetail').mockReturnValue({ name, version: { [name]: '9.0.0' } }) + instance.setupTestFramework() + return instance.getTestFramework() + } + + it('constructs the jasmine framework for WebdriverIO-jasmine', () => { + const framework = setup('WebdriverIO-jasmine') + expect(framework).toBeInstanceOf(WdioJasmineTestFramework) + expect(framework).not.toBeInstanceOf(WdioMochaTestFramework) + expect(framework).toBeInstanceOf(TestFramework) + expect(framework!.getTestFrameworks()).toEqual(['WebdriverIO-jasmine']) + expect(framework!.getTestFrameworksVersions()).toEqual({ 'WebdriverIO-jasmine': '9.0.0' }) + }) + + it('still constructs the mocha framework for WebdriverIO-mocha', () => { + expect(setup('WebdriverIO-mocha')).toBeInstanceOf(WdioMochaTestFramework) + }) + + it('still constructs the cucumber framework for WebdriverIO-cucumber', () => { + expect(setup('WebdriverIO-cucumber')).toBeInstanceOf(WdioCucumberTestFramework) + }) + + it('leaves an unknown name unregistered', () => { + expect(setup('WebdriverIO-unknown')).toBeNull() + }) + }) }) diff --git a/packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts deleted file mode 100644 index 424744d9..00000000 --- a/packages/browserstack-service/tests/cli/wdioJasmineGate.test.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { describe, expect, it, vi, afterEach } from 'vitest' -import * as bstackLogger from '../../src/bstackLogger.js' - -import { BrowserstackCLI } from '../../src/cli/index.js' -import { CLIUtils } from '../../src/cli/cliUtils.js' -import TestFramework from '../../src/cli/frameworks/testFramework.js' -import WdioMochaTestFramework from '../../src/cli/frameworks/wdioMochaTestFramework.js' -import WdioCucumberTestFramework from '../../src/cli/frameworks/wdioCucumberTestFramework.js' -import WdioJasmineTestFramework from '../../src/cli/frameworks/wdioJasmineTestFramework.js' - -vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) - -describe('CLIUtils.checkCLISupportedFrameworks', () => { - it.each(['mocha', 'cucumber', 'jasmine'])('routes %s to the CLI flow', (framework) => { - expect(CLIUtils.checkCLISupportedFrameworks(framework)).toBe(true) - }) - - it.each([undefined, 'WebdriverIO-jasmine', 'Jasmine', 'unknown'])('keeps %s on the legacy flow', (framework) => { - expect(CLIUtils.checkCLISupportedFrameworks(framework)).toBe(false) - }) -}) - -describe('BrowserstackCLI.setupTestFramework', () => { - const cli = BrowserstackCLI.getInstance() - - afterEach(() => { - vi.restoreAllMocks() - cli.testFramework = null - }) - - const setup = (name: string) => { - vi.spyOn(CLIUtils, 'getTestFrameworkDetail').mockReturnValue({ name, version: { [name]: '9.0.0' } }) - cli.setupTestFramework() - return cli.getTestFramework() - } - - it('constructs the jasmine framework for WebdriverIO-jasmine', () => { - const framework = setup('WebdriverIO-jasmine') - expect(framework).toBeInstanceOf(WdioJasmineTestFramework) - expect(framework).not.toBeInstanceOf(WdioMochaTestFramework) - expect(framework).toBeInstanceOf(TestFramework) - expect(framework!.getTestFrameworks()).toEqual(['WebdriverIO-jasmine']) - expect(framework!.getTestFrameworksVersions()).toEqual({ 'WebdriverIO-jasmine': '9.0.0' }) - }) - - it('still constructs the mocha framework for WebdriverIO-mocha', () => { - expect(setup('WebdriverIO-mocha')).toBeInstanceOf(WdioMochaTestFramework) - }) - - it('still constructs the cucumber framework for WebdriverIO-cucumber', () => { - expect(setup('WebdriverIO-cucumber')).toBeInstanceOf(WdioCucumberTestFramework) - }) - - it('leaves an unknown name unregistered', () => { - expect(setup('WebdriverIO-unknown')).toBeNull() - }) -}) diff --git a/packages/browserstack-service/tests/insights-handler.test.ts b/packages/browserstack-service/tests/insights-handler.test.ts index 7034c976..6d081093 100644 --- a/packages/browserstack-service/tests/insights-handler.test.ts +++ b/packages/browserstack-service/tests/insights-handler.test.ts @@ -10,6 +10,8 @@ import * as utils from '../src/util.js' import * as bstackLogger from '../src/bstackLogger.js' import { TESTOPS_SCREENSHOT_ENV } from '../src/constants.js' import { BrowserstackCLI } from '../src/cli/index.js' +import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' +import { HookState } from '../src/cli/states/hookState.js' const log = logger('test') let insightsHandler: InsightsHandler @@ -736,6 +738,101 @@ describe('browserCommand', () => { insightsHandler.browserCommand('client:afterCommand', { sessionId: 's', method: 'm', endpoint: 'e', result: { value: 'random' } }, {}) expect(uploadEventDataSpy).toBeCalledTimes(0) }) + + describe('on the CLI flow', () => { + const cliBrowser = { on: vi.fn(), sessionId: 's', capabilities: {}, config: {}, execute: vi.fn() } as any + const command = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/title', body: {} } + const result = { ...command, result: { value: 'StackDemo' } } + const lastSpec = { description: 'outer passing test', fullName: 'Nested outer outer passing test', file: '/p/nested.spec.js' } as any + let getInstanceSpy: ReturnType | undefined + + const cliWith = (framework: unknown) => { + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => framework, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as any) + } + + const handlerFor = (framework: string) => { + const handler = new InsightsHandler(cliBrowser, framework) + handler['getIdentifier'] = vi.fn().mockReturnValue('Nested outer outer passing test') + handler['_tests'] = { 'Nested outer outer passing test': { uuid: 'spec-uuid' } } + return handler + } + + beforeEach(() => { + delete process.env[TESTOPS_SCREENSHOT_ENV] + commandSpy.mockRestore() + }) + + afterEach(() => { + getInstanceSpy?.mockRestore() + getInstanceSpy = undefined + }) + + it('sends the HTTP log over gRPC to the named spec, never to the legacy listener', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const handler = handlerFor('jasmine') + const logCreated = vi.spyOn(handler['listener'], 'logCreated').mockImplementation(() => {}) + + await handler.browserCommand('client:beforeCommand', { ...command } as any, lastSpec) + await handler.browserCommand('client:afterCommand', { ...result } as any, lastSpec) + + expect(logCreated).not.toHaveBeenCalled() + expect(trackEvent).toHaveBeenCalledTimes(1) + const [state, hook, { logEntry }] = trackEvent.mock.calls[0] + expect([state, hook]).toEqual([TestFrameworkState.LOG, HookState.POST]) + expect(logEntry.kind).toBe('HTTP') + expect(logEntry.test_run_uuid).toBe('spec-uuid') + expect(JSON.parse(logEntry.message)).toEqual({ path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } }) + }) + + it('keeps the legacy HTTP log shape on the listener when the CLI is not running', async () => { + const handler = handlerFor('jasmine') + const logCreated = vi.spyOn(handler['listener'], 'logCreated').mockImplementation(() => {}) + + await handler.browserCommand('client:beforeCommand', { ...command } as any, lastSpec) + await handler.browserCommand('client:afterCommand', { ...result } as any, lastSpec) + + expect(logCreated).toHaveBeenCalledWith([{ + test_run_uuid: 'spec-uuid', + timestamp: expect.any(String), + kind: 'HTTP', + http_response: { path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } } + }]) + }) + + it('names the spec on a jasmine screenshot, and leaves the mocha screenshot entry unchanged', async () => { + process.env[TESTOPS_SCREENSHOT_ENV] = 'true' + const screenshot = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/screenshot', result: { value: 'b64' } } + + const jasmineTrack = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent: jasmineTrack }) + await handlerFor('jasmine').browserCommand('client:afterCommand', { ...screenshot } as any, lastSpec) + getInstanceSpy!.mockRestore() + + const mochaTrack = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent: mochaTrack }) + await handlerFor('mocha').browserCommand('client:afterCommand', { ...screenshot } as any, { title: 't' } as any) + + expect(jasmineTrack.mock.calls[0][2].logEntry).toEqual({ kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: expect.any(String), test_run_uuid: 'spec-uuid' }) + expect(Object.keys(mochaTrack.mock.calls[0][2].logEntry)).toEqual(['kind', 'message', 'timestamp']) + }) + + it('drops commands with no spec yet, as legacy did before the first spec', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const handler = handlerFor('jasmine') + handler['getIdentifier'] = vi.fn().mockReturnValue(undefined) + + await handler.browserCommand('client:beforeCommand', { ...command } as any, {} as any) + await handler.browserCommand('client:afterCommand', { ...result } as any, {} as any) + + expect(trackEvent).not.toHaveBeenCalled() + }) + }) }) describe('getIdentifier', () => { diff --git a/packages/browserstack-service/tests/reporter.jasmineCli.test.ts b/packages/browserstack-service/tests/reporter.jasmineCli.test.ts deleted file mode 100644 index 76ea1a68..00000000 --- a/packages/browserstack-service/tests/reporter.jasmineCli.test.ts +++ /dev/null @@ -1,193 +0,0 @@ -import path from 'node:path' -import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' - -import TestReporter from '../src/reporter.js' -import { BrowserstackCLI } from '../src/cli/index.js' -import WdioJasmineTestFramework from '../src/cli/frameworks/wdioJasmineTestFramework.js' -import * as utils from '../src/util.js' -import * as bstackLogger from '../src/bstackLogger.js' - -vi.mock('uuid', () => ({ v4: () => 'legacy-uuid' })) -vi.mock('@wdio/reporter', () => import(path.join(process.cwd(), '__mocks__', '@wdio/reporter'))) -vi.mock('@wdio/logger', () => import(path.join(process.cwd(), '__mocks__', '@wdio/logger'))) - -vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) - -const runnerConfig = { - type: 'runner', - cid: '0-0', - capabilities: { browserName: 'chrome', browserVersion: '151' }, - config: { framework: 'jasmine', hostname: 'hub.browserstack.com' }, - specs: ['/work/test/p2/nested.spec.js'], - sessionId: 'sessionId' -} - -const suite = { title: 'Nested outer', file: '/work/test/p2/nested.spec.js' } -const testStats = () => ({ - type: 'test', - uid: 'outer passing test0', - title: 'outer passing test', - fullTitle: 'Nested outer outer passing test', - start: new Date('2026-09-25T15:34:12.259Z'), - _duration: 0, - retries: 0, - state: 'pending', -}) -const hookStats = () => ({ type: 'hook', uid: 'h', title: '"before all" hook', start: new Date(), _duration: 0 }) -const logEntry = () => ({ timestamp: new Date().toISOString(), level: 'INFO', message: 'console line', kind: 'TEST_LOG' as const, http_response: {} }) - -describe('reporter jasmine arm', () => { - let reporter: TestReporter - let listener: Record> - let framework: WdioJasmineTestFramework - - const setCli = (running: boolean, testFramework: unknown) => { - vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ - isRunning: () => running, - getTestFramework: () => testFramework, - } as unknown as BrowserstackCLI) - } - - beforeEach(async () => { - vi.spyOn(utils, 'getCloudProvider').mockReturnValue('browserstack') - vi.spyOn(utils, 'getGitMetaData').mockResolvedValue(undefined as any) - framework = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin') - for (const m of ['onReporterTestStart', 'onReporterTestEnd', 'onReporterHookStart', 'onReporterHookEnd', 'onReporterLog'] as const) { - vi.spyOn(framework, m) - } - vi.mocked(framework.onReporterTestStart).mockReturnValue('cli-uuid') - vi.mocked(framework.onReporterTestEnd).mockReturnValue(undefined) - vi.mocked(framework.onReporterHookStart).mockReturnValue(undefined) - vi.mocked(framework.onReporterHookEnd).mockReturnValue(undefined) - vi.mocked(framework.onReporterLog).mockReturnValue(undefined) - - reporter = new TestReporter({}) - await reporter.onRunnerStart(runnerConfig as any) - reporter.onSuiteStart(suite as any) - listener = { - testStarted: vi.spyOn(reporter['listener'], 'testStarted').mockImplementation(() => {}), - testFinished: vi.spyOn(reporter['listener'], 'testFinished').mockImplementation(() => {}), - hookStarted: vi.spyOn(reporter['listener'], 'hookStarted').mockImplementation(() => {}), - hookFinished: vi.spyOn(reporter['listener'], 'hookFinished').mockImplementation(() => {}), - logCreated: vi.spyOn(reporter['listener'], 'logCreated').mockImplementation(() => {}), - } - }) - - afterEach(() => { - vi.restoreAllMocks() - }) - - describe('on the CLI flow', () => { - beforeEach(() => setCli(true, framework)) - - it('feeds every test and hook event to the framework and enqueues nothing on the legacy Listener', async () => { - const stats = testStats() - await reporter.onHookStart(hookStats() as any) - await reporter.onHookEnd(hookStats() as any) - await reporter.onTestStart(stats as any) - await reporter.onTestEnd({ ...stats, state: 'passed' } as any) - await reporter.appendTestItemLog(logEntry()) - - expect(framework.onReporterHookStart).toHaveBeenCalledTimes(1) - expect(framework.onReporterHookEnd).toHaveBeenCalledTimes(1) - expect(framework.onReporterTestStart).toHaveBeenCalledTimes(1) - expect(framework.onReporterTestEnd).toHaveBeenCalledTimes(1) - expect(framework.onReporterLog).toHaveBeenCalledTimes(1) - for (const spy of Object.values(listener)) { - expect(spy).not.toHaveBeenCalled() - } - }) - - it('passes the suite stack and suite file', async () => { - await reporter.onTestStart(testStats() as any) - expect(framework.onReporterTestStart).toHaveBeenCalledWith( - expect.objectContaining({ fullTitle: 'Nested outer outer passing test' }), - { scopes: ['Nested outer'], suiteFile: '/work/test/p2/nested.spec.js' } - ) - }) - - it('records the CLI uuid for the spec so command-result lookups resolve to the wire uuid', async () => { - await reporter.onTestStart(testStats() as any) - expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: 'cli-uuid' }) - }) - - it('names the current test for Percy\'s testCase, as legacy getRunData did', async () => { - await reporter.onTestStart(testStats() as any) - expect((TestReporter as any).currentTest).toMatchObject({ uuid: 'cli-uuid', name: 'outer passing test' }) - }) - - it('keeps legacy\'s end stamp and forced hook pass on the WDIO stats objects', async () => { - const stats = testStats() as Record - await reporter.onTestEnd(stats as any) - expect(stats.end).toBeInstanceOf(Date) - const hook = hookStats() as Record - await reporter.onHookEnd(hook as any) - expect(hook.state).toBe('passed') - }) - - it('drops ', async () => { - await reporter.onTestStart({ ...testStats(), fullTitle: '' } as any) - await reporter.onTestEnd({ ...testStats(), fullTitle: '' } as any) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() - expect(framework.onReporterTestEnd).not.toHaveBeenCalled() - }) - - it('sends nothing when Test Observability is opted out', async () => { - reporter['_observability'] = false - await reporter.onTestStart(testStats() as any) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() - expect(listener.testStarted).not.toHaveBeenCalled() - }) - - it('never falls back to the legacy Listener when the tracker is missing', async () => { - setCli(true, null) - await reporter.onTestStart(testStats() as any) - await reporter.appendTestItemLog(logEntry()) - expect(listener.testStarted).not.toHaveBeenCalled() - expect(listener.logCreated).not.toHaveBeenCalled() - }) - - it('reports unsynchronised while the framework has queued events', () => { - const spy = vi.spyOn(WdioJasmineTestFramework, 'isIdle').mockReturnValue(false) - expect(reporter.isSynchronised).toBe(false) - spy.mockReturnValue(true) - expect(reporter.isSynchronised).toBe(true) - }) - }) - - describe('on the legacy flow', () => { - beforeEach(() => setCli(false, null)) - - it('enqueues on the legacy Listener and never touches the framework', async () => { - const stats = testStats() - await reporter.onHookStart(hookStats() as any) - await reporter.onHookEnd(hookStats() as any) - await reporter.onTestStart(stats as any) - await reporter.onTestEnd({ ...stats, state: 'passed' } as any) - reporter['_currentHook'] = {} - await reporter.appendTestItemLog(logEntry()) - - expect(listener.hookStarted).toHaveBeenCalledTimes(1) - expect(listener.hookFinished).toHaveBeenCalledTimes(1) - expect(listener.testStarted).toHaveBeenCalledTimes(1) - expect(listener.testFinished).toHaveBeenCalledTimes(1) - expect(listener.logCreated).toHaveBeenCalledTimes(1) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() - expect(framework.onReporterLog).not.toHaveBeenCalled() - expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: 'legacy-uuid' }) - }) - }) - - describe('mocha on the CLI flow', () => { - beforeEach(() => setCli(true, framework)) - - it('leaves the mocha arm unchanged: no start/end publishing from the reporter', async () => { - reporter['_config']!.framework = 'mocha' - await reporter.onTestStart(testStats() as any) - await reporter.onTestEnd({ ...testStats(), state: 'passed' } as any) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() - expect(listener.testStarted).not.toHaveBeenCalled() - expect(listener.testFinished).not.toHaveBeenCalled() - }) - }) -}) diff --git a/packages/browserstack-service/tests/reporter.test.ts b/packages/browserstack-service/tests/reporter.test.ts index da59c2c2..2714f675 100644 --- a/packages/browserstack-service/tests/reporter.test.ts +++ b/packages/browserstack-service/tests/reporter.test.ts @@ -4,6 +4,8 @@ import { describe, expect, it, vi, beforeEach, afterEach, beforeAll, afterAll } import type { StdLog } from '../src/index.js' import TestReporter from '../src/reporter.js' +import { BrowserstackCLI } from '../src/cli/index.js' +import WdioJasmineTestFramework from '../src/cli/frameworks/wdioJasmineTestFramework.js' import * as utils from '../src/util.js' import * as bstackLogger from '../src/bstackLogger.js' @@ -354,4 +356,192 @@ describe('test-reporter', () => { expect(sendDataSpy).toBeCalledTimes(0) }) }) + + describe('jasmine CLI feed', () => { + const jasmineRunnerConfig = { + type: 'runner', + cid: '0-0', + capabilities: { browserName: 'chrome', browserVersion: '151' }, + config: { framework: 'jasmine', hostname: 'hub.browserstack.com' }, + specs: ['/work/test/p2/nested.spec.js'], + sessionId: 'sessionId' + } + const suite = { title: 'Nested outer', file: '/work/test/p2/nested.spec.js' } + const jasmineTestStats = () => ({ + type: 'test', + uid: 'outer passing test0', + title: 'outer passing test', + fullTitle: 'Nested outer outer passing test', + start: new Date('2026-09-25T15:34:12.259Z'), + _duration: 0, + retries: 0, + state: 'pending', + }) + const hookStats = () => ({ type: 'hook', uid: 'h', title: '"before all" hook', start: new Date(), _duration: 0 }) + const logEntry = () => ({ timestamp: new Date().toISOString(), level: 'INFO', message: 'console line', kind: 'TEST_LOG' as const, http_response: {} }) + + let reporter: TestReporter + let listener: Record> + let framework: WdioJasmineTestFramework + let getInstanceSpy: ReturnType | undefined + let getGitMetaDataSpy: ReturnType + let isIdleSpy: ReturnType | undefined + + const setCli = (running: boolean, testFramework: unknown) => { + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => running, + getTestFramework: () => testFramework, + } as unknown as BrowserstackCLI) + } + + beforeEach(async () => { + vi.spyOn(utils, 'getCloudProvider').mockReturnValue('browserstack') + getGitMetaDataSpy = vi.spyOn(utils, 'getGitMetaData').mockResolvedValue(undefined as any) + framework = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin') + for (const m of ['onReporterTestStart', 'onReporterTestEnd', 'onReporterHookStart', 'onReporterHookEnd', 'onReporterLog'] as const) { + vi.spyOn(framework, m) + } + vi.mocked(framework.onReporterTestStart).mockReturnValue('cli-uuid') + vi.mocked(framework.onReporterTestEnd).mockReturnValue(undefined) + vi.mocked(framework.onReporterHookStart).mockReturnValue(undefined) + vi.mocked(framework.onReporterHookEnd).mockReturnValue(undefined) + vi.mocked(framework.onReporterLog).mockReturnValue(undefined) + + reporter = new TestReporter({}) + await reporter.onRunnerStart(jasmineRunnerConfig as any) + reporter.onSuiteStart(suite as any) + listener = { + testStarted: vi.spyOn(reporter['listener'], 'testStarted').mockImplementation(() => {}), + testFinished: vi.spyOn(reporter['listener'], 'testFinished').mockImplementation(() => {}), + hookStarted: vi.spyOn(reporter['listener'], 'hookStarted').mockImplementation(() => {}), + hookFinished: vi.spyOn(reporter['listener'], 'hookFinished').mockImplementation(() => {}), + logCreated: vi.spyOn(reporter['listener'], 'logCreated').mockImplementation(() => {}), + } + }) + + afterEach(() => { + getInstanceSpy?.mockRestore() + getInstanceSpy = undefined + isIdleSpy?.mockRestore() + isIdleSpy = undefined + getGitMetaDataSpy.mockRestore() + for (const spy of Object.values(listener)) { + spy.mockRestore() + } + }) + + describe('on the CLI flow', () => { + beforeEach(() => setCli(true, framework)) + + it('feeds every test and hook event to the framework and enqueues nothing on the legacy Listener', async () => { + const stats = jasmineTestStats() + await reporter.onHookStart(hookStats() as any) + await reporter.onHookEnd(hookStats() as any) + await reporter.onTestStart(stats as any) + await reporter.onTestEnd({ ...stats, state: 'passed' } as any) + await reporter.appendTestItemLog(logEntry()) + + expect(framework.onReporterHookStart).toHaveBeenCalledTimes(1) + expect(framework.onReporterHookEnd).toHaveBeenCalledTimes(1) + expect(framework.onReporterTestStart).toHaveBeenCalledTimes(1) + expect(framework.onReporterTestEnd).toHaveBeenCalledTimes(1) + expect(framework.onReporterLog).toHaveBeenCalledTimes(1) + for (const spy of Object.values(listener)) { + expect(spy).not.toHaveBeenCalled() + } + }) + + it('passes the suite stack and suite file', async () => { + await reporter.onTestStart(jasmineTestStats() as any) + expect(framework.onReporterTestStart).toHaveBeenCalledWith( + expect.objectContaining({ fullTitle: 'Nested outer outer passing test' }), + { scopes: ['Nested outer'], suiteFile: '/work/test/p2/nested.spec.js' } + ) + }) + + it('records the CLI uuid for the spec so command-result lookups resolve to the wire uuid', async () => { + await reporter.onTestStart(jasmineTestStats() as any) + expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: 'cli-uuid' }) + }) + + it('names the current test for Percy\'s testCase, as legacy getRunData did', async () => { + await reporter.onTestStart(jasmineTestStats() as any) + expect((TestReporter as any).currentTest).toMatchObject({ uuid: 'cli-uuid', name: 'outer passing test' }) + }) + + it('keeps legacy\'s end stamp and forced hook pass on the WDIO stats objects', async () => { + const stats = jasmineTestStats() as Record + await reporter.onTestEnd(stats as any) + expect(stats.end).toBeInstanceOf(Date) + const hook = hookStats() as Record + await reporter.onHookEnd(hook as any) + expect(hook.state).toBe('passed') + }) + + it('drops ', async () => { + await reporter.onTestStart({ ...jasmineTestStats(), fullTitle: '' } as any) + await reporter.onTestEnd({ ...jasmineTestStats(), fullTitle: '' } as any) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(framework.onReporterTestEnd).not.toHaveBeenCalled() + }) + + it('sends nothing when Test Observability is opted out', async () => { + reporter['_observability'] = false + await reporter.onTestStart(jasmineTestStats() as any) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(listener.testStarted).not.toHaveBeenCalled() + }) + + it('never falls back to the legacy Listener when the tracker is missing', async () => { + setCli(true, null) + await reporter.onTestStart(jasmineTestStats() as any) + await reporter.appendTestItemLog(logEntry()) + expect(listener.testStarted).not.toHaveBeenCalled() + expect(listener.logCreated).not.toHaveBeenCalled() + }) + + it('reports unsynchronised while the framework has queued events', () => { + isIdleSpy = vi.spyOn(WdioJasmineTestFramework, 'isIdle').mockReturnValue(false) + expect(reporter.isSynchronised).toBe(false) + isIdleSpy.mockReturnValue(true) + expect(reporter.isSynchronised).toBe(true) + }) + }) + + describe('on the legacy flow', () => { + beforeEach(() => setCli(false, null)) + + it('enqueues on the legacy Listener and never touches the framework', async () => { + const stats = jasmineTestStats() + await reporter.onHookStart(hookStats() as any) + await reporter.onHookEnd(hookStats() as any) + await reporter.onTestStart(stats as any) + await reporter.onTestEnd({ ...stats, state: 'passed' } as any) + reporter['_currentHook'] = {} + await reporter.appendTestItemLog(logEntry()) + + expect(listener.hookStarted).toHaveBeenCalledTimes(1) + expect(listener.hookFinished).toHaveBeenCalledTimes(1) + expect(listener.testStarted).toHaveBeenCalledTimes(1) + expect(listener.testFinished).toHaveBeenCalledTimes(1) + expect(listener.logCreated).toHaveBeenCalledTimes(1) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(framework.onReporterLog).not.toHaveBeenCalled() + expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: '123456789' }) + }) + }) + + describe('mocha on the CLI flow', () => { + beforeEach(() => setCli(true, framework)) + + it('leaves the mocha arm unchanged: no start/end publishing from the reporter', async () => { + reporter['_config']!.framework = 'mocha' + await reporter.onTestStart(jasmineTestStats() as any) + await reporter.onTestEnd({ ...jasmineTestStats(), state: 'passed' } as any) + expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(listener.testStarted).not.toHaveBeenCalled() + expect(listener.testFinished).not.toHaveBeenCalled() + }) + }) + }) }) diff --git a/packages/browserstack-service/tests/service.jasmineCliGates.test.ts b/packages/browserstack-service/tests/service.jasmineCliGates.test.ts deleted file mode 100644 index 3a891c8f..00000000 --- a/packages/browserstack-service/tests/service.jasmineCliGates.test.ts +++ /dev/null @@ -1,254 +0,0 @@ -import path from 'node:path' - -import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest' - -import BrowserstackService from '../src/service.js' -import InsightsHandler from '../src/insights-handler.js' -import * as utils from '../src/util.js' -import * as skipReporter from '../src/cli/skipReporter.js' -import { BrowserstackCLI } from '../src/cli/index.js' -import * as bstackLogger from '../src/bstackLogger.js' -import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' -import { HookState } from '../src/cli/states/hookState.js' -import { TESTOPS_SCREENSHOT_ENV } from '../src/constants.js' -import TestFramework from '../src/cli/frameworks/testFramework.js' - -vi.mock('fetch') -vi.mock('@wdio/logger', () => import(path.join(process.cwd(), '__mocks__', '@wdio/logger'))) -vi.mock('uuid', () => ({ v4: () => '123456789' })) -vi.mock('../src/data-store.js', () => ({ saveWorkerData: vi.fn() })) -vi.mock('../src/instrumentation/performance/performance-tester.js', () => ({ - default: { - start: vi.fn(), - end: vi.fn(), - startMonitoring: vi.fn(), - measureWrapper: vi.fn().mockImplementation((_name: string, fn: Function) => fn), - Measure: vi.fn().mockImplementation(() => (_target: any, _propertyKey: string, descriptor: PropertyDescriptor) => descriptor), - browser: undefined, - scenarioThatRan: [], - } -})) -vi.mock('../src/cli/index.js', () => ({ - BrowserstackCLI: { - getInstance: () => ({ - isRunning: () => false, - getTestFramework: () => null, - getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) - }) - } -})) - -vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) - -// What WDIO hands a jasmine hook: a copy of the last started spec (no `title`), then the hookName. -const lastSpec = { description: 'outer passing test', fullName: 'Nested outer outer passing test', file: '/p/nested.spec.js' } as any -const mochaHook = (title: string) => ({ title, ctx: { test: { parent: { title: 'suite', tests: [], suites: [] } } } }) as any - -let getInstanceSpy: ReturnType | undefined - -const cliWith = (framework: unknown) => { - getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ - isRunning: () => true, - getTestFramework: () => framework, - getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) - } as any) -} - -const makeService = (framework: string) => new BrowserstackService({} as any, [] as any, { user: 'foo', key: 'bar', framework } as any) - -afterEach(() => { - getInstanceSpy?.mockRestore() - getInstanceSpy = undefined -}) - -describe('service hooks on the CLI flow (S4)', () => { - it('classifies jasmine hooks by hookName, without touching the missing title', async () => { - const trackEvent = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent }) - const getHookTypeSpy = vi.spyOn(utils, 'getHookType') - const service = makeService('jasmine') - - for (const hookName of ['beforeAll', 'beforeEach']) { - await service.beforeHook(lastSpec, {}, hookName) - } - for (const hookName of ['afterEach', 'afterAll']) { - await service.afterHook(lastSpec, {}, { passed: true } as any, hookName) - } - - expect(getHookTypeSpy).not.toHaveBeenCalled() - expect(trackEvent.mock.calls.map(([state, hook]) => [state, hook])).toEqual([ - [TestFrameworkState.BEFORE_ALL, HookState.PRE], - [TestFrameworkState.BEFORE_EACH, HookState.PRE], - [TestFrameworkState.AFTER_EACH, HookState.POST], - [TestFrameworkState.AFTER_ALL, HookState.POST], - ]) - getHookTypeSpy.mockRestore() - }) - - it('sends nothing for a jasmine hook without a hookName, and does not throw', async () => { - const trackEvent = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent }) - const service = makeService('jasmine') - - await expect(service.beforeHook({} as any, {})).resolves.toBeUndefined() - await expect(service.afterHook({} as any, {}, { passed: true } as any)).resolves.toBeUndefined() - expect(trackEvent).not.toHaveBeenCalled() - }) - - it('never runs the mocha skip cascade for a failed jasmine hook, but still records the hook failure', async () => { - cliWith({ trackEvent: vi.fn().mockResolvedValue(undefined) }) - const cascade = vi.spyOn(skipReporter, 'reportSuiteSkipped').mockResolvedValue(undefined as any) - const service = makeService('jasmine') - - await service.afterHook({ ...lastSpec, ctx: { test: { parent: {} } } }, {}, { passed: false, error: new Error('beforeAll failed') } as any, 'beforeAll') - - expect(cascade).not.toHaveBeenCalled() - expect(service['_hookFailReasons']).toEqual(['beforeAll failed']) - cascade.mockRestore() - }) - - it('keeps mocha on its title classification and skip cascade', async () => { - const trackEvent = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent }) - const cascade = vi.spyOn(skipReporter, 'reportSuiteSkipped').mockResolvedValue(undefined as any) - const service = makeService('mocha') - - await service.beforeHook(mochaHook('"before each" hook for "t"'), {}, 'beforeEach') - await service.afterHook(mochaHook('"before all" hook for "t"'), {}, { passed: false, error: new Error('x') } as any, 'afterAll') - - expect(trackEvent.mock.calls.map(([state, hook]) => [state, hook])).toEqual([ - [TestFrameworkState.BEFORE_EACH, HookState.PRE], - [TestFrameworkState.BEFORE_ALL, HookState.POST], - ]) - expect(cascade).toHaveBeenCalledTimes(1) - cascade.mockRestore() - }) -}) - -describe('service.beforeTest on the CLI flow (S5)', () => { - const runBeforeTest = async (framework: string, test: Record) => { - const trackEvent = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent }) - const service = makeService(framework) - const annotate = vi.spyOn(service as any, '_setAnnotation').mockResolvedValue(undefined) - const getState = vi.spyOn(TestFramework, 'getState').mockReturnValue('spec-uuid') - await service.beforeTest(test as any) - getState.mockRestore() - return { annotate, trackEvent } - } - - it('annotates each jasmine spec with its full name, before the modules see TEST/PRE', async () => { - const { annotate, trackEvent } = await runBeforeTest('jasmine', lastSpec) - expect(annotate).toHaveBeenCalledWith('Test: Nested outer outer passing test') - const testPre = trackEvent.mock.calls.findIndex(([state]) => state === TestFrameworkState.TEST) - expect(annotate.mock.invocationCallOrder[0]).toBeLessThan(trackEvent.mock.invocationCallOrder[testPre]) - }) - - it('does not annotate mocha tests on the CLI flow', async () => { - const { annotate } = await runBeforeTest('mocha', { title: 't', parent: 'suite' }) - expect(annotate).not.toHaveBeenCalled() - }) -}) - -describe('service.before on the CLI flow (S6)', () => { - const registeredEvents = async (framework: string) => { - process.env.BROWSERSTACK_OBSERVABILITY = 'true' - cliWith(null) - const browser = { on: vi.fn(), sessionId: 's1', capabilities: {}, config: {}, execute: vi.fn(), executeScript: vi.fn() } as any - const service = new BrowserstackService({} as any, [{}] as any, { user: 'foo', key: 'bar', framework, capabilities: {} } as any) - await service.before(service['_config'] as any, [], browser) - delete process.env.BROWSERSTACK_OBSERVABILITY - return vi.mocked(browser.on).mock.calls.map(([event]: [string]) => event) - } - - it('registers command and result for jasmine', async () => { - const events = await registeredEvents('jasmine') - expect(events).toContain('command') - expect(events).toContain('result') - }) - - it.each(['mocha', 'cucumber'])('keeps %s on result only', async (framework) => { - const events = await registeredEvents(framework) - expect(events).toContain('result') - expect(events).not.toContain('command') - }) -}) - -describe('insights-handler.browserCommand on the CLI flow (S6)', () => { - const browser = { on: vi.fn(), sessionId: 's', capabilities: {}, config: {}, execute: vi.fn() } as any - const command = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/title', body: {} } - const result = { ...command, result: { value: 'StackDemo' } } - - const handlerFor = (framework: string) => { - const handler = new InsightsHandler(browser, framework) - handler['getIdentifier'] = vi.fn().mockReturnValue('Nested outer outer passing test') - handler['_tests'] = { 'Nested outer outer passing test': { uuid: 'spec-uuid' } } - return handler - } - - beforeEach(() => { - delete process.env[TESTOPS_SCREENSHOT_ENV] - }) - - it('sends the HTTP log over gRPC to the named spec, never to the legacy listener', async () => { - const trackEvent = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent }) - const handler = handlerFor('jasmine') - const logCreated = vi.spyOn(handler['listener'], 'logCreated').mockImplementation(() => {}) - - await handler.browserCommand('client:beforeCommand', { ...command } as any, lastSpec) - await handler.browserCommand('client:afterCommand', { ...result } as any, lastSpec) - - expect(logCreated).not.toHaveBeenCalled() - expect(trackEvent).toHaveBeenCalledTimes(1) - const [state, hook, { logEntry }] = trackEvent.mock.calls[0] - expect([state, hook]).toEqual([TestFrameworkState.LOG, HookState.POST]) - expect(logEntry.kind).toBe('HTTP') - expect(logEntry.test_run_uuid).toBe('spec-uuid') - expect(JSON.parse(logEntry.message)).toEqual({ path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } }) - }) - - it('keeps the legacy HTTP log shape on the listener when the CLI is not running', async () => { - const handler = handlerFor('jasmine') - const logCreated = vi.spyOn(handler['listener'], 'logCreated').mockImplementation(() => {}) - - await handler.browserCommand('client:beforeCommand', { ...command } as any, lastSpec) - await handler.browserCommand('client:afterCommand', { ...result } as any, lastSpec) - - expect(logCreated).toHaveBeenCalledWith([{ - test_run_uuid: 'spec-uuid', - timestamp: expect.any(String), - kind: 'HTTP', - http_response: { path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } } - }]) - }) - - it('names the spec on a jasmine screenshot, and leaves the mocha screenshot entry unchanged', async () => { - process.env[TESTOPS_SCREENSHOT_ENV] = 'true' - const screenshot = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/screenshot', result: { value: 'b64' } } - - const jasmineTrack = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent: jasmineTrack }) - await handlerFor('jasmine').browserCommand('client:afterCommand', { ...screenshot } as any, lastSpec) - getInstanceSpy!.mockRestore() - - const mochaTrack = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent: mochaTrack }) - await handlerFor('mocha').browserCommand('client:afterCommand', { ...screenshot } as any, { title: 't' } as any) - - expect(jasmineTrack.mock.calls[0][2].logEntry).toEqual({ kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: expect.any(String), test_run_uuid: 'spec-uuid' }) - expect(Object.keys(mochaTrack.mock.calls[0][2].logEntry)).toEqual(['kind', 'message', 'timestamp']) - }) - - it('drops commands with no spec yet, as legacy did before the first spec', async () => { - const trackEvent = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent }) - const handler = handlerFor('jasmine') - handler['getIdentifier'] = vi.fn().mockReturnValue(undefined) - - await handler.browserCommand('client:beforeCommand', { ...command } as any, {} as any) - await handler.browserCommand('client:afterCommand', { ...result } as any, {} as any) - - expect(trackEvent).not.toHaveBeenCalled() - }) -}) diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index aa32efbc..15907aa6 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -7,6 +7,8 @@ import BrowserstackService from '../src/service.js' import * as utils from '../src/util.js' import InsightsHandler from '../src/insights-handler.js' import { BrowserstackCLI } from '../src/cli/index.js' +import * as skipReporter from '../src/cli/skipReporter.js' +import TestFramework from '../src/cli/frameworks/testFramework.js' import AccessibilityModule from '../src/cli/modules/accessibilityModule.js' import * as bstackLogger from '../src/bstackLogger.js' import AutomationFramework from '../src/cli/frameworks/automationFramework.js' @@ -20,6 +22,8 @@ const jasmineSuiteTitle = 'Jasmine__TopLevel__Suite' const sessionBaseUrl = 'https://api.browserstack.com/automate/sessions' const sessionId = 'session123' const sessionIdA = 'session456' +// What WDIO hands a jasmine hook: a copy of the last started spec (no `title`), then the hookName. +const jasmineLastSpec = { description: 'outer passing test', fullName: 'Nested outer outer passing test', file: '/p/nested.spec.js' } as any vi.mock('fetch') vi.mock('@wdio/logger', () => import(path.join(process.cwd(), '__mocks__', '@wdio/logger'))) @@ -850,6 +854,41 @@ describe('before', () => { expect(browserA.overwriteCommand).not.toHaveBeenCalled() expect(browserB.overwriteCommand).toHaveBeenCalledWith('execute', expect.any(Function)) }) + + describe('command and result registration on the CLI flow', () => { + let getInstanceSpy: ReturnType | undefined + + afterEach(() => { + getInstanceSpy?.mockRestore() + getInstanceSpy = undefined + }) + + const registeredEvents = async (framework: string) => { + process.env.BROWSERSTACK_OBSERVABILITY = 'true' + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => null, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as any) + const cliBrowser = { on: vi.fn(), sessionId: 's1', capabilities: {}, config: {}, execute: vi.fn(), executeScript: vi.fn() } as any + const service = new BrowserstackService({} as any, [{}] as any, { user: 'foo', key: 'bar', framework, capabilities: {} } as any) + await service.before(service['_config'] as any, [], cliBrowser) + delete process.env.BROWSERSTACK_OBSERVABILITY + return vi.mocked(cliBrowser.on).mock.calls.map(([event]: [string]) => event) + } + + it('registers command and result for jasmine', async () => { + const events = await registeredEvents('jasmine') + expect(events).toContain('command') + expect(events).toContain('result') + }) + + it.each(['mocha', 'cucumber'])('keeps %s on result only', async (framework) => { + const events = await registeredEvents(framework) + expect(events).toContain('result') + expect(events).not.toContain('command') + }) + }) }) describe('beforeHook', () => { @@ -1154,6 +1193,42 @@ describe('beforeTest', () => { ) }) }) + + describe('on the CLI flow', () => { + let getInstanceSpy: ReturnType | undefined + + afterEach(() => { + getInstanceSpy?.mockRestore() + getInstanceSpy = undefined + }) + + const runBeforeTest = async (framework: string, test: Record) => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => ({ trackEvent }), + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as any) + const service = new BrowserstackService({} as any, [] as any, { user: 'foo', key: 'bar', framework } as any) + const annotate = vi.spyOn(service as any, '_setAnnotation').mockResolvedValue(undefined) + const getState = vi.spyOn(TestFramework, 'getState').mockReturnValue('spec-uuid') + await service.beforeTest(test as any) + getState.mockRestore() + return { annotate, trackEvent } + } + + it('annotates each jasmine spec with its full name, before the modules see TEST/PRE', async () => { + const { annotate, trackEvent } = await runBeforeTest('jasmine', jasmineLastSpec) + expect(annotate).toHaveBeenCalledWith('Test: Nested outer outer passing test') + const testPre = trackEvent.mock.calls.findIndex(([state]) => state === TestFrameworkState.TEST) + expect(annotate.mock.invocationCallOrder[0]).toBeLessThan(trackEvent.mock.invocationCallOrder[testPre]) + }) + + it('does not annotate mocha tests on the CLI flow', async () => { + const { annotate } = await runBeforeTest('mocha', { title: 't', parent: 'suite' }) + expect(annotate).not.toHaveBeenCalled() + }) + }) }) describe('afterTest', () => { @@ -2638,6 +2713,80 @@ describe('beforeHook (CLI hook reporting)', () => { expect(service['_insightsHandler']!.beforeHook).toHaveBeenCalledTimes(1) }) + + describe('classification by framework', () => { + const mochaHook = (title: string) => ({ title, ctx: { test: { parent: { title: 'suite', tests: [], suites: [] } } } }) as any + const cliWith = (framework: unknown) => { + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => framework, + getAutomationFramework: () => ({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + } as any) + } + const makeService = (framework: string) => new BrowserstackService({} as any, [] as any, { user: 'foo', key: 'bar', framework } as any) + + it('classifies jasmine hooks by hookName, without touching the missing title', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const getHookTypeSpy = vi.spyOn(utils, 'getHookType') + const service = makeService('jasmine') + + for (const hookName of ['beforeAll', 'beforeEach']) { + await service.beforeHook(jasmineLastSpec, {}, hookName) + } + for (const hookName of ['afterEach', 'afterAll']) { + await service.afterHook(jasmineLastSpec, {}, { passed: true } as any, hookName) + } + + expect(getHookTypeSpy).not.toHaveBeenCalled() + expect(trackEvent.mock.calls.map(([state, hook]) => [state, hook])).toEqual([ + [TestFrameworkState.BEFORE_ALL, HookState.PRE], + [TestFrameworkState.BEFORE_EACH, HookState.PRE], + [TestFrameworkState.AFTER_EACH, HookState.POST], + [TestFrameworkState.AFTER_ALL, HookState.POST], + ]) + getHookTypeSpy.mockRestore() + }) + + it('sends nothing for a jasmine hook without a hookName, and does not throw', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const service = makeService('jasmine') + + await expect(service.beforeHook({} as any, {})).resolves.toBeUndefined() + await expect(service.afterHook({} as any, {}, { passed: true } as any)).resolves.toBeUndefined() + expect(trackEvent).not.toHaveBeenCalled() + }) + + it('never runs the mocha skip cascade for a failed jasmine hook, but still records the hook failure', async () => { + cliWith({ trackEvent: vi.fn().mockResolvedValue(undefined) }) + const cascade = vi.spyOn(skipReporter, 'reportSuiteSkipped').mockResolvedValue(undefined as any) + const service = makeService('jasmine') + + await service.afterHook({ ...jasmineLastSpec, ctx: { test: { parent: {} } } }, {}, { passed: false, error: new Error('beforeAll failed') } as any, 'beforeAll') + + expect(cascade).not.toHaveBeenCalled() + expect(service['_hookFailReasons']).toEqual(['beforeAll failed']) + cascade.mockRestore() + }) + + it('keeps mocha on its title classification and skip cascade', async () => { + const trackEvent = vi.fn().mockResolvedValue(undefined) + cliWith({ trackEvent }) + const cascade = vi.spyOn(skipReporter, 'reportSuiteSkipped').mockResolvedValue(undefined as any) + const service = makeService('mocha') + + await service.beforeHook(mochaHook('"before each" hook for "t"'), {}, 'beforeEach') + await service.afterHook(mochaHook('"before all" hook for "t"'), {}, { passed: false, error: new Error('x') } as any, 'afterAll') + + expect(trackEvent.mock.calls.map(([state, hook]) => [state, hook])).toEqual([ + [TestFrameworkState.BEFORE_EACH, HookState.PRE], + [TestFrameworkState.BEFORE_ALL, HookState.POST], + ]) + expect(cascade).toHaveBeenCalledTimes(1) + cascade.mockRestore() + }) + }) }) describe('_isAppAutomate honors skipAppOverride', () => { From 05be96ac38a8f0001271f9c683ee5bbf16c23dc6 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 7 Oct 2026 11:37:16 +0530 Subject: [PATCH 14/24] refactor(cli): give the jasmine framework one entry point and move its session verdict to AutomateModule (SDK-7772) The reporter now feeds WdioJasmineTestFramework through trackEvent with source 'reporter'; reporter events are applied before trackEvent awaits, and TEST PRE hands the minted uuid back on args.testUuid. TestHubModule is wired once in loadModules instead of on every reporter event. service.after() passes legacy's session verdict inputs (runner result, specsRan and the three reason lists) on EXECUTE POST for jasmine only; AutomateModule computes the verdict from them, so the framework class no longer keeps verdict state. The pending-event counter and isIdle() are per instance. Co-Authored-By: Claude Opus 5.5 --- .../frameworks/wdioJasmineTestFramework.ts | 171 ++++--------- .../browserstack-service/src/cli/index.ts | 6 +- .../src/cli/modules/automateModule.ts | 58 ++++- packages/browserstack-service/src/reporter.ts | 26 +- packages/browserstack-service/src/service.ts | 13 +- .../tests/cli/index.test.ts | 37 +++ .../tests/cli/modules/automateModule.test.ts | 107 ++++++--- .../cli/wdioJasmineTestFramework.test.ts | 226 ++++++++---------- .../tests/reporter.test.ts | 88 ++++--- .../tests/service.test.ts | 63 +++++ 10 files changed, 456 insertions(+), 339 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts index 7b946760..5b983ebe 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -5,8 +5,6 @@ import type { Frameworks } from '@wdio/types' import type { HookStats, TestStats } from '@wdio/reporter' import TestFramework from './testFramework.js' -import AutomationFramework from './automationFramework.js' -import { AutomationFrameworkConstants } from './constants/automationFrameworkConstants.js' import { TestFrameworkState } from '../states/testFrameworkState.js' import { HookState } from '../states/hookState.js' import TestFrameworkInstance from '../instances/testFrameworkInstance.js' @@ -25,17 +23,6 @@ export interface JasmineSuiteContext { suiteFile?: string } -/** The Automate verdict legacy `service.after()` marks a session with. */ -export interface SessionVerdict { - status: 'passed' | 'failed' - reason?: string -} - -interface SessionFailure { - reason: string - fromHook: boolean -} - interface FailureFields { failure?: Array<{ backtrace: string[] }> failureReason?: string @@ -48,10 +35,11 @@ interface FailureFields { * Extends the BASE TestFramework, never WdioMochaTestFramework: jasmine's identity (fullName), * hook taxonomy and skip/exclude states differ from mocha's. * - * Two event sources, as on the legacy flow: - * - the WDIO reporter (`reporter.ts`) feeds Test Observability only — it is the one jasmine source that - * sees pending, focused-out and beforeAll-failed specs. Its hooks are not awaited by WDIO, so its events - * are applied through one ordered queue and sent straight to TestHubModule, never to the other modules; + * Two event sources, as on the legacy flow, both through `trackEvent`: + * - the WDIO reporter (`reporter.ts`, `source: 'reporter'`) feeds Test Observability only — it is the one + * jasmine source that sees pending, focused-out and beforeAll-failed specs. Its hooks are not awaited by + * WDIO, so its events are applied through one ordered queue and sent straight to TestHubModule, never to + * the other modules; * - the service's awaited beforeTest/afterTest drive the product modules (a11y, Automate, Percy) through * the observers, for executed specs only, with TestHub skipped. */ @@ -70,13 +58,6 @@ export default class WdioJasmineTestFramework extends TestFramework { static KEY_HOOK_FAILURE_TYPE = 'hook_failure_type' static KEY_HOOK_FAILURE_REASON = 'hook_failure_reason' - static #pendingEvents = 0 - static #serviceHookStates = new Set([ - TestFrameworkState.BEFORE_ALL, - TestFrameworkState.BEFORE_EACH, - TestFrameworkState.AFTER_EACH, - TestFrameworkState.AFTER_ALL, - ]) static #hookTypes = new Map([ ['beforeAll', 'BEFORE_ALL'], ['afterAll', 'AFTER_ALL'], @@ -97,23 +78,16 @@ export default class WdioJasmineTestFramework extends TestFramework { #openHook: TestFrameworkInstance | null = null #lastSpec: TestFrameworkInstance | null = null #queue: Promise = Promise.resolve() - // Legacy `service.ts` session verdict inputs: `_specsRan` and the runner result are per worker, - // the reason lists per session (`onReload` resets them). - #specsRan = false - #runnerFailedCount = 0 - // In the order they happened, as legacy pushed them - #sessionFailures = new Map() - static #current: WdioJasmineTestFramework | null = null + #pendingEvents = 0 constructor(testFrameworks: string[], testFrameworkVersions: Record, binSessionId: string) { super(testFrameworks, testFrameworkVersions, binSessionId) - WdioJasmineTestFramework.#current = this logger.debug('WdioJasmineTestFramework: constructed') } /** True when no reporter event is still waiting to be dispatched (the reporter's `isSynchronised`). */ - static isIdle() { - return WdioJasmineTestFramework.#pendingEvents === 0 + isIdle() { + return this.#pendingEvents === 0 } /** The WDIO `hookName` a jasmine hook runs under, as the hook-type key `getHookType` returns for mocha titles. */ @@ -121,27 +95,32 @@ export default class WdioJasmineTestFramework extends TestFramework { return WdioJasmineTestFramework.#hookTypes.get(hookName ?? '') ?? 'unknown' } + /** The state the reporter reports a hook in: BEFORE_ALL/AFTER_ALL, else NONE (each-hooks are not reported). */ + static reporterHookState(hookTitle: string | undefined): State { + return WdioJasmineTestFramework.#hookState(getHookType(String(hookTitle ?? '').toLowerCase())) ?? TestFrameworkState.NONE + } + setTestHubModule(testHub: TestHubModule | null | undefined) { this.#testHub = testHub ?? null } /** - * The service's hooks call this. INIT_TEST pins the spec's instance and uuid; TEST PRE/POST reach the - * product modules. LOG_REPORT carries nothing jasmine needs: the result comes from the reporter. + * Reporter events (`source: 'reporter'`) are applied before this returns its promise: TEST PRE writes the + * spec's uuid to `args.testUuid`. The service's hooks call this without a source: INIT_TEST pins the + * spec's instance and uuid; TEST PRE/POST reach the product modules. LOG_REPORT and the hook states carry + * nothing jasmine needs: results come from the reporter. */ async trackEvent(testFrameworkState: State, hookState: State, args: Record = {}) { + if (args.source === 'reporter') { + this.#trackReporterEvent(testFrameworkState, hookState, args) + return + } if (testFrameworkState === TestFrameworkState.LOG) { - this.onReporterLog(args.logEntry as Record) + this.#log(args.logEntry as Record) return } try { await this.#queue - if (WdioJasmineTestFramework.#serviceHookStates.has(testFrameworkState)) { - if (hookState === HookState.POST) { - this.#recordHookResult(args.result as Frameworks.TestResult | undefined) - } - return - } if (testFrameworkState !== TestFrameworkState.INIT_TEST && testFrameworkState !== TestFrameworkState.TEST) { return } @@ -157,7 +136,6 @@ export default class WdioJasmineTestFramework extends TestFramework { if (hookState === HookState.PRE) { this.#suiteTitles.set(instance, args.suiteTitle) } else { - this.#recordTestResult(args.result as Frameworks.TestResult | undefined) // afterTest hands over the raw `Jasmine__TopLevel__Suite`; the modules need the describe chain beforeTest derived moduleArgs.suiteTitle = this.#suiteTitles.get(instance) ?? args.suiteTitle this.#suiteTitles.delete(instance) @@ -171,7 +149,24 @@ export default class WdioJasmineTestFramework extends TestFramework { } } - onReporterTestStart(testStats: TestStats, context: JasmineSuiteContext) { + #trackReporterEvent(testFrameworkState: State, hookState: State, args: Record) { + const context = args.context as JasmineSuiteContext + if (testFrameworkState === TestFrameworkState.TEST) { + if (hookState === HookState.PRE) { + args.testUuid = this.#testStarted(args.testStats as TestStats, context) + } else { + this.#testEnded(args.testStats as TestStats, context) + } + } else if (testFrameworkState === TestFrameworkState.LOG) { + this.#log(args.logEntry as Record | undefined) + } else if (hookState === HookState.PRE) { + this.#hookStarted(testFrameworkState, args.hookStats as HookStats, context) + } else { + this.#hookEnded(testFrameworkState, args.hookStats as HookStats) + } + } + + #testStarted(testStats: TestStats, context: JasmineSuiteContext) { try { logger.debug(`WdioJasmineTestFramework: test start uid=${testStats.uid}`) const instance = this.#createInstance() @@ -207,7 +202,7 @@ export default class WdioJasmineTestFramework extends TestFramework { } } - onReporterTestEnd(testStats: TestStats, context: JasmineSuiteContext) { + #testEnded(testStats: TestStats, context: JasmineSuiteContext) { try { const instance = this.#specInstances.get(testStats.uid) if (!instance) { @@ -221,9 +216,6 @@ export default class WdioJasmineTestFramework extends TestFramework { const state = testStats.state const error = testStats.error - if (state === 'failed') { - this.#runnerFailedCount++ - } let result: string = state let failure: FailureFields = {} if (state === 'failed') { @@ -261,12 +253,11 @@ export default class WdioJasmineTestFramework extends TestFramework { } } - onReporterHookStart(hookStats: HookStats, context: JasmineSuiteContext) { + #hookStarted(hookFrameworkState: State, hookStats: HookStats, context: JasmineSuiteContext) { try { const title = hookStats.title const key = getHookType(String(title ?? '').toLowerCase()) - const hookFrameworkState = WdioJasmineTestFramework.#hookState(key) - if (!hookFrameworkState || context.scopes.length === 0) { + if (!WdioJasmineTestFramework.#hookState(key) || context.scopes.length === 0) { logger.debug(`WdioJasmineTestFramework: hook start not reported title=${title} scopes=${context.scopes.length}`) return } @@ -306,7 +297,7 @@ export default class WdioJasmineTestFramework extends TestFramework { } } - onReporterHookEnd(hookStats: HookStats) { + #hookEnded(hookFrameworkState: State, hookStats: HookStats) { try { const instance = this.#hookInstances.get(hookStats.uid) if (!instance) { @@ -314,7 +305,6 @@ export default class WdioJasmineTestFramework extends TestFramework { } this.#hookInstances.delete(hookStats.uid) const key = getHookType(String(hookStats.title ?? '').toLowerCase()) - const hookFrameworkState = WdioJasmineTestFramework.#hookState(key)! const state = hookStats.state const error = hookStats.error @@ -364,7 +354,7 @@ export default class WdioJasmineTestFramework extends TestFramework { * Console logs: an open beforeAll/afterAll wins, else the last-started spec, even after it ended. * HTTP command logs and screenshots name their spec (`test_run_uuid`), as legacy did; unknown uuids are dropped. */ - onReporterLog(logEntry: Record | undefined) { + #log(logEntry: Record | undefined) { try { if (!logEntry || !shouldProcessEventForTesthub('LogCreated')) { return @@ -395,73 +385,6 @@ export default class WdioJasmineTestFramework extends TestFramework { } } - /** - * The verdict legacy `service.after()` would mark the live session with: passed only when the runner - * reported no failed spec, a spec ran, and no test or hook failed, with `ignoreHooksStatus` handled as - * coded there. `null` for any other session: legacy marked a reloaded session once, in `onReload`. - * `undefined` when this worker does not run jasmine on the CLI flow. - */ - static sessionVerdict(sessionId: string, ignoreHooksStatus: boolean): SessionVerdict | null | undefined { - const current = WdioJasmineTestFramework.#current - return current ? current.#verdict(sessionId, ignoreHooksStatus) : undefined - } - - #verdict(sessionId: string, ignoreHooksStatus: boolean): SessionVerdict | null { - if (!sessionId || sessionId !== WdioJasmineTestFramework.#liveSessionId()) { - return null - } - const failures = this.#sessionFailures.get(sessionId) ?? [] - const pureTestFailReasons = failures.filter(f => !f.fromHook).map(f => f.reason) - const hookFailReasons = failures.filter(f => f.fromHook).map(f => f.reason) - const failReasons = ignoreHooksStatus ? pureTestFailReasons : failures.map(f => f.reason) - const joined = (reasons: string[]) => reasons.length > 0 ? reasons.join('\n') : undefined - - if (this.#runnerFailedCount === 0 && this.#specsRan) { - const reasons = ignoreHooksStatus ? pureTestFailReasons : failReasons - return reasons.length > 0 ? { status: 'failed', reason: joined(reasons) } : { status: 'passed' } - } - if (ignoreHooksStatus && this.#specsRan) { - const hasOnlyHookFailures = failReasons.length === 0 && hookFailReasons.length > 0 - if (hasOnlyHookFailures && pureTestFailReasons.length === 0) { - return { status: 'passed' } - } - return { status: 'failed', reason: joined(pureTestFailReasons) } - } - return { - status: 'failed', - reason: ignoreHooksStatus && pureTestFailReasons.length > 0 ? joined(pureTestFailReasons) : joined(failReasons), - } - } - - /** Legacy `service.afterTest()`: the spec ran; a failure counts toward the session. */ - #recordTestResult(result: Frameworks.TestResult | undefined) { - this.#specsRan = true - if (!result || result.passed || result.skipped) { - return - } - this.#recordFailure((result.error && result.error.message) || 'Unknown Error', false) - } - - /** Legacy `service.afterHook()`, any of the four hook types. */ - #recordHookResult(result: Frameworks.TestResult | undefined) { - if (!result || result.passed) { - return - } - this.#recordFailure((result.error && result.error.message) || 'Hook failed', true) - } - - #recordFailure(reason: string, fromHook: boolean) { - const sessionId = WdioJasmineTestFramework.#liveSessionId() - const failures = this.#sessionFailures.get(sessionId) ?? [] - failures.push({ reason, fromHook }) - this.#sessionFailures.set(sessionId, failures) - } - - static #liveSessionId(): string { - const autoInstance = AutomationFramework.getTrackedInstance() - return autoInstance ? String(AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) || '') : '' - } - #createInstance() { const trackedContext = TrackedInstance.createContext(CLIUtils.getCurrentInstanceName()) const instance = new TestFrameworkInstance( @@ -525,11 +448,11 @@ export default class WdioJasmineTestFramework extends TestFramework { } #enqueue(label: string, step: () => Promise) { - WdioJasmineTestFramework.#pendingEvents++ + this.#pendingEvents++ this.#queue = this.#queue.then(step).catch((error) => { logger.error(`WdioJasmineTestFramework: ${label} failed: ${util.format(error)}`) }).finally(() => { - WdioJasmineTestFramework.#pendingEvents-- + this.#pendingEvents-- }) } diff --git a/packages/browserstack-service/src/cli/index.ts b/packages/browserstack-service/src/cli/index.ts index 735ccc6d..56eaf74c 100644 --- a/packages/browserstack-service/src/cli/index.ts +++ b/packages/browserstack-service/src/cli/index.ts @@ -179,7 +179,11 @@ export class BrowserstackCLI { this.modules[ObservabilityModule.MODULE_NAME] = new ObservabilityModule(startBinResponse.observability) } - this.modules[TestHubModule.MODULE_NAME] = new TestHubModule(startBinResponse.testhub) + const testHubModule = new TestHubModule(startBinResponse.testhub) + this.modules[TestHubModule.MODULE_NAME] = testHubModule + if (this.testFramework instanceof WdioJasmineTestFramework) { + this.testFramework.setTestHubModule(testHubModule) + } // Custom-tag (multi Test-Case-ID) tagging rides the per-test event_json // to TestHub, so it is gated on the testhub pipeline being active. diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index c6554f50..666ce064 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -1,5 +1,4 @@ import BaseModule from './baseModule.js' -import WdioJasmineTestFramework from '../frameworks/wdioJasmineTestFramework.js' import { BrowserstackCLI } from '../index.js' import { BStackLogger } from '../cliLogger.js' import TestFramework from '../frameworks/testFramework.js' @@ -25,6 +24,20 @@ interface TestResult { reason?: string } +/** What legacy `service.after()` computed the jasmine session verdict from, as the service tracked it. */ +interface SessionVerdictInputs { + result: number + specsRan: boolean + failReasons: string[] + pureTestFailReasons: string[] + hookFailReasons: string[] +} + +interface SessionVerdict { + status: 'passed' | 'failed' + reason?: string +} + interface SessionData { lastTestName: string appliedName?: string // last name successfully PUT for this session, for de-duping @@ -361,19 +374,18 @@ export default class AutomateModule extends BaseModule { } } - async onAfterExecute() { + async onAfterExecute(args: { sessionVerdictInputs?: SessionVerdictInputs } = {}) { this.logger.debug('onAfterExecute: inside automate module after execute hook!') const userName = this.config.userName as string const accessKey = this.config.accessKey as string const testContextOptions = this.config.testContextOptions as TestContextOptions - // The binary's config echo carries `testObservabilityOptions` empty, so read the worker's own service options - const serviceOptions = BrowserstackCLI.getInstance().options as { testObservabilityOptions?: { ignoreHooksStatus?: boolean } } - const ignoreHooksStatus = serviceOptions?.testObservabilityOptions?.ignoreHooksStatus === true - const liveSessionId = this.liveSessionId() - if (WdioJasmineTestFramework.sessionVerdict(liveSessionId, ignoreHooksStatus) !== undefined) { - await this.markJasmineSessions(liveSessionId, ignoreHooksStatus) + if (args?.sessionVerdictInputs) { + // The binary's config echo carries `testObservabilityOptions` empty, so read the worker's own service options + const serviceOptions = BrowserstackCLI.getInstance().options as { testObservabilityOptions?: { ignoreHooksStatus?: boolean } } + const ignoreHooksStatus = serviceOptions?.testObservabilityOptions?.ignoreHooksStatus === true + await this.markJasmineSessions(this.liveSessionId(), this.jasmineVerdict(args.sessionVerdictInputs, ignoreHooksStatus)) this.sessionMap.clear() return } @@ -424,12 +436,37 @@ export default class AutomateModule extends BaseModule { return autoInstance ? String(AutomationFramework.getState(autoInstance, AutomationFrameworkConstants.KEY_FRAMEWORK_SESSION_ID) || '') : '' } + /** + * Legacy `service.after()`'s session status: passed only when the runner reported no failed spec, a spec + * ran, and no test or hook failed, with `ignoreHooksStatus` handled as coded there. + */ + private jasmineVerdict(inputs: SessionVerdictInputs, ignoreHooksStatus: boolean): SessionVerdict { + const { result, specsRan, failReasons, pureTestFailReasons, hookFailReasons } = inputs + const joined = (reasons: string[]) => reasons.length > 0 ? reasons.join('\n') : undefined + + if (result === 0 && specsRan) { + const reasons = ignoreHooksStatus ? pureTestFailReasons : failReasons + return reasons.length > 0 ? { status: 'failed', reason: joined(reasons) } : { status: 'passed' } + } + if (ignoreHooksStatus && specsRan) { + const hasOnlyHookFailures = failReasons.length === 0 && hookFailReasons.length > 0 + if (hasOnlyHookFailures && pureTestFailReasons.length === 0) { + return { status: 'passed' } + } + return { status: 'failed', reason: joined(pureTestFailReasons) } + } + return { + status: 'failed', + reason: ignoreHooksStatus && pureTestFailReasons.length > 0 ? joined(pureTestFailReasons) : joined(failReasons), + } + } + /** * Jasmine: legacy `service.after()` marked only the live session, with the worker's status, the last * name, and the test and hook failure reasons. A session nothing registered (a beforeAll failed before * any spec ran) is still marked, and a reloaded one is left to the mark `onReload` already sent. */ - private async markJasmineSessions(liveSessionId: string, ignoreHooksStatus: boolean) { + private async markJasmineSessions(liveSessionId: string, verdict: SessionVerdict) { const testContextOptions = this.config.testContextOptions as TestContextOptions const auth = { user: this.config.userName as string, key: this.config.accessKey as string } const sessionIds = new Set(this.sessionMap.keys()) @@ -440,8 +477,7 @@ export default class AutomateModule extends BaseModule { for (const sessionId of sessionIds) { try { await this.flushSessionName(sessionId) - const verdict = WdioJasmineTestFramework.sessionVerdict(sessionId, ignoreHooksStatus) - if (!verdict || testContextOptions.skipSessionStatus) { + if (!liveSessionId || sessionId !== liveSessionId || testContextOptions.skipSessionStatus) { continue } const name = testContextOptions.skipSessionName ? undefined : this.sessionMap.get(sessionId)?.lastTestName || undefined diff --git a/packages/browserstack-service/src/reporter.ts b/packages/browserstack-service/src/reporter.ts index c4d839e3..1345c8b5 100644 --- a/packages/browserstack-service/src/reporter.ts +++ b/packages/browserstack-service/src/reporter.ts @@ -7,7 +7,8 @@ import { BrowserstackCLI } from './cli/index.js' import { reportSkippedTest, resolveSpecFile } from './cli/skipReporter.js' import WdioJasmineTestFramework from './cli/frameworks/wdioJasmineTestFramework.js' import type { JasmineSuiteContext } from './cli/frameworks/wdioJasmineTestFramework.js' -import type TestHubModule from './cli/modules/testHubModule.js' +import { TestFrameworkState } from './cli/states/testFrameworkState.js' +import { HookState } from './cli/states/hookState.js' import * as url from 'node:url' import { v4 as uuidv4 } from 'uuid' @@ -76,7 +77,7 @@ class _TestReporter extends WDIOReporter { public async appendTestItemLog(stdLog: StdLog) { if (this.isCliJasmine()) { - this.cliJasmineFramework()?.onReporterLog(stdLog as unknown as Record) + this.cliJasmineFramework()?.trackEvent(TestFrameworkState.LOG, HookState.POST, { source: 'reporter', logEntry: stdLog }) return } if (this._currentHook.uuid && !this._currentHook.finished) { @@ -146,7 +147,7 @@ class _TestReporter extends WDIOReporter { * asynchronously, so hold the worker until it has drained. Always true on every other path. */ get isSynchronised() { - return WdioJasmineTestFramework.isIdle() + return !this.isCliJasmine() || (this.cliJasmineFramework()?.isIdle() ?? true) } /** On the CLI flow jasmine's test/hook/log events feed the framework tracker, never the legacy Listener. */ @@ -155,13 +156,8 @@ class _TestReporter extends WDIOReporter { } cliJasmineFramework() { - const cli = BrowserstackCLI.getInstance() - const framework = cli.getTestFramework() - if (!(framework instanceof WdioJasmineTestFramework)) { - return null - } - framework.setTestHubModule(cli.modules?.TestHubModule as TestHubModule | undefined) - return framework + const framework = BrowserstackCLI.getInstance().getTestFramework() + return framework instanceof WdioJasmineTestFramework ? framework : null } jasmineSuiteContext(): JasmineSuiteContext { @@ -194,7 +190,7 @@ class _TestReporter extends WDIOReporter { testStats.end ||= new Date() if (this.isCliJasmine()) { - this.cliJasmineFramework()?.onReporterTestEnd(testStats, this.jasmineSuiteContext()) + this.cliJasmineFramework()?.trackEvent(TestFrameworkState.TEST, HookState.POST, { source: 'reporter', testStats, context: this.jasmineSuiteContext() }) return } this.listener.testFinished(await this.getRunData(testStats, 'TestRunFinished')) @@ -208,7 +204,9 @@ class _TestReporter extends WDIOReporter { return } if (this.isCliJasmine()) { - const cliUuid = this.cliJasmineFramework()?.onReporterTestStart(testStats, this.jasmineSuiteContext()) + const args: Record = { source: 'reporter', testStats, context: this.jasmineSuiteContext() } + this.cliJasmineFramework()?.trackEvent(TestFrameworkState.TEST, HookState.PRE, args) + const cliUuid = args.testUuid as string | undefined if (cliUuid) { _TestReporter.currentTest.uuid = cliUuid _TestReporter.currentTest.name = testStats.title @@ -230,7 +228,7 @@ class _TestReporter extends WDIOReporter { return } if (this.isCliJasmine()) { - this.cliJasmineFramework()?.onReporterHookStart(hookStats, this.jasmineSuiteContext()) + this.cliJasmineFramework()?.trackEvent(WdioJasmineTestFramework.reporterHookState(hookStats.title), HookState.PRE, { source: 'reporter', hookStats, context: this.jasmineSuiteContext() }) return } @@ -252,7 +250,7 @@ class _TestReporter extends WDIOReporter { if (!hookStats.state && !hookStats.error) { hookStats.state = 'passed' } - this.cliJasmineFramework()?.onReporterHookEnd(hookStats) + this.cliJasmineFramework()?.trackEvent(WdioJasmineTestFramework.reporterHookState(hookStats.title), HookState.POST, { source: 'reporter', hookStats, context: this.jasmineSuiteContext() }) return } const identifier = this.getHookIdentifier(hookStats) diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 104a1a5d..4a24bd1e 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -794,7 +794,18 @@ export default class BrowserstackService implements Services.ServiceInstance { } catch (flushErr) { BStackLogger.debug(`Exception flushing deferred test finish in after(): ${util.format(flushErr)}`) } - await BrowserstackCLI.getInstance().getAutomationFramework()!.trackEvent(AutomationFrameworkState.EXECUTE, HookState.POST, {}) + const executeArgs = this._config.framework === 'jasmine' + ? { + sessionVerdictInputs: { + result, + specsRan: this._specsRan, + failReasons: [...this._failReasons], + pureTestFailReasons: [...this._pureTestFailReasons], + hookFailReasons: [...this._hookFailReasons], + } + } + : {} + await BrowserstackCLI.getInstance().getAutomationFramework()!.trackEvent(AutomationFrameworkState.EXECUTE, HookState.POST, executeArgs) } // if (setSessionStatus) { diff --git a/packages/browserstack-service/tests/cli/index.test.ts b/packages/browserstack-service/tests/cli/index.test.ts index 1e723f8b..6c8bf92c 100644 --- a/packages/browserstack-service/tests/cli/index.test.ts +++ b/packages/browserstack-service/tests/cli/index.test.ts @@ -4,6 +4,7 @@ import { BStackLogger } from '../../src/cli/cliLogger.js' import { BrowserstackCLI } from '../../src/cli/index.js' import { CLIUtils } from '../../src/cli/cliUtils.js' +import APIUtils from '../../src/cli/apiUtils.js' import TestFramework from '../../src/cli/frameworks/testFramework.js' import WdioMochaTestFramework from '../../src/cli/frameworks/wdioMochaTestFramework.js' import WdioCucumberTestFramework from '../../src/cli/frameworks/wdioCucumberTestFramework.js' @@ -157,4 +158,40 @@ describe('BrowserstackCLI bootstrap error surfacing', () => { expect(setup('WebdriverIO-unknown')).toBeNull() }) }) + + describe('loadModules TestHub wiring', () => { + const savedEnv = { ...process.env } + + afterEach(() => { + instance.testFramework = null + instance.modulesLoaded = false + process.env = { ...savedEnv } + }) + + const load = (name: string) => { + vi.spyOn(CLIUtils, 'getTestFrameworkDetail').mockReturnValue({ name, version: { [name]: '9.0.0' } }) + vi.spyOn(APIUtils, 'updateURLSForGRR').mockImplementation(() => {}) + vi.spyOn(CLIUtils, 'getAutomationFrameworkDetail').mockReturnValue({ name: 'WebdriverIO', version: { WebdriverIO: '9.0.0' } }) + instance.modulesLoaded = false + instance.loadModules({ + binSessionId: 'b1', + config: '{}', + testhub: { jwt: 'jwt', buildHashedId: 'build' } + } as any) + } + + it('hands the jasmine framework its TestHubModule once, when the module is built', () => { + const setTestHubModule = vi.spyOn(WdioJasmineTestFramework.prototype, 'setTestHubModule') + load('WebdriverIO-jasmine') + expect(setTestHubModule).toHaveBeenCalledTimes(1) + expect(setTestHubModule.mock.calls[0][0]).toBe(instance.modules.TestHubModule) + }) + + it('leaves the mocha framework untouched', () => { + const setTestHubModule = vi.spyOn(WdioJasmineTestFramework.prototype, 'setTestHubModule') + load('WebdriverIO-mocha') + expect(instance.modules.TestHubModule).toBeDefined() + expect(setTestHubModule).not.toHaveBeenCalled() + }) + }) }) diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index 4d1fbf45..3d75673f 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -9,7 +9,6 @@ import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/te import { isBrowserstackSession } from '../../../src/util.js' import PerformanceTester from '../../../src/instrumentation/performance/performance-tester.js' import { _fetch as fetch } from '../../../src/fetchWrapper.js' -import WdioJasmineTestFramework from '../../../src/cli/frameworks/wdioJasmineTestFramework.js' import { BrowserstackCLI } from '../../../src/cli/index.js' import type { Options } from '@wdio/types' @@ -29,13 +28,6 @@ vi.mock('../../../src/cli/index.js', () => ({ } })) -// undefined = this worker is not jasmine on the CLI flow, so the existing aggregation applies -vi.mock('../../../src/cli/frameworks/wdioJasmineTestFramework.js', () => ({ - default: { - sessionVerdict: vi.fn(() => undefined) - } -})) - vi.mock('../../../src/cli/frameworks/automationFramework.js', () => ({ default: { getTrackedInstance: vi.fn(), @@ -1024,6 +1016,11 @@ describe('AutomateModule preferScenarioName', () => { describe('AutomateModule — jasmine session verdict', () => { let automateModule: AutomateModule const putBodies = () => vi.mocked(fetch).mock.calls.map(([url, opts]) => [String(url).split('/sessions/')[1], JSON.parse((opts as { body: string }).body)]) + const inputs = (overrides: Record = {}) => ({ + result: 0, specsRan: true, failReasons: [], pureTestFailReasons: [], hookFailReasons: [], ...overrides + }) + const verdict = (overrides: Record, ignoreHooksStatus: boolean) => + (automateModule as any).jasmineVerdict(inputs(overrides), ignoreHooksStatus) beforeEach(() => { vi.clearAllMocks() @@ -1040,17 +1037,61 @@ describe('AutomateModule — jasmine session verdict', () => { } as any }) - afterEach(() => { - vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReset().mockReturnValue(undefined) + describe('jasmineVerdict (legacy service.after())', () => { + it('passes a clean run with no reason, whatever ignoreHooksStatus says', () => { + expect(verdict({}, false)).toEqual({ status: 'passed' }) + expect(verdict({}, true)).toEqual({ status: 'passed' }) + }) + + it('fails a runner-clean run on any recorded reason, joined in order', () => { + expect(verdict({ failReasons: ['beforeAll hook failure', 'boom'], pureTestFailReasons: ['boom'], hookFailReasons: ['beforeAll hook failure'] }, false)) + .toEqual({ status: 'failed', reason: 'beforeAll hook failure\nboom' }) + }) + + it('under ignoreHooksStatus judges a runner-clean run on its pure test failures only', () => { + expect(verdict({ hookFailReasons: ['afterAll hook failure'] }, true)).toEqual({ status: 'passed' }) + expect(verdict({ failReasons: ['boom'], pureTestFailReasons: ['boom'], hookFailReasons: ['afterAll hook failure'] }, true)) + .toEqual({ status: 'failed', reason: 'boom' }) + }) + + it('under ignoreHooksStatus passes a runner-failed run whose only failures are hooks', () => { + expect(verdict({ result: 2, hookFailReasons: ['beforeEach hook failure'] }, true)).toEqual({ status: 'passed' }) + }) + + it('under ignoreHooksStatus fails a runner-failed run with pure failures, on those reasons', () => { + expect(verdict({ result: 1, failReasons: ['boom'], pureTestFailReasons: ['boom'], hookFailReasons: ['hook'] }, true)) + .toEqual({ status: 'failed', reason: 'boom' }) + }) + + it('under ignoreHooksStatus fails a runner-failed run with no recorded failure, with no reason', () => { + expect(verdict({ result: 1 }, true)).toEqual({ status: 'failed', reason: undefined }) + }) + + it('fails a runner-failed run with every recorded reason when hooks count', () => { + expect(verdict({ result: 1, failReasons: ['beforeEach hook failure', 'boom'], pureTestFailReasons: ['boom'], hookFailReasons: ['beforeEach hook failure'] }, false)) + .toEqual({ status: 'failed', reason: 'beforeEach hook failure\nboom' }) + expect(verdict({ result: 1 }, false)).toEqual({ status: 'failed', reason: undefined }) + }) + + it('fails a worker where no spec ran, preferring pure reasons under ignoreHooksStatus', () => { + expect(verdict({ specsRan: false }, false)).toEqual({ status: 'failed', reason: undefined }) + expect(verdict({ specsRan: false, failReasons: ['beforeAll hook failure'], hookFailReasons: ['beforeAll hook failure'] }, false)) + .toEqual({ status: 'failed', reason: 'beforeAll hook failure' }) + expect(verdict({ specsRan: false, hookFailReasons: ['beforeAll hook failure'] }, true)).toEqual({ status: 'failed', reason: undefined }) + expect(verdict({ specsRan: false, failReasons: ['boom'], pureTestFailReasons: ['boom'] }, true)).toEqual({ status: 'failed', reason: 'boom' }) + }) + + it('reproduces the CP0 hookfail verdict: failed, every hook failure in the order it happened', () => { + const hookFailures = ['beforeAll hook failure', 'beforeEach hook failure', 'beforeEach hook failure', 'afterEach hook failure', 'afterEach hook failure'] + expect(verdict({ result: 6, failReasons: hookFailures, hookFailReasons: hookFailures }, false)) + .toEqual({ status: 'failed', reason: hookFailures.join('\n') }) + }) }) - it('marks the live session with the framework verdict, the last name and the joined reasons', async () => { - vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockImplementation((id) => id === 'live' - ? { status: 'failed', reason: 'beforeAll hook failure\nafterEach hook failure' } - : null) - ;(automateModule as any).sessionMap.set('live', { lastTestName: 'Hookfail afterEach suite', appliedName: 'Hookfail afterEach suite', testResults: new Map(), scenariosRan: 0 }) + it('marks the live session with the verdict, the last name and the joined reasons', async () => { + (automateModule as any).sessionMap.set('live', { lastTestName: 'Hookfail afterEach suite', appliedName: 'Hookfail afterEach suite', testResults: new Map(), scenariosRan: 0 }) - await automateModule.onAfterExecute() + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1, failReasons: ['beforeAll hook failure', 'afterEach hook failure'] }) }) expect(putBodies()).toEqual([ ['live.json', { status: 'failed', name: 'Hookfail afterEach suite', reason: 'beforeAll hook failure\nafterEach hook failure' }] @@ -1058,18 +1099,15 @@ describe('AutomateModule — jasmine session verdict', () => { }) it('marks a live session no spec registered, without a name', async () => { - vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReturnValue({ status: 'failed', reason: 'beforeAll hook failure' }) - - await automateModule.onAfterExecute() + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1, specsRan: false, failReasons: ['beforeAll hook failure'] }) }) expect(putBodies()).toEqual([['live.json', { status: 'failed', reason: 'beforeAll hook failure' }]]) }) it('leaves a reloaded session to its onReload mark, but still names it', async () => { - vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockImplementation((id) => id === 'live' ? { status: 'passed' } : null) - ;(automateModule as any).sessionMap.set('old', { lastTestName: 'Suite A', testResults: new Map(), scenariosRan: 0 }) + (automateModule as any).sessionMap.set('old', { lastTestName: 'Suite A', testResults: new Map(), scenariosRan: 0 }) - await automateModule.onAfterExecute() + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs() }) expect(putBodies()).toEqual([ ['old.json', { name: 'Suite A' }], @@ -1077,32 +1115,39 @@ describe('AutomateModule — jasmine session verdict', () => { ]) }) - it('passes ignoreHooksStatus from the worker\'s service options to the verdict', async () => { - vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReturnValue({ status: 'passed' }) + it('marks no session status when no session is live', async () => { + vi.mocked(AutomationFramework.getState).mockReturnValue('' as any) + ;(automateModule as any).sessionMap.set('old', { lastTestName: 'Suite A', testResults: new Map(), scenariosRan: 0 }) + + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1 }) }) + + expect(putBodies()).toEqual([['old.json', { name: 'Suite A' }]]) + }) + + it('reads ignoreHooksStatus from the worker\'s service options', async () => { vi.mocked(BrowserstackCLI.getInstance).mockReturnValueOnce({ options: { testObservabilityOptions: { ignoreHooksStatus: true } } } as any) - await automateModule.onAfterExecute() + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1, hookFailReasons: ['beforeEach hook failure'] }) }) - expect(WdioJasmineTestFramework.sessionVerdict).toHaveBeenCalledWith('live', true) + expect(putBodies()).toEqual([['live.json', { status: 'passed' }]]) }) it('honours skipSessionStatus and skipSessionName', async () => { - vi.mocked(WdioJasmineTestFramework.sessionVerdict).mockReturnValue({ status: 'failed', reason: 'x' }) - ;(automateModule as any).sessionMap.set('live', { lastTestName: 'Suite', testResults: new Map(), scenariosRan: 0 }) + (automateModule as any).sessionMap.set('live', { lastTestName: 'Suite', testResults: new Map(), scenariosRan: 0 }) ;(automateModule.config as any).testContextOptions = { skipSessionName: true, skipSessionStatus: true } - await automateModule.onAfterExecute() + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1, failReasons: ['x'] }) }) expect(fetch).not.toHaveBeenCalled() }) - it('keeps the existing per-test aggregation when the worker is not jasmine', async () => { + it('keeps the existing per-test aggregation when no verdict inputs arrive (mocha, cucumber)', async () => { (automateModule as any).sessionMap.set('live', { lastTestName: 'Suite', appliedName: 'Suite', scenariosRan: 0, testResults: new Map([['t', { testName: 'Suite', status: 'failed', reason: 'boom' }]]) }) - await automateModule.onAfterExecute() + await automateModule.onAfterExecute({}) expect(putBodies()).toEqual([['live.json', { status: 'failed', reason: 'boom' }]]) }) diff --git a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts index dd1e3b9b..25978192 100644 --- a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts @@ -8,7 +8,6 @@ import { TestFrameworkState } from '../../src/cli/states/testFrameworkState.js' import { HookState } from '../../src/cli/states/hookState.js' import type TestFrameworkInstance from '../../src/cli/instances/testFrameworkInstance.js' import TestHubModule from '../../src/cli/modules/testHubModule.js' -import AutomationFramework from '../../src/cli/frameworks/automationFramework.js' vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) @@ -60,6 +59,23 @@ describe('WdioJasmineTestFramework', () => { let testHub: Record> const drain = () => framework.trackEvent(TestFrameworkState.LOG_REPORT, HookState.POST, {}) + // The reporter does not await trackEvent: its events are applied before the call returns. + const reporterTestStart = (testStats: unknown, ctx: unknown) => { + const args: Record = { source: 'reporter', testStats, context: ctx } + framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, args) + return args.testUuid as string | undefined + } + const reporterTestEnd = (testStats: unknown, ctx: unknown) => { + framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { source: 'reporter', testStats, context: ctx }) + } + const reporterHook = (hookState: State, hookStats: { title: string }, ctx: unknown) => { + framework.trackEvent(WdioJasmineTestFramework.reporterHookState(hookStats.title), hookState, { source: 'reporter', hookStats, context: ctx }) + } + const reporterHookStart = (hookStats: { title: string }, ctx: unknown) => reporterHook(HookState.PRE, hookStats, ctx) + const reporterHookEnd = (hookStats: { title: string }, ctx: unknown = context()) => reporterHook(HookState.POST, hookStats, ctx) + const reporterLog = (logEntry: Record) => { + framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { source: 'reporter', logEntry }) + } beforeEach(() => { process.env.BROWSERSTACK_OBSERVABILITY = 'true' @@ -92,7 +108,7 @@ describe('WdioJasmineTestFramework', () => { }) it('sends legacy identity for a spec, char-for-char with the CP0 TestRun', async () => { - framework.onReporterTestStart(testStats() as any, context()) + reporterTestStart(testStats(), context()) await drain() expect(dispatches).toHaveLength(1) @@ -116,15 +132,15 @@ describe('WdioJasmineTestFramework', () => { }) it('keeps @tags from describes and the spec, sigil included', async () => { - framework.onReporterTestStart(testStats({ title: 'tagged @smoke', fullTitle: 'Outer @regression tagged @smoke' }) as any, context(['Outer @regression'])) + reporterTestStart(testStats({ title: 'tagged @smoke', fullTitle: 'Outer @regression tagged @smoke' }), context(['Outer @regression'])) await drain() expect(dispatches[0].data.test_tags).toEqual(['@regression', '@smoke']) }) it('finishes a passed spec on the same uuid with reporter timing and the result timestamp', async () => { const stats = testStats() - framework.onReporterTestStart(stats as any, context()) - framework.onReporterTestEnd({ ...stats, state: 'passed', end: new Date('2026-09-25T15:34:16.645Z'), _duration: 4386 } as any, context()) + reporterTestStart(stats, context()) + reporterTestEnd({ ...stats, state: 'passed', end: new Date('2026-09-25T15:34:16.645Z'), _duration: 4386 }, context()) await drain() expect(dispatches.map(d => [d.state, d.hook])).toEqual([[TestFrameworkState.TEST, HookState.PRE], [TestFrameworkState.TEST, HookState.POST]]) @@ -140,9 +156,9 @@ describe('WdioJasmineTestFramework', () => { }) it('reports a pending spec as Start + Finish(skipped)', async () => { - framework.onReporterTestStart(testStats({ uid: 'xit0', title: 'xit skipped test', fullTitle: 'Pending suite xit skipped test' }) as any, context(['Pending suite'])) + reporterTestStart(testStats({ uid: 'xit0', title: 'xit skipped test', fullTitle: 'Pending suite xit skipped test' }), context(['Pending suite'])) // @wdio/reporter replaces the TestStats object on test:pending; only the uid carries over - framework.onReporterTestEnd(testStats({ uid: 'xit0', title: 'xit skipped test', fullTitle: 'Pending suite xit skipped test', state: 'skipped', end: new Date() }) as any, context(['Pending suite'])) + reporterTestEnd(testStats({ uid: 'xit0', title: 'xit skipped test', fullTitle: 'Pending suite xit skipped test', state: 'skipped', end: new Date() }), context(['Pending suite'])) await drain() expect(dispatches).toHaveLength(2) @@ -153,8 +169,8 @@ describe('WdioJasmineTestFramework', () => { it('keeps jasmine\'s exact reason on a beforeAll-failed child', async () => { const error = { message: BEFORE_ALL_REASON, stack: '' } - framework.onReporterTestStart(testStats({ uid: 'child0' }) as any, context()) - framework.onReporterTestEnd(testStats({ uid: 'child0', state: 'failed', error, end: new Date() }) as any, context()) + reporterTestStart(testStats({ uid: 'child0' }), context()) + reporterTestEnd(testStats({ uid: 'child0', state: 'failed', error, end: new Date() }), context()) await drain() const finish = dispatches[1].data @@ -165,17 +181,17 @@ describe('WdioJasmineTestFramework', () => { }) it('maps an AssertionError message to failure_type AssertionError', async () => { - framework.onReporterTestStart(testStats() as any, context()) - framework.onReporterTestEnd(testStats({ state: 'failed', error: { message: 'AssertionError: nope', stack: 'at x' } }) as any, context()) + reporterTestStart(testStats(), context()) + reporterTestEnd(testStats({ state: 'failed', error: { message: 'AssertionError: nope', stack: 'at x' } }), context()) await drain() expect(dispatches[1].data.test_failure_type).toBe('AssertionError') }) it('gives overlapping specs distinct instances and closes each on its own uuid', async () => { - framework.onReporterTestStart(testStats({ uid: 'a' }) as any, context()) - framework.onReporterTestStart(testStats({ uid: 'b', title: 'b', fullTitle: 'Nested outer b' }) as any, context()) - framework.onReporterTestEnd(testStats({ uid: 'a', state: 'passed', end: new Date() }) as any, context()) - framework.onReporterTestEnd(testStats({ uid: 'b', title: 'b', fullTitle: 'Nested outer b', state: 'passed', end: new Date() }) as any, context()) + reporterTestStart(testStats({ uid: 'a' }), context()) + reporterTestStart(testStats({ uid: 'b', title: 'b', fullTitle: 'Nested outer b' }), context()) + reporterTestEnd(testStats({ uid: 'a', state: 'passed', end: new Date() }), context()) + reporterTestEnd(testStats({ uid: 'b', title: 'b', fullTitle: 'Nested outer b', state: 'passed', end: new Date() }), context()) await drain() const uuidOf = (i: number) => dispatches[i].data.test_uuid @@ -187,11 +203,11 @@ describe('WdioJasmineTestFramework', () => { it('reports beforeAll/afterAll with legacy hook identity and no test linkage', async () => { const before = hookStats('"before all" hook') - framework.onReporterHookStart(before as any, context()) - framework.onReporterHookEnd({ ...before, state: 'passed', end: new Date('2026-09-25T15:34:12.257Z'), _duration: 4087 } as any) + reporterHookStart(before, context()) + reporterHookEnd({ ...before, state: 'passed', end: new Date('2026-09-25T15:34:12.257Z'), _duration: 4087 }) const after = hookStats('"after all" hook') - framework.onReporterHookStart(after as any, context(['Nested outer', 'Nested middle'])) - framework.onReporterHookEnd({ ...after, state: 'passed', end: new Date() } as any) + reporterHookStart(after, context(['Nested outer', 'Nested middle'])) + reporterHookEnd({ ...after, state: 'passed', end: new Date() }) await drain() expect(dispatches.map(d => [d.state, d.hook])).toEqual([ @@ -228,8 +244,8 @@ describe('WdioJasmineTestFramework', () => { it('carries failure fields on a failed beforeAll', async () => { const before = hookStats('"before all" hook') - framework.onReporterHookStart(before as any, context()) - framework.onReporterHookEnd({ ...before, state: 'failed', error: { message: 'boom', stack: 'Error: boom' }, end: new Date() } as any) + reporterHookStart(before, context()) + reporterHookEnd({ ...before, state: 'failed', error: { message: 'boom', stack: 'Error: boom' }, end: new Date() }) await drain() const finished = (dispatches[1].data.test_hooks_finished as Record[]>).BEFORE_ALL[0] expect(finished).toMatchObject({ @@ -241,8 +257,8 @@ describe('WdioJasmineTestFramework', () => { }) it('never reports each-hooks, or a hook outside any describe', async () => { - framework.onReporterHookStart(hookStats('"before each" hook') as any, context()) - framework.onReporterHookStart(hookStats('"before all" hook') as any, context([])) + reporterHookStart(hookStats('"before each" hook'), context()) + reporterHookStart(hookStats('"before all" hook'), context([])) await drain() expect(dispatches).toHaveLength(0) }) @@ -252,10 +268,10 @@ describe('WdioJasmineTestFramework', () => { process.env.BROWSERSTACK_ACCESSIBILITY = 'true' try { const before = hookStats('"before all" hook') - framework.onReporterHookStart(before as any, context()) - framework.onReporterHookEnd({ ...before, state: 'passed', end: new Date() } as any) - framework.onReporterTestStart(testStats() as any, context()) - framework.onReporterLog({ level: 'INFO', message: 'hi', timestamp: 't', kind: 'TEST_LOG' }) + reporterHookStart(before, context()) + reporterHookEnd({ ...before, state: 'passed', end: new Date() }) + reporterTestStart(testStats(), context()) + reporterLog({ level: 'INFO', message: 'hi', timestamp: 't', kind: 'TEST_LOG' }) await drain() expect(dispatches.map(d => d.state)).toEqual([TestFrameworkState.TEST]) expect(logSends).toHaveLength(0) @@ -266,8 +282,8 @@ describe('WdioJasmineTestFramework', () => { it('sends reporter events to TestHub only, never through the module observers', async () => { const stats = testStats() - framework.onReporterTestStart(stats as any, context()) - framework.onReporterTestEnd({ ...stats, state: 'passed', end: new Date() } as any, context()) + reporterTestStart(stats, context()) + reporterTestEnd({ ...stats, state: 'passed', end: new Date() }, context()) await drain() expect(testHub.sendTestFrameworkEvent).toHaveBeenCalledTimes(2) expect(sessionEvents).toBe(1) @@ -276,7 +292,7 @@ describe('WdioJasmineTestFramework', () => { it('drives the modules from the service hooks on the reporter\'s instance, with TestHub skipped', async () => { const spec = { description: 'outer passing test', fullName: 'Nested outer outer passing test' } - framework.onReporterTestStart(testStats() as any, context()) + reporterTestStart(testStats(), context()) await framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test: spec }) const uuid = dispatches[0].data.test_uuid expect(TestFramework.getState(TestFramework.getTrackedInstance(), 'test_uuid')).toBe(uuid) @@ -297,8 +313,8 @@ describe('WdioJasmineTestFramework', () => { }) it('never shows a spec the reporter saw but the service did not (pending, excluded, beforeAll-failed) to the modules', async () => { - framework.onReporterTestStart(testStats({ uid: 'x' }) as any, context()) - framework.onReporterTestEnd(testStats({ uid: 'x', state: 'skipped', end: new Date() }) as any, context()) + reporterTestStart(testStats({ uid: 'x' }), context()) + reporterTestEnd(testStats({ uid: 'x', state: 'skipped', end: new Date() }), context()) await drain() expect(dispatches).toHaveLength(2) expect(moduleDispatches).toHaveLength(0) @@ -316,13 +332,13 @@ describe('WdioJasmineTestFramework', () => { }) it('sends an open all-hook\'s logs in the hook state, else the last-started spec\'s in the test state', async () => { - framework.onReporterTestStart(testStats() as any, context()) - framework.onReporterLog({ level: 'INFO', message: 'in test', timestamp: 't1', kind: 'TEST_LOG' }) + reporterTestStart(testStats(), context()) + reporterLog({ level: 'INFO', message: 'in test', timestamp: 't1', kind: 'TEST_LOG' }) const after = hookStats('"after all" hook') - framework.onReporterHookStart(after as any, context()) - framework.onReporterLog({ level: 'INFO', message: 'in hook', timestamp: 't2', kind: 'TEST_LOG' }) - framework.onReporterHookEnd({ ...after, state: 'passed', end: new Date() } as any) - framework.onReporterLog({ level: 'INFO', message: 'after hook', timestamp: 't3', kind: 'TEST_LOG' }) + reporterHookStart(after, context()) + reporterLog({ level: 'INFO', message: 'in hook', timestamp: 't2', kind: 'TEST_LOG' }) + reporterHookEnd({ ...after, state: 'passed', end: new Date() }) + reporterLog({ level: 'INFO', message: 'after hook', timestamp: 't3', kind: 'TEST_LOG' }) await drain() expect(logSends.map(l => l.state)).toEqual(['TestFrameworkState.TEST', 'TestFrameworkState.AFTER_ALL', 'TestFrameworkState.TEST']) @@ -335,7 +351,7 @@ describe('WdioJasmineTestFramework', () => { }) it('keeps a screenshot entry\'s kind on the log path', async () => { - framework.onReporterTestStart(testStats() as any, context()) + reporterTestStart(testStats(), context()) await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: 't', level: 'INFO' } }) await drain() expect(logSends[0].entries[0].kind).toBe('TEST_SCREENSHOT') @@ -343,10 +359,10 @@ describe('WdioJasmineTestFramework', () => { it('sends an HTTP command log to the spec it names, even inside an all-hook or after the spec ended', async () => { const first = testStats() - const uuid = framework.onReporterTestStart(first as any, context()) - framework.onReporterTestEnd({ ...first, state: 'passed', end: new Date() } as any, context()) + const uuid = reporterTestStart(first, context()) + reporterTestEnd({ ...first, state: 'passed', end: new Date() }, context()) const after = hookStats('"after all" hook') - framework.onReporterHookStart(after as any, context()) + reporterHookStart(after, context()) const message = JSON.stringify({ path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } }) await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'HTTP', message, timestamp: 't', test_run_uuid: uuid } }) await drain() @@ -360,7 +376,7 @@ describe('WdioJasmineTestFramework', () => { }) it('drops a named-spec log whose uuid this worker never minted', async () => { - framework.onReporterTestStart(testStats() as any, context()) + reporterTestStart(testStats(), context()) await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'HTTP', message: '{}', timestamp: 't', test_run_uuid: 'not-ours' } }) await drain() expect(logSends).toHaveLength(0) @@ -381,102 +397,54 @@ describe('WdioJasmineTestFramework', () => { }) it('is not idle until queued events are dispatched', async () => { - framework.onReporterTestStart(testStats() as any, context()) - expect(WdioJasmineTestFramework.isIdle()).toBe(false) + reporterTestStart(testStats(), context()) + expect(framework.isIdle()).toBe(false) await drain() - expect(WdioJasmineTestFramework.isIdle()).toBe(true) + expect(framework.isIdle()).toBe(true) }) - it('logs and continues when an observer throws', async () => { - testHub.sendTestFrameworkEvent.mockRejectedValueOnce(new Error('send blew up')) - const stats = testStats() - framework.onReporterTestStart(stats as any, context()) - framework.onReporterTestEnd({ ...stats, state: 'passed', end: new Date() } as any, context()) + it('counts pending events per framework instance', async () => { + const other = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin-session') + reporterTestStart(testStats(), context()) + expect(framework.isIdle()).toBe(false) + expect(other.isIdle()).toBe(true) await drain() - expect(dispatches.map(d => d.hook)).toEqual([HookState.POST]) - expect(WdioJasmineTestFramework.isIdle()).toBe(true) }) - describe('session verdict (legacy service.after())', () => { - let liveSession = 'live' - const spec = (title: string, overrides: Record = {}) => testStats({ uid: title, title, fullTitle: `Suite ${title}`, ...overrides }) - const reporterEnd = (stats: ReturnType, state: string) => - framework.onReporterTestEnd({ ...stats, state, end: new Date() } as any, context()) - const serviceTest = (title: string, result: Record) => - framework.trackEvent(TestFrameworkState.TEST, HookState.POST, { test: { fullName: `Suite ${title}`, description: title }, result }) - const serviceHook = (state: State, result: Record) => framework.trackEvent(state, HookState.POST, { test: {}, result }) - const fail = (message: string) => ({ passed: false, error: new Error(message) }) - - beforeEach(() => { - liveSession = 'live' - vi.spyOn(AutomationFramework, 'getTrackedInstance').mockReturnValue({} as any) - vi.spyOn(AutomationFramework, 'getState').mockImplementation(() => liveSession) - }) - - it('reproduces the CP0 hookfail verdict: failed, every hook failure in the order it happened', async () => { - await serviceHook(TestFrameworkState.BEFORE_ALL, fail('beforeAll hook failure')) - for (const title of ['beforeAll child one', 'beforeAll child two', 'beforeEach child one', 'beforeEach child two']) { - if (title.startsWith('beforeEach')) { - await serviceHook(TestFrameworkState.BEFORE_EACH, fail('beforeEach hook failure')) - } - const stats = spec(title) - framework.onReporterTestStart(stats as any, context()) - reporterEnd(stats, 'failed') - } - for (const title of ['afterEach child one', 'afterEach child two']) { - const stats = spec(title) - framework.onReporterTestStart(stats as any, context()) - await serviceTest(title, { passed: true }) - await serviceHook(TestFrameworkState.AFTER_EACH, fail('afterEach hook failure')) - reporterEnd(stats, 'failed') - } - await drain() - - expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ - status: 'failed', - reason: 'beforeAll hook failure\nbeforeEach hook failure\nbeforeEach hook failure\nafterEach hook failure\nafterEach hook failure' - }) - }) - - it('fails a session where no spec ran, even with no failure recorded', async () => { - expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'failed', reason: undefined }) - }) - - it('passes when specs ran and nothing failed, with no reason', async () => { - const stats = spec('ok') - framework.onReporterTestStart(stats as any, context()) - await serviceTest('ok', { passed: true }) - reporterEnd(stats, 'passed') - await drain() - expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'passed' }) - }) + it('mints a reporter spec before trackEvent returns, so the uuid and the service lookup are immediate', async () => { + const args: Record = { source: 'reporter', testStats: testStats(), context: context() } + const pending = framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, args) + const uuid = args.testUuid + expect(uuid).toEqual(expect.any(String)) + expect(TestFramework.getState(TestFramework.getTrackedInstance(), 'test_uuid')).toBe(uuid) + expect(dispatches).toHaveLength(0) - it('does not count a pending() spec as a failure', async () => { - await serviceTest('pending', { passed: false, skipped: true }) - expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'passed' }) - }) + await framework.trackEvent(TestFrameworkState.INIT_TEST, HookState.PRE, { test: { description: 'outer passing test', fullName: 'Nested outer outer passing test' } }) + await pending + expect(process.env.TEST_ANALYTICS_ID).toBe(uuid) + expect(dispatches[0].data.test_uuid).toBe(uuid) + }) - it('under ignoreHooksStatus passes a run whose only failures are hooks, and keeps test failures', async () => { - await serviceTest('ok', { passed: true }) - await serviceHook(TestFrameworkState.BEFORE_EACH, fail('beforeEach hook failure')) - const child = spec('child') - framework.onReporterTestStart(child as any, context()) - reporterEnd(child, 'failed') - await drain() - expect(WdioJasmineTestFramework.sessionVerdict('live', true)).toEqual({ status: 'passed' }) - expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toEqual({ status: 'failed', reason: 'beforeEach hook failure' }) + it('leaves no uuid on the args when minting fails, and logs instead of throwing', async () => { + const args: Record = { source: 'reporter', testStats: undefined, context: context() } + await expect(framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, args)).resolves.toBeUndefined() + expect(args.testUuid).toBeUndefined() + expect(framework.isIdle()).toBe(true) + }) - await serviceTest('bad', fail('boom')) - expect(WdioJasmineTestFramework.sessionVerdict('live', true)).toEqual({ status: 'failed', reason: 'boom' }) - }) + it('reports a hook in BEFORE_ALL/AFTER_ALL and every other hook in NONE', () => { + expect(['"before all" hook', '"after all" hook', '"before each" hook', '"after each" hook', undefined].map(t => WdioJasmineTestFramework.reporterHookState(t))) + .toEqual([TestFrameworkState.BEFORE_ALL, TestFrameworkState.AFTER_ALL, TestFrameworkState.NONE, TestFrameworkState.NONE, TestFrameworkState.NONE]) + }) - it('keeps failures per session and gives no verdict for a session that is no longer live', async () => { - await serviceTest('first', fail('before reload')) - liveSession = 'reloaded' - await serviceTest('second', { passed: true }) - expect(WdioJasmineTestFramework.sessionVerdict('reloaded', false)).toEqual({ status: 'passed' }) - expect(WdioJasmineTestFramework.sessionVerdict('live', false)).toBeNull() - }) + it('logs and continues when an observer throws', async () => { + testHub.sendTestFrameworkEvent.mockRejectedValueOnce(new Error('send blew up')) + const stats = testStats() + reporterTestStart(stats, context()) + reporterTestEnd({ ...stats, state: 'passed', end: new Date() }, context()) + await drain() + expect(dispatches.map(d => d.hook)).toEqual([HookState.POST]) + expect(framework.isIdle()).toBe(true) }) // The class calls these TestHubModule methods directly; renaming or removing one must fail here. diff --git a/packages/browserstack-service/tests/reporter.test.ts b/packages/browserstack-service/tests/reporter.test.ts index 2714f675..5f8a169a 100644 --- a/packages/browserstack-service/tests/reporter.test.ts +++ b/packages/browserstack-service/tests/reporter.test.ts @@ -6,6 +6,8 @@ import type { StdLog } from '../src/index.js' import TestReporter from '../src/reporter.js' import { BrowserstackCLI } from '../src/cli/index.js' import WdioJasmineTestFramework from '../src/cli/frameworks/wdioJasmineTestFramework.js' +import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' +import { HookState } from '../src/cli/states/hookState.js' import * as utils from '../src/util.js' import * as bstackLogger from '../src/bstackLogger.js' @@ -385,7 +387,8 @@ describe('test-reporter', () => { let framework: WdioJasmineTestFramework let getInstanceSpy: ReturnType | undefined let getGitMetaDataSpy: ReturnType - let isIdleSpy: ReturnType | undefined + // the reporter-sourced trackEvent calls, as [state, hookState, args] + const reporterCalls = () => vi.mocked(framework.trackEvent).mock.calls.filter(([, , args]) => (args as Record)?.source === 'reporter') const setCli = (running: boolean, testFramework: unknown) => { getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ @@ -398,14 +401,11 @@ describe('test-reporter', () => { vi.spyOn(utils, 'getCloudProvider').mockReturnValue('browserstack') getGitMetaDataSpy = vi.spyOn(utils, 'getGitMetaData').mockResolvedValue(undefined as any) framework = new WdioJasmineTestFramework(['WebdriverIO-jasmine'], { 'WebdriverIO-jasmine': '9.39.0' }, 'bin') - for (const m of ['onReporterTestStart', 'onReporterTestEnd', 'onReporterHookStart', 'onReporterHookEnd', 'onReporterLog'] as const) { - vi.spyOn(framework, m) - } - vi.mocked(framework.onReporterTestStart).mockReturnValue('cli-uuid') - vi.mocked(framework.onReporterTestEnd).mockReturnValue(undefined) - vi.mocked(framework.onReporterHookStart).mockReturnValue(undefined) - vi.mocked(framework.onReporterHookEnd).mockReturnValue(undefined) - vi.mocked(framework.onReporterLog).mockReturnValue(undefined) + vi.spyOn(framework, 'trackEvent').mockImplementation(async (state, hookState, args = {}) => { + if (state === TestFrameworkState.TEST && hookState === HookState.PRE) { + args.testUuid = 'cli-uuid' + } + }) reporter = new TestReporter({}) await reporter.onRunnerStart(jasmineRunnerConfig as any) @@ -422,8 +422,6 @@ describe('test-reporter', () => { afterEach(() => { getInstanceSpy?.mockRestore() getInstanceSpy = undefined - isIdleSpy?.mockRestore() - isIdleSpy = undefined getGitMetaDataSpy.mockRestore() for (const spy of Object.values(listener)) { spy.mockRestore() @@ -441,11 +439,14 @@ describe('test-reporter', () => { await reporter.onTestEnd({ ...stats, state: 'passed' } as any) await reporter.appendTestItemLog(logEntry()) - expect(framework.onReporterHookStart).toHaveBeenCalledTimes(1) - expect(framework.onReporterHookEnd).toHaveBeenCalledTimes(1) - expect(framework.onReporterTestStart).toHaveBeenCalledTimes(1) - expect(framework.onReporterTestEnd).toHaveBeenCalledTimes(1) - expect(framework.onReporterLog).toHaveBeenCalledTimes(1) + expect(reporterCalls().map(([state, hookState]) => [state, hookState])).toEqual([ + [TestFrameworkState.BEFORE_ALL, HookState.PRE], + [TestFrameworkState.BEFORE_ALL, HookState.POST], + [TestFrameworkState.TEST, HookState.PRE], + [TestFrameworkState.TEST, HookState.POST], + [TestFrameworkState.LOG, HookState.POST], + ]) + expect(framework.trackEvent).toHaveBeenCalledTimes(5) for (const spy of Object.values(listener)) { expect(spy).not.toHaveBeenCalled() } @@ -453,10 +454,38 @@ describe('test-reporter', () => { it('passes the suite stack and suite file', async () => { await reporter.onTestStart(jasmineTestStats() as any) - expect(framework.onReporterTestStart).toHaveBeenCalledWith( - expect.objectContaining({ fullTitle: 'Nested outer outer passing test' }), - { scopes: ['Nested outer'], suiteFile: '/work/test/p2/nested.spec.js' } - ) + expect(framework.trackEvent).toHaveBeenCalledWith(TestFrameworkState.TEST, HookState.PRE, expect.objectContaining({ + source: 'reporter', + testStats: expect.objectContaining({ fullTitle: 'Nested outer outer passing test' }), + context: { scopes: ['Nested outer'], suiteFile: '/work/test/p2/nested.spec.js' }, + })) + }) + + it('passes hook stats with the suite context, and log entries as they came', async () => { + const hook = hookStats() + await reporter.onHookStart(hook as any) + await reporter.onHookEnd(hook as any) + const entry = logEntry() + await reporter.appendTestItemLog(entry) + const context = { scopes: ['Nested outer'], suiteFile: '/work/test/p2/nested.spec.js' } + expect(reporterCalls().map(([, , args]) => args)).toEqual([ + { source: 'reporter', hookStats: hook, context }, + { source: 'reporter', hookStats: hook, context }, + { source: 'reporter', logEntry: entry }, + ]) + }) + + it('reports an each-hook in NONE, leaving the class to drop it', async () => { + await reporter.onHookStart({ ...hookStats(), title: '"before each" hook' } as any) + expect(framework.trackEvent).toHaveBeenCalledWith(TestFrameworkState.NONE, HookState.PRE, expect.objectContaining({ source: 'reporter' })) + }) + + it('records nothing for the spec when the framework minted no uuid', async () => { + vi.mocked(framework.trackEvent).mockResolvedValue(undefined) + ;(TestReporter as any).currentTest = {} + await reporter.onTestStart({ ...jasmineTestStats(), fullTitle: 'No uuid spec' } as any) + expect(TestReporter.getTests()['No uuid spec']).toBeUndefined() + expect((TestReporter as any).currentTest).toEqual({}) }) it('records the CLI uuid for the spec so command-result lookups resolve to the wire uuid', async () => { @@ -481,14 +510,13 @@ describe('test-reporter', () => { it('drops ', async () => { await reporter.onTestStart({ ...jasmineTestStats(), fullTitle: '' } as any) await reporter.onTestEnd({ ...jasmineTestStats(), fullTitle: '' } as any) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() - expect(framework.onReporterTestEnd).not.toHaveBeenCalled() + expect(framework.trackEvent).not.toHaveBeenCalled() }) it('sends nothing when Test Observability is opted out', async () => { reporter['_observability'] = false await reporter.onTestStart(jasmineTestStats() as any) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(framework.trackEvent).not.toHaveBeenCalled() expect(listener.testStarted).not.toHaveBeenCalled() }) @@ -501,9 +529,14 @@ describe('test-reporter', () => { }) it('reports unsynchronised while the framework has queued events', () => { - isIdleSpy = vi.spyOn(WdioJasmineTestFramework, 'isIdle').mockReturnValue(false) + const isIdle = vi.spyOn(framework, 'isIdle').mockReturnValue(false) expect(reporter.isSynchronised).toBe(false) - isIdleSpy.mockReturnValue(true) + isIdle.mockReturnValue(true) + expect(reporter.isSynchronised).toBe(true) + }) + + it('is synchronised when the CLI test framework is not jasmine\'s', () => { + setCli(true, { isIdle: () => false }) expect(reporter.isSynchronised).toBe(true) }) }) @@ -525,8 +558,7 @@ describe('test-reporter', () => { expect(listener.testStarted).toHaveBeenCalledTimes(1) expect(listener.testFinished).toHaveBeenCalledTimes(1) expect(listener.logCreated).toHaveBeenCalledTimes(1) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() - expect(framework.onReporterLog).not.toHaveBeenCalled() + expect(framework.trackEvent).not.toHaveBeenCalled() expect(TestReporter.getTests()['Nested outer outer passing test']).toEqual({ uuid: '123456789' }) }) }) @@ -538,7 +570,7 @@ describe('test-reporter', () => { reporter['_config']!.framework = 'mocha' await reporter.onTestStart(jasmineTestStats() as any) await reporter.onTestEnd({ ...jasmineTestStats(), state: 'passed' } as any) - expect(framework.onReporterTestStart).not.toHaveBeenCalled() + expect(framework.trackEvent).not.toHaveBeenCalled() expect(listener.testStarted).not.toHaveBeenCalled() expect(listener.testFinished).not.toHaveBeenCalled() }) diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index 15907aa6..a9b697bd 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -17,6 +17,7 @@ import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' import { HookState } from '../src/cli/states/hookState.js' import { AutomationFrameworkConstants } from '../src/cli/frameworks/constants/automationFrameworkConstants.js' import { AutomationFrameworkState } from '../src/cli/states/automationFrameworkState.js' +import PerformanceTester from '../src/instrumentation/performance/performance-tester.js' const jasmineSuiteTitle = 'Jasmine__TopLevel__Suite' const sessionBaseUrl = 'https://api.browserstack.com/automate/sessions' @@ -3275,3 +3276,65 @@ describe('_cucumberTestResult failure reason adjacent', () => { expect(result.error?.message).toBe('Unknown Error') }) }) + +describe('after hands AutomateModule the jasmine session verdict inputs', () => { + // describe('after') swaps the prototype method for a stand-in; these cases need the real one + const after = BrowserstackService.prototype.after + let trackEvent: ReturnType + let getInstanceSpy: ReturnType | undefined + + const makeService = (framework: string) => new BrowserstackService( + { testObservability: false, setSessionStatus: true } as never, + [] as never, + { user: 'foo', key: 'bar', framework } as never + ) + const executeArgs = () => trackEvent.mock.calls.find(([state, hook]) => state === AutomationFrameworkState.EXECUTE && hook === HookState.POST)?.[2] + + beforeEach(() => { + trackEvent = vi.fn().mockResolvedValue(undefined) + getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ + isRunning: () => true, + getTestFramework: () => null, + getAutomationFramework: () => ({ trackEvent }), + modules: {} + } as never) + ;(PerformanceTester as unknown as Record).stopAndGenerate = vi.fn().mockResolvedValue(undefined) + }) + + afterEach(() => { + getInstanceSpy?.mockRestore() + delete (PerformanceTester as unknown as Record).stopAndGenerate + }) + + it.each(['mocha', 'cucumber'])('sends nothing extra for %s', async (framework) => { + const service = makeService(framework) + service['_specsRan'] = true + service['_failReasons'] = ['boom'] + + await after.call(service, 1) + + expect(executeArgs()).toEqual({}) + }) + + it('sends the runner result and a copy of the tracked reasons for jasmine', async () => { + const service = makeService('jasmine') + service['_specsRan'] = true + service['_failReasons'] = ['beforeEach hook failure', 'boom'] + service['_pureTestFailReasons'] = ['boom'] + service['_hookFailReasons'] = ['beforeEach hook failure'] + + await after.call(service, 2) + + const args = executeArgs() as { sessionVerdictInputs: Record } + expect(args).toEqual({ + sessionVerdictInputs: { + result: 2, + specsRan: true, + failReasons: ['beforeEach hook failure', 'boom'], + pureTestFailReasons: ['boom'], + hookFailReasons: ['beforeEach hook failure'], + } + }) + expect(args.sessionVerdictInputs.failReasons).not.toBe(service['_failReasons']) + }) +}) From 7d13ba0202af5f835ff0490baf7aa4b82ec93c52 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 7 Oct 2026 16:05:00 +0530 Subject: [PATCH 15/24] refactor(cli): drop the jasmine framework's #iso helper for optional-chained toISOString (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../src/cli/frameworks/wdioJasmineTestFramework.ts | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts index 5b983ebe..ff4041ab 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -181,7 +181,7 @@ export default class WdioJasmineTestFramework extends TestFramework { [TestFrameworkConstants.KEY_TEST_SCOPES]: scopes, [TestFrameworkConstants.KEY_TEST_TAGS]: getTestTags(testStats as unknown as Frameworks.Test, scopes), ...this.#filePaths(context.suiteFile), - [TestFrameworkConstants.KEY_TEST_STARTED_AT]: this.#iso(testStats.start), + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: testStats.start?.toISOString(), [WdioJasmineTestFramework.KEY_TEST_DURATION]: testStats._duration, [WdioJasmineTestFramework.KEY_TEST_RETRIES]: { limit: testStats.retries || 0, attempts: testStats.retries || 0 }, }) @@ -226,8 +226,8 @@ export default class WdioJasmineTestFramework extends TestFramework { } const entries: Record = { [TestFrameworkConstants.KEY_TEST_RESULT]: result, - [TestFrameworkConstants.KEY_TEST_STARTED_AT]: this.#iso(testStats.start), - [TestFrameworkConstants.KEY_TEST_ENDED_AT]: this.#iso(testStats.end), + [TestFrameworkConstants.KEY_TEST_STARTED_AT]: testStats.start?.toISOString(), + [TestFrameworkConstants.KEY_TEST_ENDED_AT]: testStats.end?.toISOString(), [WdioJasmineTestFramework.KEY_TEST_DURATION]: testStats._duration, [TestFrameworkConstants.KEY_TEST_RESULT_AT]: new Date().toISOString(), } @@ -268,7 +268,7 @@ export default class WdioJasmineTestFramework extends TestFramework { key, [TestFrameworkConstants.KEY_HOOK_ID]: hookId, [TestFrameworkConstants.KEY_HOOK_RESULT]: TestFrameworkConstants.DEFAULT_HOOK_RESULT, - [TestFrameworkConstants.KEY_EVENT_STARTED_AT]: this.#iso(hookStats.start), + [TestFrameworkConstants.KEY_EVENT_STARTED_AT]: hookStats.start?.toISOString(), [TestFrameworkConstants.KEY_HOOK_LOGS]: [], [TestFrameworkConstants.KEY_HOOK_NAME]: title, [WdioJasmineTestFramework.KEY_HOOK_IDENTIFIER]: `${title} for ${scopes[scopes.length - 1]}`, @@ -316,7 +316,7 @@ export default class WdioJasmineTestFramework extends TestFramework { failure = this.#failureFields(error) } } - const endedAt = this.#iso(hookStats.end) + const endedAt = hookStats.end?.toISOString() const duration = hookStats._duration const emitHook = shouldProcessEventForTesthub('HookRunFinished') this.#enqueue(`${key}/POST`, async () => { @@ -484,10 +484,6 @@ export default class WdioJasmineTestFramework extends TestFramework { return fullName.slice(0, fullName.indexOf(testStats.title || '') - 1) } - #iso(date: Date | undefined) { - return date ? date.toISOString() : undefined - } - static #hookState(key: string) { if (key === 'BEFORE_ALL') { return TestFrameworkState.BEFORE_ALL From 8bec24d7376257b16d8145aff22c6dee0ae8d69a Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 7 Oct 2026 16:19:36 +0530 Subject: [PATCH 16/24] feat(cli): annotate each mocha and jasmine test from AutomateModule (SDK-7772) The per-test `Test: ` Automate annotation moves from the jasmine-only arm of service.beforeTest into AutomateModule.onBeforeTest, so mocha on the CLI flow annotates its tests again, as it did on the legacy flow. Cucumber keeps its own Feature/Scenario/Step annotations. Skip reports carry `skipReport: true` on TEST/PRE and are not annotated, since beforeTest never ran for a skipped test. A failed annotate is logged and does not break the hook. Co-Authored-By: Claude Opus 5.5 --- .../src/cli/modules/automateModule.ts | 13 ++++ .../src/cli/skipReporter.ts | 2 +- packages/browserstack-service/src/service.ts | 3 - .../tests/cli/modules/automateModule.test.ts | 59 +++++++++++++++++++ .../tests/cli/skipReporter.test.ts | 1 + .../tests/service.test.ts | 15 ++--- 6 files changed, 80 insertions(+), 13 deletions(-) diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index 666ce064..7da8dbcd 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -89,6 +89,11 @@ export default class AutomateModule extends BaseModule { return } + // Legacy annotated from service.beforeTest, which never runs for cucumber or a skipped test + if (!this.isCucumberInstance(instace) && args.skipReport !== true) { + await this.annotate(browser, `Test: ${test.fullName ?? test.title}`) + } + // `setSessionName: false` suppresses the NAME, not the registration. The session still has // to enter sessionMap or onAfterExecute has nothing to status-mark, and legacy marks it // either way — its `after()` status block gates on setSessionStatus alone. Registering with @@ -357,6 +362,14 @@ export default class AutomateModule extends BaseModule { return typeof format === 'function' ? format as TestContextOptions['sessionNameFormat'] : undefined } + private async annotate(browser: WebdriverIO.Browser, data: string) { + try { + await browser.executeScript(`browserstack_executor: ${JSON.stringify({ action: 'annotate', arguments: { data, level: 'info' } })}`, []) + } catch (error) { + this.logger.error(`annotate: failed to annotate the session: ${util.format(error)}`) + } + } + private isCucumberInstance(instance: TestFrameworkInstance): boolean { const frameworkName = String(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME) || '') return frameworkName.toLowerCase().includes('cucumber') diff --git a/packages/browserstack-service/src/cli/skipReporter.ts b/packages/browserstack-service/src/cli/skipReporter.ts index 4fff79d5..7ec13504 100644 --- a/packages/browserstack-service/src/cli/skipReporter.ts +++ b/packages/browserstack-service/src/cli/skipReporter.ts @@ -69,7 +69,7 @@ async function emitSkipReport({ framework, test, result, suiteTitle }: QueuedSki // gated on it, not on TEST/POST) — same sequence afterTest uses const steps: Array<[State, State, Record]> = [ [TestFrameworkState.INIT_TEST, HookState.PRE, { test }], - [TestFrameworkState.TEST, HookState.PRE, { test, suiteTitle }], + [TestFrameworkState.TEST, HookState.PRE, { test, suiteTitle, skipReport: true }], [TestFrameworkState.LOG_REPORT, HookState.POST, { test, result }], [TestFrameworkState.TEST, HookState.POST, { test, result, suiteTitle }], ] diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 4a24bd1e..16adc4bf 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -644,9 +644,6 @@ export default class BrowserstackService implements Services.ServiceInstance { // skip reporter must never re-report it from onTestSkip markTestStarted(getUniqueIdentifier(test, this._config.framework)) this._insightsHandler?.setTestData(test, uuid) - if (this._config.framework === 'jasmine') { - await this._setAnnotation(`Test: ${test.fullName ?? test.title}`) - } await BrowserstackCLI.getInstance().getTestFramework()!.trackEvent(TestFrameworkState.TEST, HookState.PRE, { test, suiteTitle }) return } diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index 3d75673f..aed20e4e 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -6,6 +6,7 @@ import { TestFrameworkState } from '../../../src/cli/states/testFrameworkState.j import { AutomationFrameworkState } from '../../../src/cli/states/automationFrameworkState.js' import { HookState } from '../../../src/cli/states/hookState.js' import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/testFrameworkConstants.js' +import { BStackLogger } from '../../../src/cli/cliLogger.js' import { isBrowserstackSession } from '../../../src/util.js' import PerformanceTester from '../../../src/instrumentation/performance/performance-tester.js' import { _fetch as fetch } from '../../../src/fetchWrapper.js' @@ -1153,6 +1154,64 @@ describe('AutomateModule — jasmine session verdict', () => { }) }) +describe('AutomateModule — per-test annotation', () => { + let automateModule: AutomateModule + let executeScript: ReturnType + const annotation = (data: string) => `browserstack_executor: ${JSON.stringify({ action: 'annotate', arguments: { data, level: 'info' } })}` + + const runBeforeTest = (frameworkName: string, test: Record, extra: Record = {}) => { + vi.mocked(TestFramework.getState).mockImplementation((_i, key) => key === TestFrameworkConstants.KEY_TEST_FRAMEWORK_NAME ? frameworkName : undefined) + return automateModule.onBeforeTest({ instance: {}, test, suiteTitle: 'Suite', ...extra }) + } + + beforeEach(() => { + vi.clearAllMocks() + executeScript = vi.fn().mockResolvedValue(undefined) + vi.mocked(AutomationFramework.getTrackedInstance).mockReturnValue({} as any) + vi.mocked(AutomationFramework.getDriver).mockReturnValue({ sessionId: 's1', executeScript } as any) + vi.mocked(AutomationFramework.getState).mockImplementation((_i, key) => key === 'framework_session_id' ? 's1' : {}) + vi.mocked(isBrowserstackSession).mockReturnValue(true) + vi.mocked(fetch).mockResolvedValue({ json: async () => ({}) } as any) + automateModule = new AutomateModule({} as Options.Testrunner) + automateModule.config = { + testContextOptions: { skipSessionName: false, skipSessionStatus: false }, + userName: 'u', + accessKey: 'k' + } as any + }) + + it('annotates a mocha test with its title, as legacy beforeTest did', async () => { + await runBeforeTest('WebdriverIO-mocha', { title: 't', parent: 'Suite' }) + expect(executeScript).toHaveBeenCalledExactlyOnceWith(annotation('Test: t'), []) + }) + + it('annotates a jasmine spec with its full name', async () => { + await runBeforeTest('WebdriverIO-jasmine', { description: 'outer passing test', fullName: 'Nested outer outer passing test' }) + expect(executeScript).toHaveBeenCalledExactlyOnceWith(annotation('Test: Nested outer outer passing test'), []) + }) + + it('annotates even when the session name is skipped', async () => { + (automateModule.config as any).testContextOptions.skipSessionName = true + await runBeforeTest('WebdriverIO-mocha', { title: 't', parent: 'Suite' }) + expect(executeScript).toHaveBeenCalledExactlyOnceWith(annotation('Test: t'), []) + }) + + it('never annotates cucumber, a skip report, or a non-BrowserStack session', async () => { + await runBeforeTest('WebdriverIO-cucumber', { title: 'Scenario', parent: 'Feature' }) + await runBeforeTest('WebdriverIO-mocha', { title: 't', parent: 'Suite' }, { skipReport: true }) + vi.mocked(isBrowserstackSession).mockReturnValue(false) + await runBeforeTest('WebdriverIO-mocha', { title: 't', parent: 'Suite' }) + expect(executeScript).not.toHaveBeenCalled() + }) + + it('logs a failed annotate and still names the session', async () => { + executeScript.mockRejectedValue(new Error('annotate blew up')) + await expect(runBeforeTest('WebdriverIO-mocha', { title: 't', parent: 'Suite' })).resolves.toBeUndefined() + expect(BStackLogger.error).toHaveBeenCalledWith(expect.stringContaining('annotate blew up')) + expect((automateModule as any).sessionMap.get('s1')?.lastTestName).toBe('Suite - t') + }) +}) + describe('AutomateModule — jasmine sessionNameFormat', () => { let automateModule: AutomateModule const format = vi.fn((_config: unknown, _caps: unknown, suiteTitle: string, testTitle?: string) => `fmt[${suiteTitle}][${String(testTitle)}]`) diff --git a/packages/browserstack-service/tests/cli/skipReporter.test.ts b/packages/browserstack-service/tests/cli/skipReporter.test.ts index 6846e985..32a9f094 100644 --- a/packages/browserstack-service/tests/cli/skipReporter.test.ts +++ b/packages/browserstack-service/tests/cli/skipReporter.test.ts @@ -33,6 +33,7 @@ describe('skipReporter', () => { [TestFrameworkState.TEST, HookState.POST], ]) expect(calls[2][2]).toMatchObject({ result: { passed: false, skipped: true } }) + expect(calls.map(([, , args]) => (args as Record).skipReport)).toEqual([undefined, true, undefined, undefined]) }) it('does not re-report the same identifier', async () => { diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index a9b697bd..1a146ab8 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -1218,16 +1218,13 @@ describe('beforeTest', () => { return { annotate, trackEvent } } - it('annotates each jasmine spec with its full name, before the modules see TEST/PRE', async () => { - const { annotate, trackEvent } = await runBeforeTest('jasmine', jasmineLastSpec) - expect(annotate).toHaveBeenCalledWith('Test: Nested outer outer passing test') - const testPre = trackEvent.mock.calls.findIndex(([state]) => state === TestFrameworkState.TEST) - expect(annotate.mock.invocationCallOrder[0]).toBeLessThan(trackEvent.mock.invocationCallOrder[testPre]) - }) - - it('does not annotate mocha tests on the CLI flow', async () => { - const { annotate } = await runBeforeTest('mocha', { title: 't', parent: 'suite' }) + it.each([ + ['jasmine', jasmineLastSpec], + ['mocha', { title: 't', parent: 'suite' }], + ])('leaves the %s test annotation to AutomateModule and still sends TEST/PRE', async (framework, test) => { + const { annotate, trackEvent } = await runBeforeTest(framework, test as Record) expect(annotate).not.toHaveBeenCalled() + expect(trackEvent).toHaveBeenCalledWith(TestFrameworkState.TEST, HookState.PRE, expect.objectContaining({ test })) }) }) }) From d3c406205aa5f8599f42fc8399bb4b50ae1ed432 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Wed, 7 Oct 2026 22:39:33 +0530 Subject: [PATCH 17/24] refactor(cli): capture WebDriver command logs in TestHubModule (SDK-7772) TestHubModule registers the command/result listeners once per browser on driver creation, gated as before on shouldProcessEventForTesthub, and pairs results on the live session id. Screenshots go to every framework and HTTP command logs only to frameworks that opt in through capturesHttpCommandLogs(), which only jasmine does. Entries carry `commandLog: true` and the framework attributes them: - jasmine sends them to the last started spec, cleared by an `` so they drop until the next spec, as the runner's own current test does - cucumber drops them until a scenario has started - mocha keeps its LOG rail, so a screenshot in a hook now attaches to the hook run as it did on the legacy flow The service no longer registers CLI listeners, browserCommand keeps only the legacy path, and setTestData no longer seeds the test map that only the removed CLI gate read. Co-Authored-By: Claude Opus 5.5 --- .../src/cli/frameworks/testFramework.ts | 5 + .../frameworks/wdioCucumberTestFramework.ts | 9 ++ .../frameworks/wdioJasmineTestFramework.ts | 26 ++-- .../browserstack-service/src/cli/index.ts | 1 + .../src/cli/modules/testHubModule.ts | 84 ++++++++++- .../src/insights-handler.ts | 54 +------ packages/browserstack-service/src/reporter.ts | 3 + packages/browserstack-service/src/service.ts | 29 ---- .../tests/cli/index.test.ts | 5 + .../testHubModule.deferredFinish.test.ts | 2 +- .../tests/cli/modules/testHubModule.test.ts | 138 +++++++++++++++++- .../cli/skipReporter.afterOrdering.test.ts | 2 +- .../cli/wdioCucumberTestFramework.test.ts | 22 +++ .../cli/wdioJasmineTestFramework.test.ts | 36 ++++- .../tests/insights-handler.test.ts | 48 +++--- ...vability-build-start-success-false.test.ts | 4 +- .../tests/reporter.test.ts | 6 +- .../tests/service.test.ts | 17 +-- 18 files changed, 349 insertions(+), 142 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/testFramework.ts b/packages/browserstack-service/src/cli/frameworks/testFramework.ts index f291505c..70654310 100644 --- a/packages/browserstack-service/src/cli/frameworks/testFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/testFramework.ts @@ -73,6 +73,11 @@ export default class TestFramework { return this.testFrameworks } + /** Whether WebDriver HTTP command logs are reported for this framework; screenshots always are. */ + capturesHttpCommandLogs() { + return false + } + /** * Track an event * @param {TestFrameworkState} testFrameworkState diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index d225e6b2..1f988c44 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -116,6 +116,8 @@ export default class WdioCucumberTestFramework extends TestFramework { /** The hook currently open on this worker — started and not yet finished. */ private openHook: { key: string, hookId: string } | null = null + // WebDriver command logs attach only once a scenario has started, as legacy's current-test gate did. + private scenarioStarted = false constructor(testFrameworks: string[], testFrameworkVersions: Record, binSessionId: string) { super(testFrameworks, testFrameworkVersions, binSessionId) @@ -222,6 +224,13 @@ export default class WdioCucumberTestFramework extends TestFramework { logger.debug(`WdioCucumberTestFramework.trackEvent: testFrameworkState=${testFrameworkState} hookState=${hookState}`) await super.trackEvent(testFrameworkState, hookState, args) + if (testFrameworkState === TestFrameworkState.TEST && hookState === HookState.PRE) { + this.scenarioStarted = true + } else if (testFrameworkState === TestFrameworkState.LOG && args.commandLog === true && !this.scenarioStarted) { + logger.debug('trackEvent: command log before the first scenario dropped') + return + } + const instance = this.resolveInstance(testFrameworkState, hookState) if (!instance) { // Console output emitted before the first scenario (or before BeforeAll) has nothing diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts index ff4041ab..0dc71e4c 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -68,8 +68,6 @@ export default class WdioJasmineTestFramework extends TestFramework { #specInstances = new Map() // Open specs by fullName: how the service's hooks find the instance the reporter minted. #specsByFullName = new Map() - // Every spec of this worker by uuid: legacy attaches HTTP logs and screenshots to the last started spec, even after it ended. - #specsByUuid = new Map() #serviceOnly = new Set() #suiteTitles = new Map() #testHub: TestHubModule | null = null @@ -77,6 +75,9 @@ export default class WdioJasmineTestFramework extends TestFramework { #hookInstances = new Map() #openHook: TestFrameworkInstance | null = null #lastSpec: TestFrameworkInstance | null = null + // The spec WebDriver command logs attach to: the last started spec, even inside an all-hook or after it + // ended, unset by a `` (the runner's last spec then has no uuid, so its commands drop). + #commandLogSpec: TestFrameworkInstance | null = null #queue: Promise = Promise.resolve() #pendingEvents = 0 @@ -100,6 +101,10 @@ export default class WdioJasmineTestFramework extends TestFramework { return WdioJasmineTestFramework.#hookState(getHookType(String(hookTitle ?? '').toLowerCase())) ?? TestFrameworkState.NONE } + capturesHttpCommandLogs() { + return true + } + setTestHubModule(testHub: TestHubModule | null | undefined) { this.#testHub = testHub ?? null } @@ -116,7 +121,7 @@ export default class WdioJasmineTestFramework extends TestFramework { return } if (testFrameworkState === TestFrameworkState.LOG) { - this.#log(args.logEntry as Record) + this.#log(args.logEntry as Record, args.commandLog === true) return } try { @@ -152,7 +157,11 @@ export default class WdioJasmineTestFramework extends TestFramework { #trackReporterEvent(testFrameworkState: State, hookState: State, args: Record) { const context = args.context as JasmineSuiteContext if (testFrameworkState === TestFrameworkState.TEST) { - if (hookState === HookState.PRE) { + if (hookState === HookState.PRE && args.unknownTest === true) { + this.#enqueue('UNKNOWN_TEST', async () => { + this.#commandLogSpec = null + }) + } else if (hookState === HookState.PRE) { args.testUuid = this.#testStarted(args.testStats as TestStats, context) } else { this.#testEnded(args.testStats as TestStats, context) @@ -187,13 +196,13 @@ export default class WdioJasmineTestFramework extends TestFramework { }) this.#specInstances.set(testStats.uid, instance) this.#specsByFullName.set(fullTitle, instance) - this.#specsByUuid.set(TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) as string, instance) // Registered synchronously: the service's beforeTest for this spec may read it before the queue runs. TestFramework.setTrackedInstance(instance.getContext(), instance) const args = { test: this.#specArg(testStats, context), suiteTitle: this.#suiteTitle(testStats) } this.#enqueue('TEST/PRE', async () => { this.#lastSpec = instance + this.#commandLogSpec = instance await this.#toTestHub(instance, TestFrameworkState.TEST, HookState.PRE, args) }) return TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) as string @@ -352,16 +361,15 @@ export default class WdioJasmineTestFramework extends TestFramework { /** * Console logs: an open beforeAll/afterAll wins, else the last-started spec, even after it ended. - * HTTP command logs and screenshots name their spec (`test_run_uuid`), as legacy did; unknown uuids are dropped. + * WebDriver command logs (HTTP, screenshots) always go to the command-log spec, as legacy did. */ - #log(logEntry: Record | undefined) { + #log(logEntry: Record | undefined, commandLog = false) { try { if (!logEntry || !shouldProcessEventForTesthub('LogCreated')) { return } - const targetUuid = logEntry.test_run_uuid as string | undefined this.#enqueue('LOG/POST', async () => { - const instance = targetUuid ? this.#specsByUuid.get(targetUuid) : (this.#openHook ?? this.#lastSpec) + const instance = commandLog ? this.#commandLogSpec : (this.#openHook ?? this.#lastSpec) if (!instance) { return } diff --git a/packages/browserstack-service/src/cli/index.ts b/packages/browserstack-service/src/cli/index.ts index 56eaf74c..acc8133e 100644 --- a/packages/browserstack-service/src/cli/index.ts +++ b/packages/browserstack-service/src/cli/index.ts @@ -181,6 +181,7 @@ export class BrowserstackCLI { const testHubModule = new TestHubModule(startBinResponse.testhub) this.modules[TestHubModule.MODULE_NAME] = testHubModule + testHubModule.setTestFramework(this.testFramework) if (this.testFramework instanceof WdioJasmineTestFramework) { this.testFramework.setTestHubModule(testHubModule) } diff --git a/packages/browserstack-service/src/cli/modules/testHubModule.ts b/packages/browserstack-service/src/cli/modules/testHubModule.ts index 1830be1e..606e23bf 100644 --- a/packages/browserstack-service/src/cli/modules/testHubModule.ts +++ b/packages/browserstack-service/src/cli/modules/testHubModule.ts @@ -11,15 +11,26 @@ import type TestFrameworkInstance from '../instances/testFrameworkInstance.js' // eslint-disable-next-line camelcase import type { LogCreatedEventRequest, LogCreatedEventRequest_LogEntry, TestFrameworkEventRequest, TestSessionEventRequest, AutomationSession } from '../../grpc/index.js' import type { Frameworks } from '@wdio/types' +import type { AfterCommandArgs } from '@wdio/reporter' import WdioMochaTestFramework from '../frameworks/wdioMochaTestFramework.js' import type AutomationFrameworkInstance from '../instances/automationFrameworkInstance.js' import AutomationFramework from '../frameworks/automationFramework.js' import { AutomationFrameworkConstants } from '../frameworks/constants/automationFrameworkConstants.js' -import { isLoadTestingSession, getLtsSessionId } from '../../util.js' +import { isLoadTestingSession, getLtsSessionId, isFalse, isScreenshotCommand } from '../../util.js' +import { AutomationFrameworkState } from '../states/automationFrameworkState.js' +import { shouldProcessEventForTesthub } from '../../testHub/utils.js' +import { TESTOPS_SCREENSHOT_ENV } from '../../constants.js' /** * TestHub Module for BrowserStack */ +interface WebDriverCommand { + method: string + endpoint: string + body?: unknown + result?: unknown +} + export default class TestHubModule extends BaseModule { logger = BStackLogger @@ -56,6 +67,12 @@ export default class TestHubModule extends BaseModule { */ private pendingTestFinishes: Map, uuid: string }> = new Map() + private testFramework: TestFramework | null = null + // A reloaded session keeps its browser object, so listeners are registered once per object. + private commandLogBrowsers = new WeakSet() + // Commands seen, keyed by live session, method and endpoint; a result only logs HTTP for a seen command. + private seenCommands = new Set() + /** * Create a new TestHubModule */ @@ -65,6 +82,7 @@ export default class TestHubModule extends BaseModule { this.testhubConfig = testhubConfig TestFramework.registerObserver(TestFrameworkState.TEST, HookState.PRE, this.onBeforeTest.bind(this)) + AutomationFramework.registerObserver(AutomationFrameworkState.CREATE, HookState.POST, this.onDriverCreated.bind(this)) Object.values(TestFrameworkState).forEach(state => { Object.values(HookState).forEach(hook => { @@ -81,6 +99,70 @@ export default class TestHubModule extends BaseModule { return TestHubModule.MODULE_NAME } + setTestFramework(testFramework: TestFramework | null | undefined) { + this.testFramework = testFramework ?? null + } + + /** + * WebDriver command logs: screenshots for every framework, HTTP command logs for frameworks that opt in. + * The framework attributes each entry; `commandLog: true` tells it the entry came from here. + */ + onDriverCreated(args: Record) { + try { + const browser = args.browser as WebdriverIO.Browser | undefined + if (!browser || this.commandLogBrowsers.has(browser) || !shouldProcessEventForTesthub('')) { + return + } + this.commandLogBrowsers.add(browser) + if (this.testFramework?.capturesHttpCommandLogs()) { + browser.on('command', (command: WebDriverCommand) => { + if (shouldProcessEventForTesthub('')) { + this.seenCommands.add(this.commandKey(browser, command)) + } + }) + } + browser.on('result', (result: WebDriverCommand) => { + if (shouldProcessEventForTesthub('')) { + void this.onCommandResult(browser, result) + } + }) + } catch (error) { + this.logger.error(`onDriverCreated: failed to register command log listeners: ${util.format(error)}`) + } + } + + private async onCommandResult(browser: WebdriverIO.Browser, result: WebDriverCommand) { + try { + const testFramework = this.testFramework + if (!testFramework) { + return + } + const value = (result.result as { value?: string } | undefined)?.value + // `allow_screenshots` is a string on the wire, so a denial arrives as 'false' + const allowScreenshots = process.env[TESTOPS_SCREENSHOT_ENV] + if (Boolean(allowScreenshots) && !isFalse(allowScreenshots) && isScreenshotCommand(result as AfterCommandArgs) && value) { + await testFramework.trackEvent(TestFrameworkState.LOG, HookState.POST, { + logEntry: { kind: TestFrameworkConstants.KIND_SCREENSHOT, message: value, timestamp: new Date().toISOString() }, + commandLog: true, + }) + } + if (!testFramework.capturesHttpCommandLogs() || !this.seenCommands.has(this.commandKey(browser, result))) { + return + } + const httpResponse = { path: result.endpoint, method: result.method, body: result.body, response: result.result } + await testFramework.trackEvent(TestFrameworkState.LOG, HookState.POST, { + logEntry: { kind: 'HTTP', message: JSON.stringify(httpResponse), timestamp: new Date().toISOString() }, + commandLog: true, + }) + } catch (error) { + this.logger.error(`onCommandResult: failed to log ${result?.method} ${result?.endpoint}: ${util.format(error)}`) + } + } + + private commandKey(browser: WebdriverIO.Browser, command: WebDriverCommand) { + return `${browser.sessionId}_${command.method}_${command.endpoint}` + } + onBeforeTest(args: Record) { // Jasmine reports to TestHub from its reporter feed; its module-only dispatches carry this marker if (args.skipTestHub === true) { diff --git a/packages/browserstack-service/src/insights-handler.ts b/packages/browserstack-service/src/insights-handler.ts index 1dc36338..1346b4f1 100644 --- a/packages/browserstack-service/src/insights-handler.ts +++ b/packages/browserstack-service/src/insights-handler.ts @@ -47,7 +47,6 @@ import { TESTOPS_SCREENSHOT_ENV } from './constants.js' import { BrowserstackCLI } from './cli/index.js' import { TestFrameworkState } from './cli/states/testFrameworkState.js' import { HookState } from './cli/states/hookState.js' -import { TestFrameworkConstants } from './cli/frameworks/constants/testFrameworkConstants.js' import PerformanceTester from './instrumentation/performance/performance-tester.js' import * as PERFORMANCE_SDK_EVENTS from './instrumentation/performance/constants.js' import CustomTagsHandler from './custom-tags-handler.js' @@ -797,31 +796,12 @@ class _InsightsHandler { // string 'false' — which Boolean() reads as permission granted. Honour it explicitly. const allowScreenshots = process.env[TESTOPS_SCREENSHOT_ENV] if (Boolean(allowScreenshots) && !isFalse(allowScreenshots) && isScreenshotCommand(args) && result?.value) { - // On the binary path the direct screenshot endpoint answers 401 to the binary's JWT, - // so ride the same LOG rail appendTestItemLog uses: the CLI stamps the test uuid and - // forwards the entry over gRPC, where the binary owns reporting. The framework can be - // unset while isRunning() is true — the dev-env short-circuit returns true before - // setupTestFramework() has run, and it only assigns for webdriverio-mocha — so resolve - // it rather than assert, and let an untracked framework fall through to the direct - // upload instead of throwing the screenshot away. - const cliTestFramework = BrowserstackCLI.getInstance().isRunning() - ? BrowserstackCLI.getInstance().getTestFramework() - : undefined - await (cliTestFramework - ? cliTestFramework.trackEvent(TestFrameworkState.LOG, HookState.POST, { - logEntry: { - kind: TestFrameworkConstants.KIND_SCREENSHOT, - message: result.value, - timestamp: new Date().toISOString(), - ...(this._framework === 'jasmine' ? { test_run_uuid: testMeta.uuid } : {}) - } - }) - : this.listener.onScreenshot([{ - test_run_uuid: testMeta.uuid, - timestamp: new Date().toISOString(), - message: result.value, - kind: 'TEST_SCREENSHOT' - }])) + await this.listener.onScreenshot([{ + test_run_uuid: testMeta.uuid, + timestamp: new Date().toISOString(), + message: result.value, + kind: 'TEST_SCREENSHOT' + }]) } const requestData = this._commands[dataKey] @@ -836,18 +816,6 @@ class _InsightsHandler { body, response: result } - // Only jasmine registers `command` on the CLI flow, and the legacy listener is inert there - if (BrowserstackCLI.getInstance().isRunning()) { - await BrowserstackCLI.getInstance().getTestFramework()?.trackEvent(TestFrameworkState.LOG, HookState.POST, { - logEntry: { - kind: 'HTTP', - message: JSON.stringify(httpResponse), - timestamp: new Date().toISOString(), - test_run_uuid: testMeta.uuid - } - }) - return - } this.listener.logCreated([{ test_run_uuid: testMeta.uuid, timestamp: new Date().toISOString(), @@ -1258,16 +1226,6 @@ class _InsightsHandler { public setTestData (test: Frameworks.Test | ITestCaseHookParameter, uuid: string) { // Legacy's cucumber beforeScenario records the uuid alone; only mocha carries the test. InsightsHandler.currentTest = 'pickle' in test ? { uuid } : { test, uuid } - // browserCommand resolves the active test through this map and returns before the - // screenshot branch without an entry, so the CLI cucumber path has to seed it too - // (SDK-4177). getIdentifier already keys a world by its pickle. - if (this._framework !== 'mocha' && !('pickle' in test)) { - return - } - this._tests[this.getIdentifier(test)] = { - uuid, - startedAt: (new Date()).toISOString() - } } } diff --git a/packages/browserstack-service/src/reporter.ts b/packages/browserstack-service/src/reporter.ts index 1345c8b5..a1b8d2db 100644 --- a/packages/browserstack-service/src/reporter.ts +++ b/packages/browserstack-service/src/reporter.ts @@ -201,6 +201,9 @@ class _TestReporter extends WDIOReporter { return } if (testStats.fullTitle === '') { + if (this.isCliJasmine()) { + this.cliJasmineFramework()?.trackEvent(TestFrameworkState.TEST, HookState.PRE, { source: 'reporter', unknownTest: true }) + } return } if (this.isCliJasmine()) { diff --git a/packages/browserstack-service/src/service.ts b/packages/browserstack-service/src/service.ts index 16adc4bf..ceda53de 100644 --- a/packages/browserstack-service/src/service.ts +++ b/packages/browserstack-service/src/service.ts @@ -327,35 +327,6 @@ export default class BrowserstackService implements Services.ServiceInstance { BStackLogger.info(`CLI is running, tracking insights event for before: ${sessionId}`) await BrowserstackCLI.getInstance().getAutomationFramework()!.trackEvent(AutomationFrameworkState.CREATE, HookState.POST, { browser: this._browser, hubUrl: this._config.hostname }) this._insightsHandler.setGitConfigPath() - /** - * `browserCommand` is the only producer of TEST_SCREENSHOT logs — the binary - * has no screenshot producer of its own — so the result event has to be - * registered on this path too, or a screenshot taken mid-test never reaches - * Observability (SDK-4177). The `command` (beforeCommand) event only fills - * the map browserCommand's HTTP-log half reads, so it is registered for - * jasmine alone, whose HTTP command logs go over gRPC; mocha and cucumber - * never reported them on this path. - */ - if (this._config.framework === 'jasmine') { - this._browser.on('command', (command) => { - if (shouldProcessEventForTesthub('')) { - this._insightsHandler?.browserCommand( - 'client:beforeCommand', - Object.assign(command, { sessionId }), - this._currentTest - ) - } - }) - } - this._browser.on('result', (result) => { - if (shouldProcessEventForTesthub('')) { - this._insightsHandler?.browserCommand( - 'client:afterCommand', - Object.assign(result, { sessionId }), - this._currentTest - ) - } - }) PerformanceTester.end(PERFORMANCE_SDK_EVENTS.DRIVER_EVENT.PRE_INITIALIZE) return } diff --git a/packages/browserstack-service/tests/cli/index.test.ts b/packages/browserstack-service/tests/cli/index.test.ts index 6c8bf92c..18600595 100644 --- a/packages/browserstack-service/tests/cli/index.test.ts +++ b/packages/browserstack-service/tests/cli/index.test.ts @@ -187,6 +187,11 @@ describe('BrowserstackCLI bootstrap error surfacing', () => { expect(setTestHubModule.mock.calls[0][0]).toBe(instance.modules.TestHubModule) }) + it('hands TestHubModule the test framework, whatever it is', () => { + load('WebdriverIO-mocha') + expect(instance.modules.TestHubModule['testFramework']).toBe(instance.getTestFramework()) + }) + it('leaves the mocha framework untouched', () => { const setTestHubModule = vi.spyOn(WdioJasmineTestFramework.prototype, 'setTestHubModule') load('WebdriverIO-mocha') diff --git a/packages/browserstack-service/tests/cli/modules/testHubModule.deferredFinish.test.ts b/packages/browserstack-service/tests/cli/modules/testHubModule.deferredFinish.test.ts index 3b167c93..a2aee5ad 100644 --- a/packages/browserstack-service/tests/cli/modules/testHubModule.deferredFinish.test.ts +++ b/packages/browserstack-service/tests/cli/modules/testHubModule.deferredFinish.test.ts @@ -18,7 +18,7 @@ vi.mock('../../../src/cli/frameworks/testFramework.js', () => ({ })) vi.mock('../../../src/cli/frameworks/automationFramework.js', () => ({ - default: { getTrackedInstance: vi.fn(), getState: vi.fn(), getDriver: vi.fn() } + default: { getTrackedInstance: vi.fn(), getState: vi.fn(), getDriver: vi.fn(), registerObserver: vi.fn() } })) vi.mock('../../../src/cli/grpcClient.js', () => ({ diff --git a/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts b/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts index c1031d39..5a32125d 100644 --- a/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts @@ -1,3 +1,4 @@ +import { EventEmitter } from 'node:events' import type { Mock } from 'vitest' import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest' import TestHubModule from '../../../src/cli/modules/testHubModule.js' @@ -10,6 +11,7 @@ import WdioMochaTestFramework from '../../../src/cli/frameworks/wdioMochaTestFra import { TestFrameworkConstants } from '../../../src/cli/frameworks/constants/testFrameworkConstants.js' import { AutomationFrameworkConstants } from '../../../src/cli/frameworks/constants/automationFrameworkConstants.js' import type { Frameworks } from '@wdio/types' +import { BStackLogger } from '../../../src/cli/cliLogger.js' // Mock all dependencies vi.mock('../../../src/cli/frameworks/testFramework.js', () => ({ @@ -26,7 +28,8 @@ vi.mock('../../../src/cli/frameworks/automationFramework.js', () => ({ default: { getTrackedInstance: vi.fn(), getState: vi.fn(), - getDriver: vi.fn() + getDriver: vi.fn(), + registerObserver: vi.fn() } })) @@ -617,4 +620,135 @@ describe('TestHubModule', () => { expect(call.capabilities).toEqual(new Uint8Array()) }) }) -}) \ No newline at end of file +}) +describe('TestHubModule — WebDriver command logs', () => { + const settle = () => new Promise(resolve => setImmediate(resolve)) + const title = { method: 'GET', endpoint: '/session/:sessionId/title', body: {} } + const screenshot = { method: 'GET', endpoint: '/session/:sessionId/screenshot', body: {} } + let module: TestHubModule + let browser: EventEmitter & { sessionId: string } + let framework: { trackEvent: Mock, capturesHttpCommandLogs: Mock } + const logEntries = () => framework.trackEvent.mock.calls.map(([state, hook, args]) => { + expect([state, hook]).toEqual([TestFrameworkState.LOG, HookState.POST]) + expect(args.commandLog).toBe(true) + return args.logEntry + }) + + const register = (httpLogs: boolean) => { + framework = { trackEvent: vi.fn().mockResolvedValue(undefined), capturesHttpCommandLogs: vi.fn().mockReturnValue(httpLogs) } + module.setTestFramework(framework as unknown as TestFramework) + module.onDriverCreated({ browser }) + } + const run = async (command: Record, result: unknown) => { + browser.emit('command', { ...command }) + browser.emit('result', { ...command, result }) + await settle() + } + + beforeEach(() => { + vi.clearAllMocks() + process.env.BROWSERSTACK_OBSERVABILITY = 'true' + process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = 'true' + module = new TestHubModule({}) + browser = Object.assign(new EventEmitter(), { sessionId: 's1' }) + }) + + afterEach(() => { + delete process.env.BROWSERSTACK_OBSERVABILITY + delete process.env.BS_TESTOPS_ALLOW_SCREENSHOTS + }) + + it('observes driver creation', () => { + expect(AutomationFramework.registerObserver).toHaveBeenCalledWith(expect.anything(), HookState.POST, expect.any(Function)) + }) + + it('sends a paired HTTP log when the framework opts in, in the legacy message shape', async () => { + register(true) + await run(title, { value: 'StackDemo' }) + expect(logEntries()).toEqual([{ kind: 'HTTP', message: JSON.stringify({ path: title.endpoint, method: 'GET', body: {}, response: { value: 'StackDemo' } }), timestamp: expect.any(String) }]) + }) + + it('sends no HTTP log, and listens for no command, when the framework does not opt in', async () => { + register(false) + await run(title, { value: 'StackDemo' }) + expect(browser.listenerCount('command')).toBe(0) + expect(framework.trackEvent).not.toHaveBeenCalled() + }) + + it('sends a screenshot for every framework, then the HTTP log when opted in', async () => { + register(false) + await run(screenshot, { value: 'b64' }) + expect(logEntries()).toEqual([{ kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: expect.any(String) }]) + + browser = Object.assign(new EventEmitter(), { sessionId: 's1' }) + register(true) + await run(screenshot, { value: 'b64' }) + expect(logEntries().map(e => e.kind)).toEqual(['TEST_SCREENSHOT', 'HTTP']) + }) + + it.each([undefined, 'false', ''])('sends no screenshot when allow_screenshots is %j', async (allow) => { + if (allow === undefined) { + delete process.env.BS_TESTOPS_ALLOW_SCREENSHOTS + } else { + process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = allow + } + register(false) + await run(screenshot, { value: 'b64' }) + expect(framework.trackEvent).not.toHaveBeenCalled() + }) + + it('pairs on the live session id, so a reload pairs its own commands and never an earlier session\'s', async () => { + register(true) + browser.emit('command', { ...title }) + browser.sessionId = 's2' + browser.emit('result', { ...title, result: { value: 'old' } }) + await settle() + expect(framework.trackEvent).not.toHaveBeenCalled() + + await run(title, { value: 'new' }) + expect(logEntries().map(e => JSON.parse(e.message).response.value)).toEqual(['new']) + }) + + it('pairs every result of concurrent same-key commands', async () => { + register(true) + browser.emit('command', { ...title }) + browser.emit('command', { ...title }) + browser.emit('result', { ...title, result: { value: 'a' } }) + browser.emit('result', { ...title, result: { value: 'b' } }) + await settle() + expect(logEntries()).toHaveLength(2) + }) + + it('registers once per browser, so a reload never doubles the logs', async () => { + register(true) + module.onDriverCreated({ browser }) + await run(title, { value: 'StackDemo' }) + expect(browser.listenerCount('result')).toBe(1) + expect(framework.trackEvent).toHaveBeenCalledTimes(1) + }) + + it('registers nothing, and later sends nothing, when no product reports events', async () => { + delete process.env.BROWSERSTACK_OBSERVABILITY + register(true) + expect(browser.listenerCount('result')).toBe(0) + + process.env.BROWSERSTACK_OBSERVABILITY = 'true' + browser = Object.assign(new EventEmitter(), { sessionId: 's1' }) + register(true) + delete process.env.BROWSERSTACK_OBSERVABILITY + await run(title, { value: 'StackDemo' }) + expect(framework.trackEvent).not.toHaveBeenCalled() + }) + + it('never throws out of a listener when the framework fails', async () => { + register(true) + framework.trackEvent.mockRejectedValue(new Error('track blew up')) + await expect(run(screenshot, { value: 'b64' })).resolves.toBeUndefined() + expect(BStackLogger.error).toHaveBeenCalledWith(expect.stringContaining('track blew up')) + }) + + it('logs nothing without a test framework', async () => { + module.onDriverCreated({ browser }) + await expect(run(screenshot, { value: 'b64' })).resolves.toBeUndefined() + }) +}) diff --git a/packages/browserstack-service/tests/cli/skipReporter.afterOrdering.test.ts b/packages/browserstack-service/tests/cli/skipReporter.afterOrdering.test.ts index 8ab789dc..f87e3928 100644 --- a/packages/browserstack-service/tests/cli/skipReporter.afterOrdering.test.ts +++ b/packages/browserstack-service/tests/cli/skipReporter.afterOrdering.test.ts @@ -13,7 +13,7 @@ vi.mock('../../src/cli/frameworks/testFramework.js', () => ({ })) vi.mock('../../src/cli/frameworks/automationFramework.js', () => ({ - default: { getTrackedInstance: vi.fn(), getState: vi.fn(), getDriver: vi.fn() } + default: { getTrackedInstance: vi.fn(), getState: vi.fn(), getDriver: vi.fn(), registerObserver: vi.fn() } })) vi.mock('../../src/cli/grpcClient.js', () => ({ diff --git a/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts index 53d44380..2b0a2734 100644 --- a/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it, vi, beforeEach } from 'vitest' import path from 'node:path' import * as bstackLogger from '../../src/bstackLogger.js' import WdioCucumberTestFramework from '../../src/cli/frameworks/wdioCucumberTestFramework.js' +import { TestFrameworkState } from '../../src/cli/states/testFrameworkState.js' +import { HookState } from '../../src/cli/states/hookState.js' vi.spyOn(bstackLogger.BStackLogger, 'logToFile').mockImplementation(() => {}) @@ -58,3 +60,23 @@ describe.each([ expect(meta.feature.path).not.toContain(process.cwd()) }) }) + +describe('cucumber WebDriver command logs', () => { + const commandLog = { logEntry: { kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: 't' }, commandLog: true } + + it('drops command logs until a scenario has started, then attributes them as before', async () => { + const framework = new WdioCucumberTestFramework(['WebdriverIO-cucumber'], { 'WebdriverIO-cucumber': '9.0.0' }, 'bin-1') + const resolveInstance = vi.spyOn(framework as never, 'resolveInstance').mockReturnValue(null as never) + + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { ...commandLog }) + expect(resolveInstance).not.toHaveBeenCalled() + + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'TEST_LOG', message: 'console' } }) + expect(resolveInstance).toHaveBeenCalledTimes(1) + + await framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, {}) + await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { ...commandLog }) + expect(resolveInstance).toHaveBeenCalledTimes(3) + expect(resolveInstance).toHaveBeenLastCalledWith(TestFrameworkState.LOG, HookState.POST) + }) +}) diff --git a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts index 25978192..7dcab123 100644 --- a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts @@ -350,38 +350,58 @@ describe('WdioJasmineTestFramework', () => { expect(logSends[2].data.test_uuid).toBe(specUuid) }) + const commandLog = (logEntry: Record) => + framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry, commandLog: true }) + it('keeps a screenshot entry\'s kind on the log path', async () => { reporterTestStart(testStats(), context()) - await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: 't', level: 'INFO' } }) + await commandLog({ kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: 't', level: 'INFO' }) await drain() expect(logSends[0].entries[0].kind).toBe('TEST_SCREENSHOT') }) - it('sends an HTTP command log to the spec it names, even inside an all-hook or after the spec ended', async () => { + it('sends a command log to the last started spec, even inside an all-hook or after the spec ended', async () => { const first = testStats() const uuid = reporterTestStart(first, context()) reporterTestEnd({ ...first, state: 'passed', end: new Date() }, context()) const after = hookStats('"after all" hook') reporterHookStart(after, context()) const message = JSON.stringify({ path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } }) - await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'HTTP', message, timestamp: 't', test_run_uuid: uuid } }) + await commandLog({ kind: 'HTTP', message, timestamp: 't' }) + reporterLog({ level: 'INFO', message: 'console in hook', timestamp: 't2', kind: 'TEST_LOG' }) await drain() - expect(logSends).toHaveLength(1) - expect(logSends[0].state).toBe('TestFrameworkState.TEST') + expect(logSends.map(l => l.state)).toEqual(['TestFrameworkState.TEST', 'TestFrameworkState.AFTER_ALL']) expect(logSends[0].data.test_uuid).toBe(uuid) expect(logSends[0].entries[0]).toMatchObject({ kind: 'HTTP', timestamp: 't' }) expect(logSends[0].entries[0]).not.toHaveProperty('hook_id') expect(Buffer.from(logSends[0].entries[0].message as Uint8Array).toString()).toBe(message) + expect(logSends[1].entries[0]).toHaveProperty('hook_id') }) - it('drops a named-spec log whose uuid this worker never minted', async () => { - reporterTestStart(testStats(), context()) - await framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry: { kind: 'HTTP', message: '{}', timestamp: 't', test_run_uuid: 'not-ours' } }) + it('drops command logs before the first spec, even inside an all-hook', async () => { + reporterHookStart(hookStats('"before all" hook'), context()) + await commandLog({ kind: 'HTTP', message: '{}', timestamp: 't' }) await drain() expect(logSends).toHaveLength(0) }) + it('drops command logs after an until the next spec, while console logs keep the last spec', async () => { + const first = testStats() + reporterTestStart(first, context()) + reporterTestEnd({ ...first, state: 'passed', end: new Date() }, context()) + framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { source: 'reporter', unknownTest: true }) + await commandLog({ kind: 'HTTP', message: '{}', timestamp: 't1' }) + reporterLog({ level: 'INFO', message: 'console', timestamp: 't2', kind: 'TEST_LOG' }) + const second = reporterTestStart(testStats({ uid: 'second', title: 'second', fullTitle: 'Nested outer second' }), context()) + await commandLog({ kind: 'HTTP', message: '{}', timestamp: 't3' }) + await drain() + + expect(logSends.map(l => l.entries[0].timestamp)).toEqual(['t2', 't3']) + expect(logSends[1].data.test_uuid).toBe(second) + expect(dispatches.filter(d => d.state === TestFrameworkState.TEST && d.hook === HookState.PRE)).toHaveLength(2) + }) + it('maps the WDIO hookName to the hook-type key, and nothing else', () => { expect(['beforeAll', 'afterAll', 'beforeEach', 'afterEach'].map(n => WdioJasmineTestFramework.hookTypeFromName(n))) .toEqual(['BEFORE_ALL', 'AFTER_ALL', 'BEFORE_EACH', 'AFTER_EACH']) diff --git a/packages/browserstack-service/tests/insights-handler.test.ts b/packages/browserstack-service/tests/insights-handler.test.ts index 6d081093..ba7888a7 100644 --- a/packages/browserstack-service/tests/insights-handler.test.ts +++ b/packages/browserstack-service/tests/insights-handler.test.ts @@ -10,8 +10,6 @@ import * as utils from '../src/util.js' import * as bstackLogger from '../src/bstackLogger.js' import { TESTOPS_SCREENSHOT_ENV } from '../src/constants.js' import { BrowserstackCLI } from '../src/cli/index.js' -import { TestFrameworkState } from '../src/cli/states/testFrameworkState.js' -import { HookState } from '../src/cli/states/hookState.js' const log = logger('test') let insightsHandler: InsightsHandler @@ -771,22 +769,22 @@ describe('browserCommand', () => { getInstanceSpy = undefined }) - it('sends the HTTP log over gRPC to the named spec, never to the legacy listener', async () => { + it('never routes to the CLI test framework, whose capture lives in TestHubModule', async () => { + process.env[TESTOPS_SCREENSHOT_ENV] = 'true' const trackEvent = vi.fn().mockResolvedValue(undefined) cliWith({ trackEvent }) const handler = handlerFor('jasmine') const logCreated = vi.spyOn(handler['listener'], 'logCreated').mockImplementation(() => {}) + const onScreenshot = vi.spyOn(handler['listener'], 'onScreenshot').mockImplementation(() => {}) + const screenshot = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/screenshot', result: { value: 'b64' } } await handler.browserCommand('client:beforeCommand', { ...command } as any, lastSpec) await handler.browserCommand('client:afterCommand', { ...result } as any, lastSpec) + await handler.browserCommand('client:afterCommand', { ...screenshot } as any, lastSpec) - expect(logCreated).not.toHaveBeenCalled() - expect(trackEvent).toHaveBeenCalledTimes(1) - const [state, hook, { logEntry }] = trackEvent.mock.calls[0] - expect([state, hook]).toEqual([TestFrameworkState.LOG, HookState.POST]) - expect(logEntry.kind).toBe('HTTP') - expect(logEntry.test_run_uuid).toBe('spec-uuid') - expect(JSON.parse(logEntry.message)).toEqual({ path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } }) + expect(trackEvent).not.toHaveBeenCalled() + expect(logCreated).toHaveBeenCalledTimes(1) + expect(onScreenshot).toHaveBeenCalledWith([{ test_run_uuid: 'spec-uuid', timestamp: expect.any(String), message: 'b64', kind: 'TEST_SCREENSHOT' }]) }) it('keeps the legacy HTTP log shape on the listener when the CLI is not running', async () => { @@ -804,23 +802,6 @@ describe('browserCommand', () => { }]) }) - it('names the spec on a jasmine screenshot, and leaves the mocha screenshot entry unchanged', async () => { - process.env[TESTOPS_SCREENSHOT_ENV] = 'true' - const screenshot = { sessionId: 's', method: 'GET', endpoint: '/session/:sessionId/screenshot', result: { value: 'b64' } } - - const jasmineTrack = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent: jasmineTrack }) - await handlerFor('jasmine').browserCommand('client:afterCommand', { ...screenshot } as any, lastSpec) - getInstanceSpy!.mockRestore() - - const mochaTrack = vi.fn().mockResolvedValue(undefined) - cliWith({ trackEvent: mochaTrack }) - await handlerFor('mocha').browserCommand('client:afterCommand', { ...screenshot } as any, { title: 't' } as any) - - expect(jasmineTrack.mock.calls[0][2].logEntry).toEqual({ kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: expect.any(String), test_run_uuid: 'spec-uuid' }) - expect(Object.keys(mochaTrack.mock.calls[0][2].logEntry)).toEqual(['kind', 'message', 'timestamp']) - }) - it('drops commands with no spec yet, as legacy did before the first spec', async () => { const trackEvent = vi.fn().mockResolvedValue(undefined) cliWith({ trackEvent }) @@ -1423,3 +1404,16 @@ describe('hasTestStepFailures and ignoreHooksStatus integration', () => { expect(testInsightsHandlerDefault['_options']?.testObservabilityOptions?.ignoreHooksStatus).toBeUndefined() }) }) + +describe('setTestData', () => { + it('records the current test for Percy and custom tags, and seeds no test entry', () => { + const handler = new InsightsHandler({ on: vi.fn(), sessionId: 's', capabilities: {}, config: {} } as any, 'mocha') + const test = { title: 't', parent: 'Suite' } as any + handler.setTestData(test, 'mocha-uuid') + expect(InsightsHandler.currentTest).toEqual({ test, uuid: 'mocha-uuid' }) + + handler.setTestData({ pickle: { uri: 'f.feature', astNodeIds: ['1'] } } as any, 'scenario-uuid') + expect(InsightsHandler.currentTest).toEqual({ uuid: 'scenario-uuid' }) + expect(handler['_tests']).toEqual({}) + }) +}) diff --git a/packages/browserstack-service/tests/observability-build-start-success-false.test.ts b/packages/browserstack-service/tests/observability-build-start-success-false.test.ts index 418a28df..9361951d 100644 --- a/packages/browserstack-service/tests/observability-build-start-success-false.test.ts +++ b/packages/browserstack-service/tests/observability-build-start-success-false.test.ts @@ -109,8 +109,10 @@ describe('CLI/gRPC flow: startBinSession observability success:false', () => { // TestHubModule has no shouldProcessEventForTesthub-style gate: its send* paths // call GrpcClient unconditionally. - const src = TestHubModule.toString() + Object.getOwnPropertyNames(TestHubModule.prototype) + const src = Object.getOwnPropertyNames(TestHubModule.prototype) + .filter((m) => m.startsWith('send')) .map((m) => (TestHubModule.prototype as any)[m]?.toString?.() ?? '').join('\n') + expect(src).toContain('GrpcClient') expect(src.includes('shouldProcessEventForTesthub')).toBe(false) }) diff --git a/packages/browserstack-service/tests/reporter.test.ts b/packages/browserstack-service/tests/reporter.test.ts index 5f8a169a..e526d156 100644 --- a/packages/browserstack-service/tests/reporter.test.ts +++ b/packages/browserstack-service/tests/reporter.test.ts @@ -507,10 +507,12 @@ describe('test-reporter', () => { expect(hook.state).toBe('passed') }) - it('drops ', async () => { + it('reports only as a marker, never as a spec', async () => { await reporter.onTestStart({ ...jasmineTestStats(), fullTitle: '' } as any) await reporter.onTestEnd({ ...jasmineTestStats(), fullTitle: '' } as any) - expect(framework.trackEvent).not.toHaveBeenCalled() + expect(framework.trackEvent).toHaveBeenCalledExactlyOnceWith(TestFrameworkState.TEST, HookState.PRE, expect.objectContaining({ source: 'reporter', unknownTest: true })) + expect(vi.mocked(framework.trackEvent).mock.calls[0][2]).not.toHaveProperty('testStats') + expect(TestReporter.getTests()['']).toBeUndefined() }) it('sends nothing when Test Observability is opted out', async () => { diff --git a/packages/browserstack-service/tests/service.test.ts b/packages/browserstack-service/tests/service.test.ts index 1a146ab8..0e3d9be4 100644 --- a/packages/browserstack-service/tests/service.test.ts +++ b/packages/browserstack-service/tests/service.test.ts @@ -589,10 +589,7 @@ describe('before', () => { expect(service['_sessionBaseUrl']).toEqual(sessionBaseUrl) }) - it('registers the result event on the CLI path, and only that event', async () => { - // browserCommand is the only producer of TEST_SCREENSHOT logs, so the binary flow has to - // register the result event too (SDK-4177). `command` must stay unregistered — it only - // feeds browserCommand's HTTP-log half, which the binary owns on this path. + it('registers no command or result listener on the CLI path', async () => { process.env.BROWSERSTACK_OBSERVABILITY = 'true' const getInstanceSpy = vi.spyOn(BrowserstackCLI, 'getInstance').mockReturnValue({ isRunning: () => true, @@ -610,7 +607,7 @@ describe('before', () => { await service.before(service['_config'] as any, [], browser) const events = vi.mocked(browser.on).mock.calls.map(([event]) => event) - expect(events).toContain('result') + expect(events).not.toContain('result') expect(events).not.toContain('command') getInstanceSpy.mockRestore() @@ -878,16 +875,10 @@ describe('before', () => { return vi.mocked(cliBrowser.on).mock.calls.map(([event]: [string]) => event) } - it('registers command and result for jasmine', async () => { - const events = await registeredEvents('jasmine') - expect(events).toContain('command') - expect(events).toContain('result') - }) - - it.each(['mocha', 'cucumber'])('keeps %s on result only', async (framework) => { + it.each(['jasmine', 'mocha', 'cucumber'])('leaves %s command capture to TestHubModule', async (framework) => { const events = await registeredEvents(framework) - expect(events).toContain('result') expect(events).not.toContain('command') + expect(events).not.toContain('result') }) }) }) From 7bc03737243559dfcf8bd9b2086afcd558d70c8e Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Thu, 8 Oct 2026 09:09:11 +0530 Subject: [PATCH 18/24] fix(cli): attribute WebDriver command logs as legacy did for jasmine and mocha hooks (SDK-7772) Jasmine command logs follow the spec the service's latest beforeTest or beforeHook named, resolved by full name when the command fires, rather than the last spec the reporter started: a trailing pending or excluded spec never takes the session teardown's commands, and a name with no minted spec (``, or none before the first spec) drops them. The reporter's `` marker goes away. A mocha command log taken while a hook is open carries that hook's id and state, so the binary keys it to the hook run. Console logs are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../frameworks/wdioJasmineTestFramework.ts | 37 +++++++++---- .../cli/frameworks/wdioMochaTestFramework.ts | 10 +++- .../src/cli/modules/testHubModule.ts | 2 +- packages/browserstack-service/src/reporter.ts | 3 -- .../tests/cli/modules/testHubModule.test.ts | 20 +++++++ .../cli/wdioJasmineTestFramework.test.ts | 52 +++++++++++++------ .../wdioMochaTestFramework.logKind.test.ts | 39 ++++++++++++++ .../tests/reporter.test.ts | 6 +-- 8 files changed, 131 insertions(+), 38 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts index 0dc71e4c..69e365f6 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -58,6 +58,14 @@ export default class WdioJasmineTestFramework extends TestFramework { static KEY_HOOK_FAILURE_TYPE = 'hook_failure_type' static KEY_HOOK_FAILURE_REASON = 'hook_failure_reason' + static #commandLogTargetStates = new Set([ + TestFrameworkState.INIT_TEST, + TestFrameworkState.TEST, + TestFrameworkState.BEFORE_ALL, + TestFrameworkState.BEFORE_EACH, + TestFrameworkState.AFTER_EACH, + TestFrameworkState.AFTER_ALL, + ]) static #hookTypes = new Map([ ['beforeAll', 'BEFORE_ALL'], ['afterAll', 'AFTER_ALL'], @@ -75,9 +83,11 @@ export default class WdioJasmineTestFramework extends TestFramework { #hookInstances = new Map() #openHook: TestFrameworkInstance | null = null #lastSpec: TestFrameworkInstance | null = null - // The spec WebDriver command logs attach to: the last started spec, even inside an all-hook or after it - // ended, unset by a `` (the runner's last spec then has no uuid, so its commands drop). - #commandLogSpec: TestFrameworkInstance | null = null + // WebDriver command logs go to the spec the service's latest beforeTest/beforeHook named, resolved by + // fullName when the command fires. A pending or excluded spec runs neither hook, so it never takes them; + // a name with no minted spec (``, or none before the first spec) drops them. + #commandLogFullName: string | undefined + #specsByFullNameMinted = new Map() #queue: Promise = Promise.resolve() #pendingEvents = 0 @@ -124,6 +134,9 @@ export default class WdioJasmineTestFramework extends TestFramework { this.#log(args.logEntry as Record, args.commandLog === true) return } + if (hookState === HookState.PRE && WdioJasmineTestFramework.#commandLogTargetStates.has(testFrameworkState)) { + this.#commandLogFullName = (args.test as { fullName?: string } | undefined)?.fullName + } try { await this.#queue if (testFrameworkState !== TestFrameworkState.INIT_TEST && testFrameworkState !== TestFrameworkState.TEST) { @@ -157,11 +170,7 @@ export default class WdioJasmineTestFramework extends TestFramework { #trackReporterEvent(testFrameworkState: State, hookState: State, args: Record) { const context = args.context as JasmineSuiteContext if (testFrameworkState === TestFrameworkState.TEST) { - if (hookState === HookState.PRE && args.unknownTest === true) { - this.#enqueue('UNKNOWN_TEST', async () => { - this.#commandLogSpec = null - }) - } else if (hookState === HookState.PRE) { + if (hookState === HookState.PRE) { args.testUuid = this.#testStarted(args.testStats as TestStats, context) } else { this.#testEnded(args.testStats as TestStats, context) @@ -196,13 +205,13 @@ export default class WdioJasmineTestFramework extends TestFramework { }) this.#specInstances.set(testStats.uid, instance) this.#specsByFullName.set(fullTitle, instance) + this.#specsByFullNameMinted.set(fullTitle, instance) // Registered synchronously: the service's beforeTest for this spec may read it before the queue runs. TestFramework.setTrackedInstance(instance.getContext(), instance) const args = { test: this.#specArg(testStats, context), suiteTitle: this.#suiteTitle(testStats) } this.#enqueue('TEST/PRE', async () => { this.#lastSpec = instance - this.#commandLogSpec = instance await this.#toTestHub(instance, TestFrameworkState.TEST, HookState.PRE, args) }) return TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID) as string @@ -361,15 +370,21 @@ export default class WdioJasmineTestFramework extends TestFramework { /** * Console logs: an open beforeAll/afterAll wins, else the last-started spec, even after it ended. - * WebDriver command logs (HTTP, screenshots) always go to the command-log spec, as legacy did. + * WebDriver command logs (HTTP, screenshots) go to the command-log spec, as legacy did. */ #log(logEntry: Record | undefined, commandLog = false) { try { if (!logEntry || !shouldProcessEventForTesthub('LogCreated')) { return } + const commandLogSpec = commandLog && this.#commandLogFullName !== undefined + ? this.#specsByFullNameMinted.get(this.#commandLogFullName) + : undefined + if (commandLog && !commandLogSpec) { + return + } this.#enqueue('LOG/POST', async () => { - const instance = commandLog ? this.#commandLogSpec : (this.#openHook ?? this.#lastSpec) + const instance = commandLog ? commandLogSpec : (this.#openHook ?? this.#lastSpec) if (!instance) { return } diff --git a/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts index 8a885982..66f0901b 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts @@ -92,7 +92,7 @@ export default class WdioMochaTestFramework extends TestFramework { } else if (testFrameworkState === TestFrameworkState.LOG) { const logEntry = args.logEntry as Record logEntry.uuid = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOK_ID) - this.loadLogEntries(instance, testFrameworkState, hookState, logEntry) + this.loadLogEntries(instance, testFrameworkState, hookState, logEntry, args.commandLog === true) } else if (testFrameworkState === TestFrameworkState.LOG_REPORT && hookState === HookState.POST) { logger.info('trackEvent: load test results') this.loadTestResult(instance, args) @@ -266,7 +266,7 @@ export default class WdioMochaTestFramework extends TestFramework { * @param hookState HookState * @param args Additional arguments (level, message, etc.) */ - loadLogEntries(instance: TestFrameworkInstance, testFrameworkState: State, hookState: State, logEntry: Record) { + loadLogEntries(instance: TestFrameworkInstance, testFrameworkState: State, hookState: State, logEntry: Record, commandLog = false) { const logRecord: Record = {} const { level, message, timestamp, kind } = logEntry @@ -282,6 +282,12 @@ export default class WdioMochaTestFramework extends TestFramework { // Attach to the suitable hook const lastActiveHook = WdioMochaTestFramework.lastActiveHook(instance, WdioMochaTestFramework.KEY_HOOK_LAST_STARTED) + // The instance is already in LOG here, so a WebDriver command log names the open hook itself: the + // binary keys an entry to a hook run only by a hook state. + if (commandLog && lastActiveHook) { + logRecord[TestFrameworkConstants.KEY_HOOK_ID] = lastActiveHook[TestFrameworkConstants.KEY_HOOK_ID] + logRecord.testFrameworkState = lastActiveHook.key + } if (lastActiveHook) { const hookLogs = lastActiveHook[TestFrameworkConstants.KEY_HOOK_LOGS] as unknown[] hookLogs.push(logRecord) diff --git a/packages/browserstack-service/src/cli/modules/testHubModule.ts b/packages/browserstack-service/src/cli/modules/testHubModule.ts index 606e23bf..616ae753 100644 --- a/packages/browserstack-service/src/cli/modules/testHubModule.ts +++ b/packages/browserstack-service/src/cli/modules/testHubModule.ts @@ -472,7 +472,7 @@ export default class TestHubModule extends BaseModule { const logData: LogCreatedEventRequest_LogEntry = { testFrameworkName, testFrameworkVersion, - testFrameworkState, + testFrameworkState: (logEntry.testFrameworkState as string | undefined) ?? testFrameworkState, uuid: logEntry[TestFrameworkConstants.KEY_HOOK_ID] || TestFramework.getState(instance, TestFrameworkConstants.KEY_TEST_UUID), kind: logEntry.kind as string, message: logEntry.message as Uint8Array, diff --git a/packages/browserstack-service/src/reporter.ts b/packages/browserstack-service/src/reporter.ts index a1b8d2db..1345c8b5 100644 --- a/packages/browserstack-service/src/reporter.ts +++ b/packages/browserstack-service/src/reporter.ts @@ -201,9 +201,6 @@ class _TestReporter extends WDIOReporter { return } if (testStats.fullTitle === '') { - if (this.isCliJasmine()) { - this.cliJasmineFramework()?.trackEvent(TestFrameworkState.TEST, HookState.PRE, { source: 'reporter', unknownTest: true }) - } return } if (this.isCliJasmine()) { diff --git a/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts b/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts index 5a32125d..9650fd36 100644 --- a/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts @@ -621,6 +621,26 @@ describe('TestHubModule', () => { }) }) }) +describe('TestHubModule — sendLogCreatedEvent per-entry state', () => { + it('sends an entry that names its own hook state in that state, and every other entry in the instance\'s', async () => { + const logCreatedEvent = vi.fn().mockResolvedValue(undefined) + vi.mocked(GrpcClient.getInstance).mockReturnValue({ logCreatedEvent } as any) + vi.mocked(TestFramework.getState).mockReturnValue('test-uuid') + const instance = { + getContext: () => ({ getId: () => 'ctx', getThreadId: () => 1, getProcessId: () => 2 }), + getAllData: () => new Map(), + getCurrentTestState: () => TestFrameworkState.LOG, + getCurrentHookState: () => HookState.POST, + } + await new TestHubModule({}).sendLogCreatedEvent({ instance, logEntries: [ + { kind: 'TEST_SCREENSHOT', message: 'a', timestamp: 't', [TestFrameworkConstants.KEY_HOOK_ID]: 'hook-uuid', testFrameworkState: 'BEFORE_EACH' }, + { kind: 'TEST_LOG', message: 'b', timestamp: 't' }, + ] }) + const logs = logCreatedEvent.mock.calls[0][0].logs + expect(logs.map((l: Record) => [l.testFrameworkState, l.uuid])).toEqual([['BEFORE_EACH', 'hook-uuid'], ['LOG', 'test-uuid']]) + }) +}) + describe('TestHubModule — WebDriver command logs', () => { const settle = () => new Promise(resolve => setImmediate(resolve)) const title = { method: 'GET', endpoint: '/session/:sessionId/title', body: {} } diff --git a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts index 7dcab123..d9ba9636 100644 --- a/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/wdioJasmineTestFramework.test.ts @@ -352,20 +352,27 @@ describe('WdioJasmineTestFramework', () => { const commandLog = (logEntry: Record) => framework.trackEvent(TestFrameworkState.LOG, HookState.POST, { logEntry, commandLog: true }) + // The service's beforeTest and beforeHook name the spec WDIO's jasmine adapter last started + const serviceNames = (state: State, fullName?: string) => + framework.trackEvent(state, HookState.PRE, { test: fullName === undefined ? {} : { fullName, description: 'x' } }) + const spec = (title: string) => testStats({ uid: title, title, fullTitle: `Nested outer ${title}` }) it('keeps a screenshot entry\'s kind on the log path', async () => { reporterTestStart(testStats(), context()) + await serviceNames(TestFrameworkState.INIT_TEST, 'Nested outer outer passing test') await commandLog({ kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: 't', level: 'INFO' }) await drain() expect(logSends[0].entries[0].kind).toBe('TEST_SCREENSHOT') }) - it('sends a command log to the last started spec, even inside an all-hook or after the spec ended', async () => { + it('sends a command log to the spec the service last named, even inside an all-hook after it ended', async () => { const first = testStats() const uuid = reporterTestStart(first, context()) + await serviceNames(TestFrameworkState.INIT_TEST, first.fullTitle) reporterTestEnd({ ...first, state: 'passed', end: new Date() }, context()) const after = hookStats('"after all" hook') reporterHookStart(after, context()) + await serviceNames(TestFrameworkState.AFTER_ALL, first.fullTitle) const message = JSON.stringify({ path: '/session/:sessionId/title', method: 'GET', body: {}, response: { value: 'StackDemo' } }) await commandLog({ kind: 'HTTP', message, timestamp: 't' }) reporterLog({ level: 'INFO', message: 'console in hook', timestamp: 't2', kind: 'TEST_LOG' }) @@ -379,27 +386,38 @@ describe('WdioJasmineTestFramework', () => { expect(logSends[1].entries[0]).toHaveProperty('hook_id') }) - it('drops command logs before the first spec, even inside an all-hook', async () => { - reporterHookStart(hookStats('"before all" hook'), context()) - await commandLog({ kind: 'HTTP', message: '{}', timestamp: 't' }) + it('keeps a command after trailing pending and excluded specs on the last spec that ran a hook or test', async () => { + const ran = reporterTestStart(spec('ran'), context()) + await serviceNames(TestFrameworkState.INIT_TEST, 'Nested outer ran') + reporterTestEnd({ ...spec('ran'), state: 'passed', end: new Date() }, context()) + for (const title of ['xit', 'excluded']) { + reporterTestStart(spec(title), context()) + reporterTestEnd({ ...spec(title), state: 'skipped', end: new Date() }, context()) + } + await commandLog({ kind: 'HTTP', message: '{}', timestamp: 'teardown' }) await drain() - expect(logSends).toHaveLength(0) + expect(logSends.map(l => l.data.test_uuid)).toEqual([ran]) }) - it('drops command logs after an until the next spec, while console logs keep the last spec', async () => { - const first = testStats() - reporterTestStart(first, context()) - reporterTestEnd({ ...first, state: 'passed', end: new Date() }, context()) - framework.trackEvent(TestFrameworkState.TEST, HookState.PRE, { source: 'reporter', unknownTest: true }) - await commandLog({ kind: 'HTTP', message: '{}', timestamp: 't1' }) - reporterLog({ level: 'INFO', message: 'console', timestamp: 't2', kind: 'TEST_LOG' }) - const second = reporterTestStart(testStats({ uid: 'second', title: 'second', fullTitle: 'Nested outer second' }), context()) - await commandLog({ kind: 'HTTP', message: '{}', timestamp: 't3' }) + it('resolves a hook\'s copy of the current spec by full name', async () => { + reporterTestStart(spec('a'), context()) + const b = reporterTestStart(spec('b'), context()) + await serviceNames(TestFrameworkState.BEFORE_EACH, 'Nested outer b') + await commandLog({ kind: 'HTTP', message: '{}', timestamp: 't' }) await drain() + expect(logSends.map(l => l.data.test_uuid)).toEqual([b]) + }) - expect(logSends.map(l => l.entries[0].timestamp)).toEqual(['t2', 't3']) - expect(logSends[1].data.test_uuid).toBe(second) - expect(dispatches.filter(d => d.state === TestFrameworkState.TEST && d.hook === HookState.PRE)).toHaveLength(2) + it('drops command logs before the first spec, and for a name no spec was minted under', async () => { + await commandLog({ kind: 'HTTP', message: '{}', timestamp: 'no-hook-yet' }) + reporterHookStart(hookStats('"before all" hook'), context()) + await serviceNames(TestFrameworkState.BEFORE_ALL) + await commandLog({ kind: 'HTTP', message: '{}', timestamp: 'empty-copy' }) + reporterTestStart(testStats(), context()) + await serviceNames(TestFrameworkState.BEFORE_ALL, '') + await commandLog({ kind: 'HTTP', message: '{}', timestamp: 'unknown-test' }) + await drain() + expect(logSends).toHaveLength(0) }) it('maps the WDIO hookName to the hook-type key, and nothing else', () => { diff --git a/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts b/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts index 3053e898..e6bdec83 100644 --- a/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts +++ b/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts @@ -67,3 +67,42 @@ describe('SDK-4177 — loadLogEntries must not relabel a log that carries its ow expect(load({ message: 'hello', timestamp: '2020-01-01T00:00:00.000Z' }).kind).toBe('TEST_LOG') }) }) + +describe('loadLogEntries — WebDriver command logs name the open hook', () => { + let testLogs: unknown[] + let openHook: Record | null + const instance = { getCurrentTestState: () => TestFrameworkState.LOG, updateMultipleEntries: vi.fn() } as any + const screenshot = { kind: TestFrameworkConstants.KIND_SCREENSHOT, message: 'b64', timestamp: 't' } + const consoleLog = { kind: TestFrameworkConstants.KIND_LOG, message: 'hello', level: 'info', timestamp: 't' } + + beforeEach(() => { + testLogs = [] + openHook = { key: 'BEFORE_EACH', [TestFrameworkConstants.KEY_HOOK_ID]: 'hook-uuid', [TestFrameworkConstants.KEY_HOOK_LOGS]: [] } + vi.spyOn(TestFramework, 'getState').mockReturnValue(testLogs) + vi.spyOn(WdioMochaTestFramework, 'lastActiveHook').mockImplementation(() => openHook) + }) + + afterEach(() => { + vi.restoreAllMocks() + }) + + const load = (logEntry: Record, commandLog: boolean) => + WdioMochaTestFramework.prototype.loadLogEntries.call(WdioMochaTestFramework.prototype, instance, TestFrameworkState.LOG, HookState.POST, { ...logEntry }, commandLog) + const hookLogs = () => openHook![TestFrameworkConstants.KEY_HOOK_LOGS] as Record[] + + it('stamps the open hook\'s id and state on a command log', () => { + load(screenshot, true) + expect(hookLogs()[0]).toMatchObject({ kind: 'TEST_SCREENSHOT', [TestFrameworkConstants.KEY_HOOK_ID]: 'hook-uuid', testFrameworkState: 'BEFORE_EACH' }) + }) + + it('leaves a console log in the open hook exactly as before', () => { + load(consoleLog, false) + expect(hookLogs()[0]).toEqual({ kind: 'TEST_LOG', message: Buffer.from('hello'), level: 'info', timestamp: 't' }) + }) + + it('leaves a command log in the test body on the test', () => { + openHook = null + load(screenshot, true) + expect(testLogs[0]).toEqual({ kind: 'TEST_SCREENSHOT', message: Buffer.from('b64'), level: undefined, timestamp: 't' }) + }) +}) diff --git a/packages/browserstack-service/tests/reporter.test.ts b/packages/browserstack-service/tests/reporter.test.ts index e526d156..5f8a169a 100644 --- a/packages/browserstack-service/tests/reporter.test.ts +++ b/packages/browserstack-service/tests/reporter.test.ts @@ -507,12 +507,10 @@ describe('test-reporter', () => { expect(hook.state).toBe('passed') }) - it('reports only as a marker, never as a spec', async () => { + it('drops ', async () => { await reporter.onTestStart({ ...jasmineTestStats(), fullTitle: '' } as any) await reporter.onTestEnd({ ...jasmineTestStats(), fullTitle: '' } as any) - expect(framework.trackEvent).toHaveBeenCalledExactlyOnceWith(TestFrameworkState.TEST, HookState.PRE, expect.objectContaining({ source: 'reporter', unknownTest: true })) - expect(vi.mocked(framework.trackEvent).mock.calls[0][2]).not.toHaveProperty('testStats') - expect(TestReporter.getTests()['']).toBeUndefined() + expect(framework.trackEvent).not.toHaveBeenCalled() }) it('sends nothing when Test Observability is opted out', async () => { From 7f5f89900a1a4d8f4baff8e0dfba082484e90788 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Thu, 8 Oct 2026 09:19:40 +0530 Subject: [PATCH 19/24] fix(cli): read mocha's open hook from the started-hooks map for command logs (SDK-7772) The hooks-started state is a Map, so the record-style lookup never found the open hook and every mocha command log stayed on the test. Command logs now read the open hook from the Map directly; console logs keep their path. Co-Authored-By: Claude Opus 5.5 --- .../cli/frameworks/wdioMochaTestFramework.ts | 19 +++++--- .../wdioMochaTestFramework.logKind.test.ts | 45 +++++++++++-------- 2 files changed, 40 insertions(+), 24 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts index 66f0901b..5da7c524 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts @@ -280,14 +280,16 @@ export default class WdioMochaTestFramework extends TestFramework { logRecord.level = level logRecord.timestamp = timestamp - // Attach to the suitable hook - const lastActiveHook = WdioMochaTestFramework.lastActiveHook(instance, WdioMochaTestFramework.KEY_HOOK_LAST_STARTED) // The instance is already in LOG here, so a WebDriver command log names the open hook itself: the // binary keys an entry to a hook run only by a hook state. - if (commandLog && lastActiveHook) { - logRecord[TestFrameworkConstants.KEY_HOOK_ID] = lastActiveHook[TestFrameworkConstants.KEY_HOOK_ID] - logRecord.testFrameworkState = lastActiveHook.key + const openHook = commandLog ? WdioMochaTestFramework.openHook(instance) : undefined + if (openHook) { + logRecord[TestFrameworkConstants.KEY_HOOK_ID] = openHook[TestFrameworkConstants.KEY_HOOK_ID] + logRecord.testFrameworkState = openHook.key } + + // Attach to the suitable hook + const lastActiveHook = WdioMochaTestFramework.lastActiveHook(instance, WdioMochaTestFramework.KEY_HOOK_LAST_STARTED) if (lastActiveHook) { const hookLogs = lastActiveHook[TestFrameworkConstants.KEY_HOOK_LOGS] as unknown[] hookLogs.push(logRecord) @@ -303,6 +305,13 @@ export default class WdioMochaTestFramework extends TestFramework { }) } + /** The hook started last and not yet finished: a hook's POST pops it from the started list. */ + static openHook(instance: TestFrameworkInstance): Record | undefined { + const key = TestFramework.getState(instance, WdioMochaTestFramework.KEY_HOOK_LAST_STARTED) as string | undefined + const started = TestFramework.getState(instance, TestFrameworkConstants.KEY_HOOKS_STARTED) as Map[]> | undefined + return key && started instanceof Map ? started.get(key)?.at(-1) : undefined + } + /** * Get the last active hook for the given instance and hook key. * @param instance TestFrameworkInstance diff --git a/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts b/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts index e6bdec83..a4a7341a 100644 --- a/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts +++ b/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts @@ -69,40 +69,47 @@ describe('SDK-4177 — loadLogEntries must not relabel a log that carries its ow }) describe('loadLogEntries — WebDriver command logs name the open hook', () => { - let testLogs: unknown[] - let openHook: Record | null - const instance = { getCurrentTestState: () => TestFrameworkState.LOG, updateMultipleEntries: vi.fn() } as any const screenshot = { kind: TestFrameworkConstants.KIND_SCREENSHOT, message: 'b64', timestamp: 't' } const consoleLog = { kind: TestFrameworkConstants.KIND_LOG, message: 'hello', level: 'info', timestamp: 't' } + let state: Record + const instance = { getCurrentTestState: () => TestFrameworkState.LOG, updateMultipleEntries: vi.fn() } as any + + // The instance's real shapes: hooks started is a Map of key -> started-and-not-finished hooks + const startHook = (key: string, hookId: string) => { + const started = state[TestFrameworkConstants.KEY_HOOKS_STARTED] as Map + started.set(key, [...(started.get(key) ?? []), { key, [TestFrameworkConstants.KEY_HOOK_ID]: hookId, [TestFrameworkConstants.KEY_HOOK_LOGS]: [] }]) + state[WdioMochaTestFramework.KEY_HOOK_LAST_STARTED] = key + } + const finishHook = (key: string) => (state[TestFrameworkConstants.KEY_HOOKS_STARTED] as Map).get(key)!.pop() + const load = (logEntry: Record, commandLog: boolean) => + WdioMochaTestFramework.prototype.loadLogEntries.call(WdioMochaTestFramework.prototype, instance, TestFrameworkState.LOG, HookState.POST, { ...logEntry }, commandLog) + const testLogs = () => state[TestFrameworkConstants.KEY_TEST_LOGS] as Record[] beforeEach(() => { - testLogs = [] - openHook = { key: 'BEFORE_EACH', [TestFrameworkConstants.KEY_HOOK_ID]: 'hook-uuid', [TestFrameworkConstants.KEY_HOOK_LOGS]: [] } - vi.spyOn(TestFramework, 'getState').mockReturnValue(testLogs) - vi.spyOn(WdioMochaTestFramework, 'lastActiveHook').mockImplementation(() => openHook) + state = { [TestFrameworkConstants.KEY_HOOKS_STARTED]: new Map(), [TestFrameworkConstants.KEY_TEST_LOGS]: [] } + vi.spyOn(TestFramework, 'getState').mockImplementation((_i, key) => state[key as string]) }) afterEach(() => { vi.restoreAllMocks() }) - const load = (logEntry: Record, commandLog: boolean) => - WdioMochaTestFramework.prototype.loadLogEntries.call(WdioMochaTestFramework.prototype, instance, TestFrameworkState.LOG, HookState.POST, { ...logEntry }, commandLog) - const hookLogs = () => openHook![TestFrameworkConstants.KEY_HOOK_LOGS] as Record[] - it('stamps the open hook\'s id and state on a command log', () => { + startHook('BEFORE_EACH', 'hook-uuid') load(screenshot, true) - expect(hookLogs()[0]).toMatchObject({ kind: 'TEST_SCREENSHOT', [TestFrameworkConstants.KEY_HOOK_ID]: 'hook-uuid', testFrameworkState: 'BEFORE_EACH' }) + expect(testLogs()[0]).toMatchObject({ kind: 'TEST_SCREENSHOT', [TestFrameworkConstants.KEY_HOOK_ID]: 'hook-uuid', testFrameworkState: 'BEFORE_EACH' }) }) - it('leaves a console log in the open hook exactly as before', () => { - load(consoleLog, false) - expect(hookLogs()[0]).toEqual({ kind: 'TEST_LOG', message: Buffer.from('hello'), level: 'info', timestamp: 't' }) + it('leaves a command log on the test once the hook has finished, and in the test body', () => { + startHook('BEFORE_EACH', 'hook-uuid') + finishHook('BEFORE_EACH') + load(screenshot, true) + expect(testLogs()[0]).toEqual({ kind: 'TEST_SCREENSHOT', message: Buffer.from('b64'), level: undefined, timestamp: 't' }) }) - it('leaves a command log in the test body on the test', () => { - openHook = null - load(screenshot, true) - expect(testLogs[0]).toEqual({ kind: 'TEST_SCREENSHOT', message: Buffer.from('b64'), level: undefined, timestamp: 't' }) + it('leaves a console log in an open hook exactly as before', () => { + startHook('BEFORE_EACH', 'hook-uuid') + load(consoleLog, false) + expect(testLogs()[0]).toEqual({ kind: 'TEST_LOG', message: Buffer.from('hello'), level: 'info', timestamp: 't' }) }) }) From 1929968b0360cb2aa78ad78fcdbe51ba29b0d572 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Thu, 8 Oct 2026 10:36:54 +0530 Subject: [PATCH 20/24] feat(cli): report HTTP command logs for mocha and cucumber (SDK-7772) Mocha and cucumber now opt in to HTTP command logs on the CLI flow, as their legacy flows reported them. As on legacy, they are sent only when LogCreated events are on for the run. A mocha command log follows the hook or test that started last, even after it finished, so a command after a hook (such as the session teardown after `after all`) stays on that hook run as it did on the legacy flow. Co-Authored-By: Claude Opus 5.5 --- .../frameworks/wdioCucumberTestFramework.ts | 4 ++ .../cli/frameworks/wdioMochaTestFramework.ts | 25 ++++++--- .../src/cli/modules/testHubModule.ts | 2 +- .../tests/cli/modules/testHubModule.test.ts | 13 +++++ .../cli/wdioCucumberTestFramework.test.ts | 4 ++ .../wdioMochaTestFramework.logKind.test.ts | 53 ++++++++++++++----- 6 files changed, 80 insertions(+), 21 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index 1f988c44..6a60fd33 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -124,6 +124,10 @@ export default class WdioCucumberTestFramework extends TestFramework { logger.debug('WdioCucumberTestFramework: constructed') } + capturesHttpCommandLogs() { + return true + } + /** * Feature bookkeeping. Raises no framework state — cucumber has no feature-level wire event, * and `beforeSuite`/`afterSuite` are not part of its WDIO surface. diff --git a/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts index 5da7c524..29bbe433 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts @@ -35,6 +35,10 @@ export default class WdioMochaTestFramework extends TestFramework { static KEY_HOOK_LAST_STARTED = 'test_hook_last_started' static KEY_HOOK_LAST_FINISHED = 'test_hook_last_finished' + // WebDriver command logs follow the hook or test that started last, even after it finished, as legacy's + // current test did: a hook names its hook run, a test (null) the test. + private commandLogHook: Record | null = null + /** * Constructor for the TestFramework * @param {Array} testFrameworks - List of Test frameworks @@ -45,6 +49,10 @@ export default class WdioMochaTestFramework extends TestFramework { super(testFrameworks, testFrameworkVersions, binSessionId) } + capturesHttpCommandLogs() { + return true + } + /** * Find instance and track any state for the test framework * @param {TestFrameworkState} testFrameworkState @@ -105,6 +113,11 @@ export default class WdioMochaTestFramework extends TestFramework { // test_hooks_started forever (never popped). if (CLIUtils.matchHookRegex(testFrameworkState.toString().split('.')[1])) { await this.trackHookEvents(instance, testFrameworkState, hookState, args) + if (hookState === HookState.PRE) { + this.commandLogHook = WdioMochaTestFramework.openHook(instance) ?? null + } + } else if (testFrameworkState === TestFrameworkState.TEST && hookState === HookState.PRE && args.skipReport !== true) { + this.commandLogHook = null } logger.debug(`trackEvent: tracked instance data=${JSON.stringify(Object.fromEntries(instance.getAllData()))}`) } catch (error) { @@ -280,12 +293,12 @@ export default class WdioMochaTestFramework extends TestFramework { logRecord.level = level logRecord.timestamp = timestamp - // The instance is already in LOG here, so a WebDriver command log names the open hook itself: the - // binary keys an entry to a hook run only by a hook state. - const openHook = commandLog ? WdioMochaTestFramework.openHook(instance) : undefined - if (openHook) { - logRecord[TestFrameworkConstants.KEY_HOOK_ID] = openHook[TestFrameworkConstants.KEY_HOOK_ID] - logRecord.testFrameworkState = openHook.key + // The instance is already in LOG here, so a WebDriver command log names its hook itself: the binary + // keys an entry to a hook run only by a hook state. + const commandLogHook = commandLog ? this.commandLogHook : null + if (commandLogHook) { + logRecord[TestFrameworkConstants.KEY_HOOK_ID] = commandLogHook[TestFrameworkConstants.KEY_HOOK_ID] + logRecord.testFrameworkState = commandLogHook.key } // Attach to the suitable hook diff --git a/packages/browserstack-service/src/cli/modules/testHubModule.ts b/packages/browserstack-service/src/cli/modules/testHubModule.ts index 616ae753..34684455 100644 --- a/packages/browserstack-service/src/cli/modules/testHubModule.ts +++ b/packages/browserstack-service/src/cli/modules/testHubModule.ts @@ -146,7 +146,7 @@ export default class TestHubModule extends BaseModule { commandLog: true, }) } - if (!testFramework.capturesHttpCommandLogs() || !this.seenCommands.has(this.commandKey(browser, result))) { + if (!testFramework.capturesHttpCommandLogs() || !shouldProcessEventForTesthub('LogCreated') || !this.seenCommands.has(this.commandKey(browser, result))) { return } const httpResponse = { path: result.endpoint, method: result.method, body: result.body, response: result.result } diff --git a/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts b/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts index 9650fd36..8915cb84 100644 --- a/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts @@ -760,6 +760,19 @@ describe('TestHubModule — WebDriver command logs', () => { expect(framework.trackEvent).not.toHaveBeenCalled() }) + it('sends no HTTP log when LogCreated events are off for the run (accessibility only), as legacy\'s listener did', async () => { + delete process.env.BROWSERSTACK_OBSERVABILITY + process.env.BROWSERSTACK_ACCESSIBILITY = 'true' + try { + register(true) + await run(title, { value: 'StackDemo' }) + await run(screenshot, { value: 'b64' }) + expect(logEntries().map(e => e.kind)).toEqual(['TEST_SCREENSHOT']) + } finally { + delete process.env.BROWSERSTACK_ACCESSIBILITY + } + }) + it('never throws out of a listener when the framework fails', async () => { register(true) framework.trackEvent.mockRejectedValue(new Error('track blew up')) diff --git a/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts index 2b0a2734..9c8d6e0e 100644 --- a/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts @@ -64,6 +64,10 @@ describe.each([ describe('cucumber WebDriver command logs', () => { const commandLog = { logEntry: { kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: 't' }, commandLog: true } + it('reports HTTP command logs', () => { + expect(new WdioCucumberTestFramework(['WebdriverIO-cucumber'], { 'WebdriverIO-cucumber': '9.0.0' }, 'bin-1').capturesHttpCommandLogs()).toBe(true) + }) + it('drops command logs until a scenario has started, then attributes them as before', async () => { const framework = new WdioCucumberTestFramework(['WebdriverIO-cucumber'], { 'WebdriverIO-cucumber': '9.0.0' }, 'bin-1') const resolveInstance = vi.spyOn(framework as never, 'resolveInstance').mockReturnValue(null as never) diff --git a/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts b/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts index a4a7341a..26b77632 100644 --- a/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts +++ b/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts @@ -68,47 +68,72 @@ describe('SDK-4177 — loadLogEntries must not relabel a log that carries its ow }) }) -describe('loadLogEntries — WebDriver command logs name the open hook', () => { +describe('mocha WebDriver command logs follow the hook or test that started last', () => { const screenshot = { kind: TestFrameworkConstants.KIND_SCREENSHOT, message: 'b64', timestamp: 't' } + const http = { kind: 'HTTP', message: '{}', timestamp: 't' } const consoleLog = { kind: TestFrameworkConstants.KIND_LOG, message: 'hello', level: 'info', timestamp: 't' } let state: Record - const instance = { getCurrentTestState: () => TestFrameworkState.LOG, updateMultipleEntries: vi.fn() } as any + let framework: WdioMochaTestFramework + const instance = { getCurrentTestState: () => TestFrameworkState.LOG, updateMultipleEntries: vi.fn(), getAllData: () => new Map() } as any - // The instance's real shapes: hooks started is a Map of key -> started-and-not-finished hooks const startHook = (key: string, hookId: string) => { const started = state[TestFrameworkConstants.KEY_HOOKS_STARTED] as Map started.set(key, [...(started.get(key) ?? []), { key, [TestFrameworkConstants.KEY_HOOK_ID]: hookId, [TestFrameworkConstants.KEY_HOOK_LOGS]: [] }]) state[WdioMochaTestFramework.KEY_HOOK_LAST_STARTED] = key } const finishHook = (key: string) => (state[TestFrameworkConstants.KEY_HOOKS_STARTED] as Map).get(key)!.pop() + const event = (testFrameworkState: State, hookState: State, args: Record = {}) => framework.trackEvent(testFrameworkState, hookState, args) const load = (logEntry: Record, commandLog: boolean) => - WdioMochaTestFramework.prototype.loadLogEntries.call(WdioMochaTestFramework.prototype, instance, TestFrameworkState.LOG, HookState.POST, { ...logEntry }, commandLog) + framework.loadLogEntries(instance, TestFrameworkState.LOG, HookState.POST, { ...logEntry }, commandLog) const testLogs = () => state[TestFrameworkConstants.KEY_TEST_LOGS] as Record[] beforeEach(() => { - state = { [TestFrameworkConstants.KEY_HOOKS_STARTED]: new Map(), [TestFrameworkConstants.KEY_TEST_LOGS]: [] } + state = { [TestFrameworkConstants.KEY_HOOKS_STARTED]: new Map(), [TestFrameworkConstants.KEY_HOOKS_FINISHED]: new Map(), [TestFrameworkConstants.KEY_TEST_LOGS]: [] } vi.spyOn(TestFramework, 'getState').mockImplementation((_i, key) => state[key as string]) + framework = new WdioMochaTestFramework(['WebdriverIO-mocha'], { 'WebdriverIO-mocha': '9.0.0' }, 'bin') + vi.spyOn(framework, 'resolveInstance').mockReturnValue(instance) + vi.spyOn(framework, 'runHooks').mockResolvedValue(undefined) + vi.spyOn(framework, 'trackHookEvents').mockImplementation(async (_i, testFrameworkState, hookState) => { + const key = testFrameworkState.toString().split('.')[1] + if (hookState === HookState.PRE) { + startHook(key, `${key}-uuid`) + } else { + finishHook(key) + } + }) + vi.spyOn(framework as never, 'getTestData').mockResolvedValue({} as never) }) afterEach(() => { vi.restoreAllMocks() }) - it('stamps the open hook\'s id and state on a command log', () => { - startHook('BEFORE_EACH', 'hook-uuid') - load(screenshot, true) - expect(testLogs()[0]).toMatchObject({ kind: 'TEST_SCREENSHOT', [TestFrameworkConstants.KEY_HOOK_ID]: 'hook-uuid', testFrameworkState: 'BEFORE_EACH' }) + it('reports HTTP command logs', () => { + expect(framework.capturesHttpCommandLogs()).toBe(true) }) - it('leaves a command log on the test once the hook has finished, and in the test body', () => { - startHook('BEFORE_EACH', 'hook-uuid') - finishHook('BEFORE_EACH') + it('stamps a command log in a hook with that hook\'s id and state, even after the hook finished', async () => { + await event(TestFrameworkState.AFTER_ALL, HookState.PRE, { test: {} }) + load(http, true) + await event(TestFrameworkState.AFTER_ALL, HookState.POST, { test: {} }) + load(http, true) + expect(testLogs().map(l => [l[TestFrameworkConstants.KEY_HOOK_ID], l.testFrameworkState])).toEqual([['AFTER_ALL-uuid', 'AFTER_ALL'], ['AFTER_ALL-uuid', 'AFTER_ALL']]) + }) + + it('leaves a command log on the test once a test has started, and a skip report never takes it', async () => { + await event(TestFrameworkState.BEFORE_EACH, HookState.PRE, { test: {} }) + await event(TestFrameworkState.BEFORE_EACH, HookState.POST, { test: {} }) + await event(TestFrameworkState.TEST, HookState.PRE, { test: { title: 't' } }) + load(screenshot, true) + await event(TestFrameworkState.AFTER_EACH, HookState.PRE, { test: {} }) + await event(TestFrameworkState.TEST, HookState.PRE, { test: { title: 'skipped' }, skipReport: true }) load(screenshot, true) expect(testLogs()[0]).toEqual({ kind: 'TEST_SCREENSHOT', message: Buffer.from('b64'), level: undefined, timestamp: 't' }) + expect(testLogs()[1]).toMatchObject({ [TestFrameworkConstants.KEY_HOOK_ID]: 'AFTER_EACH-uuid', testFrameworkState: 'AFTER_EACH' }) }) - it('leaves a console log in an open hook exactly as before', () => { - startHook('BEFORE_EACH', 'hook-uuid') + it('leaves a console log in a hook exactly as before', async () => { + await event(TestFrameworkState.BEFORE_EACH, HookState.PRE, { test: {} }) load(consoleLog, false) expect(testLogs()[0]).toEqual({ kind: 'TEST_LOG', message: Buffer.from('hello'), level: 'info', timestamp: 't' }) }) From 926cfce1430eac17ceb9296024713ed7b2af3f8d Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Thu, 8 Oct 2026 10:44:08 +0530 Subject: [PATCH 21/24] fix(cli): drop mocha command logs from a test's start-up, as legacy did (SDK-7772) Legacy's beforeTest annotated the session before it registered the test, so the annotation's HTTP log never reached Test Observability. The TEST/PRE observers are that start-up on the CLI flow, so mocha now drops command logs until they finish. Co-Authored-By: Claude Opus 5.5 --- .../cli/frameworks/wdioMochaTestFramework.ts | 17 ++++++++++++++++- .../cli/wdioMochaTestFramework.logKind.test.ts | 13 +++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts index 29bbe433..71423b45 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts @@ -38,6 +38,9 @@ export default class WdioMochaTestFramework extends TestFramework { // WebDriver command logs follow the hook or test that started last, even after it finished, as legacy's // current test did: a hook names its hook run, a test (null) the test. private commandLogHook: Record | null = null + // Legacy's beforeTest ran its own work (annotation, accessibility) before it registered the test, so + // commands from a test's start-up dropped; the observers of TEST/PRE are that start-up here. + private holdCommandLogs = false /** * Constructor for the TestFramework @@ -62,6 +65,10 @@ export default class WdioMochaTestFramework extends TestFramework { async trackEvent(testFrameworkState: State, hookState: State, args: Record = {}) { logger.info(`trackEvent: testFrameworkState=${testFrameworkState} hookState=${hookState}`) await super.trackEvent(testFrameworkState, hookState, args) + if (testFrameworkState === TestFrameworkState.LOG && args.commandLog === true && this.holdCommandLogs) { + logger.debug('trackEvent: command log during a test\'s start-up dropped') + return + } const instance = this.resolveInstance(testFrameworkState, hookState, args) if (instance === null) { @@ -118,13 +125,21 @@ export default class WdioMochaTestFramework extends TestFramework { } } else if (testFrameworkState === TestFrameworkState.TEST && hookState === HookState.PRE && args.skipReport !== true) { this.commandLogHook = null + this.holdCommandLogs = true } logger.debug(`trackEvent: tracked instance data=${JSON.stringify(Object.fromEntries(instance.getAllData()))}`) } catch (error) { logger.error(`trackEvent: Error in tracking events: ${error} hookState=${hookState} testFrameworkState=${testFrameworkState}`) } args.instance = instance - await this.runHooks(instance, testFrameworkState, hookState, args) + const releasesCommandLogs = testFrameworkState === TestFrameworkState.TEST && hookState === HookState.PRE && args.skipReport !== true + try { + await this.runHooks(instance, testFrameworkState, hookState, args) + } finally { + if (releasesCommandLogs) { + this.holdCommandLogs = false + } + } } /** diff --git a/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts b/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts index 26b77632..7b701aca 100644 --- a/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts +++ b/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts @@ -132,6 +132,19 @@ describe('mocha WebDriver command logs follow the hook or test that started last expect(testLogs()[1]).toMatchObject({ [TestFrameworkConstants.KEY_HOOK_ID]: 'AFTER_EACH-uuid', testFrameworkState: 'AFTER_EACH' }) }) + it('drops command logs from a test\'s start-up, as legacy did before it registered the test', async () => { + vi.mocked(framework.runHooks).mockImplementation(async (_i, testFrameworkState, hookState) => { + if (testFrameworkState === TestFrameworkState.TEST && hookState === HookState.PRE) { + await event(TestFrameworkState.LOG, HookState.POST, { logEntry: { ...http }, commandLog: true }) + await event(TestFrameworkState.LOG, HookState.POST, { logEntry: { ...consoleLog } }) + } + }) + const loadLogEntries = vi.spyOn(framework, 'loadLogEntries') + await event(TestFrameworkState.TEST, HookState.PRE, { test: { title: 't' } }) + await event(TestFrameworkState.LOG, HookState.POST, { logEntry: { ...http }, commandLog: true }) + expect(loadLogEntries.mock.calls.map(([, , , logEntry, commandLog]) => [logEntry.kind, commandLog])).toEqual([['TEST_LOG', false], ['HTTP', true]]) + }) + it('leaves a console log in a hook exactly as before', async () => { await event(TestFrameworkState.BEFORE_EACH, HookState.PRE, { test: {} }) load(consoleLog, false) From 0eb69dcf4335a06272895b9d3755ae4033aca299 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Thu, 8 Oct 2026 11:09:52 +0530 Subject: [PATCH 22/24] fix(cli): keep mocha command logs outside a hook on the last test that ran (SDK-7772) A skip report moves mocha's tracked instance to the skipped test, so the session teardown's HTTP log landed on a skipped test. A command log outside any hook now goes to the last test that ran, as legacy's current test did. Co-Authored-By: Claude Opus 5.5 --- .../cli/frameworks/wdioMochaTestFramework.ts | 14 +++++++++++++- .../cli/wdioMochaTestFramework.logKind.test.ts | 18 +++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts index 71423b45..5d91456f 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts @@ -38,6 +38,8 @@ export default class WdioMochaTestFramework extends TestFramework { // WebDriver command logs follow the hook or test that started last, even after it finished, as legacy's // current test did: a hook names its hook run, a test (null) the test. private commandLogHook: Record | null = null + // The last test that ran: a skip report moves the tracked instance to the skipped test, never this. + private commandLogTest: TestFrameworkInstance | null = null // Legacy's beforeTest ran its own work (annotation, accessibility) before it registered the test, so // commands from a test's start-up dropped; the observers of TEST/PRE are that start-up here. private holdCommandLogs = false @@ -70,7 +72,7 @@ export default class WdioMochaTestFramework extends TestFramework { return } - const instance = this.resolveInstance(testFrameworkState, hookState, args) + const instance = this.commandLogTestInstance(testFrameworkState, hookState, args) ?? this.resolveInstance(testFrameworkState, hookState, args) if (instance === null) { logger.error(`trackEvent: instance not found for testFrameworkState=${testFrameworkState} hookState=${hookState}`) return @@ -125,6 +127,7 @@ export default class WdioMochaTestFramework extends TestFramework { } } else if (testFrameworkState === TestFrameworkState.TEST && hookState === HookState.PRE && args.skipReport !== true) { this.commandLogHook = null + this.commandLogTest = instance this.holdCommandLogs = true } logger.debug(`trackEvent: tracked instance data=${JSON.stringify(Object.fromEntries(instance.getAllData()))}`) @@ -333,6 +336,15 @@ export default class WdioMochaTestFramework extends TestFramework { }) } + /** A command log outside any hook goes to the last test that ran, in LOG state, as console logs do. */ + private commandLogTestInstance(testFrameworkState: State, hookState: State, args: Record) { + if (testFrameworkState !== TestFrameworkState.LOG || args.commandLog !== true || this.commandLogHook || !this.commandLogTest) { + return null + } + this.updateInstanceState(this.commandLogTest, testFrameworkState, hookState) + return this.commandLogTest + } + /** The hook started last and not yet finished: a hook's POST pops it from the started list. */ static openHook(instance: TestFrameworkInstance): Record | undefined { const key = TestFramework.getState(instance, WdioMochaTestFramework.KEY_HOOK_LAST_STARTED) as string | undefined diff --git a/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts b/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts index 7b701aca..c942f9f3 100644 --- a/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts +++ b/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts @@ -74,7 +74,10 @@ describe('mocha WebDriver command logs follow the hook or test that started last const consoleLog = { kind: TestFrameworkConstants.KIND_LOG, message: 'hello', level: 'info', timestamp: 't' } let state: Record let framework: WdioMochaTestFramework - const instance = { getCurrentTestState: () => TestFrameworkState.LOG, updateMultipleEntries: vi.fn(), getAllData: () => new Map() } as any + const instance = { + getCurrentTestState: () => TestFrameworkState.LOG, getCurrentHookState: () => HookState.POST, updateMultipleEntries: vi.fn(), getAllData: () => new Map(), + setLastTestState: vi.fn(), setLastHookState: vi.fn(), setCurrentTestState: vi.fn(), setCurrentHookState: vi.fn(), + } as any const startHook = (key: string, hookId: string) => { const started = state[TestFrameworkConstants.KEY_HOOKS_STARTED] as Map @@ -145,6 +148,19 @@ describe('mocha WebDriver command logs follow the hook or test that started last expect(loadLogEntries.mock.calls.map(([, , , logEntry, commandLog]) => [logEntry.kind, commandLog])).toEqual([['TEST_LOG', false], ['HTTP', true]]) }) + it('keeps a command log outside any hook on the last test that ran, never a skip report\'s test', async () => { + const ran = { ...instance, setCurrentTestState: vi.fn() } + const skipped = { ...instance } + vi.mocked(framework.resolveInstance).mockReturnValueOnce(ran).mockReturnValue(skipped) + await event(TestFrameworkState.TEST, HookState.PRE, { test: { title: 'ran' } }) + await event(TestFrameworkState.INIT_TEST, HookState.PRE, { test: { title: 'skipped' } }) + await event(TestFrameworkState.TEST, HookState.PRE, { test: { title: 'skipped' }, skipReport: true }) + const loadLogEntries = vi.spyOn(framework, 'loadLogEntries') + await event(TestFrameworkState.LOG, HookState.POST, { logEntry: { ...http }, commandLog: true }) + expect(loadLogEntries.mock.calls[0][0]).toBe(ran) + expect(ran.setCurrentTestState).toHaveBeenCalledWith(TestFrameworkState.LOG) + }) + it('leaves a console log in a hook exactly as before', async () => { await event(TestFrameworkState.BEFORE_EACH, HookState.PRE, { test: {} }) load(consoleLog, false) From 3ec705d329a9a87f5f8249f4204cd751f68402e3 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Thu, 8 Oct 2026 13:57:03 +0530 Subject: [PATCH 23/24] refactor(cli): drop the HTTP command-log opt-in now that every framework reports them (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../src/cli/frameworks/testFramework.ts | 5 --- .../frameworks/wdioCucumberTestFramework.ts | 4 -- .../frameworks/wdioJasmineTestFramework.ts | 4 -- .../cli/frameworks/wdioMochaTestFramework.ts | 4 -- .../src/cli/modules/testHubModule.ts | 16 +++---- .../tests/cli/modules/testHubModule.test.ts | 44 +++++++------------ .../cli/wdioCucumberTestFramework.test.ts | 4 -- .../wdioMochaTestFramework.logKind.test.ts | 4 -- 8 files changed, 23 insertions(+), 62 deletions(-) diff --git a/packages/browserstack-service/src/cli/frameworks/testFramework.ts b/packages/browserstack-service/src/cli/frameworks/testFramework.ts index 70654310..f291505c 100644 --- a/packages/browserstack-service/src/cli/frameworks/testFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/testFramework.ts @@ -73,11 +73,6 @@ export default class TestFramework { return this.testFrameworks } - /** Whether WebDriver HTTP command logs are reported for this framework; screenshots always are. */ - capturesHttpCommandLogs() { - return false - } - /** * Track an event * @param {TestFrameworkState} testFrameworkState diff --git a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts index 6a60fd33..1f988c44 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioCucumberTestFramework.ts @@ -124,10 +124,6 @@ export default class WdioCucumberTestFramework extends TestFramework { logger.debug('WdioCucumberTestFramework: constructed') } - capturesHttpCommandLogs() { - return true - } - /** * Feature bookkeeping. Raises no framework state — cucumber has no feature-level wire event, * and `beforeSuite`/`afterSuite` are not part of its WDIO surface. diff --git a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts index 69e365f6..c03959a8 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioJasmineTestFramework.ts @@ -111,10 +111,6 @@ export default class WdioJasmineTestFramework extends TestFramework { return WdioJasmineTestFramework.#hookState(getHookType(String(hookTitle ?? '').toLowerCase())) ?? TestFrameworkState.NONE } - capturesHttpCommandLogs() { - return true - } - setTestHubModule(testHub: TestHubModule | null | undefined) { this.#testHub = testHub ?? null } diff --git a/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts b/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts index 5d91456f..b5d5831e 100644 --- a/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts +++ b/packages/browserstack-service/src/cli/frameworks/wdioMochaTestFramework.ts @@ -54,10 +54,6 @@ export default class WdioMochaTestFramework extends TestFramework { super(testFrameworks, testFrameworkVersions, binSessionId) } - capturesHttpCommandLogs() { - return true - } - /** * Find instance and track any state for the test framework * @param {TestFrameworkState} testFrameworkState diff --git a/packages/browserstack-service/src/cli/modules/testHubModule.ts b/packages/browserstack-service/src/cli/modules/testHubModule.ts index 34684455..8e4aab57 100644 --- a/packages/browserstack-service/src/cli/modules/testHubModule.ts +++ b/packages/browserstack-service/src/cli/modules/testHubModule.ts @@ -104,7 +104,7 @@ export default class TestHubModule extends BaseModule { } /** - * WebDriver command logs: screenshots for every framework, HTTP command logs for frameworks that opt in. + * WebDriver command logs: screenshots and HTTP command logs. * The framework attributes each entry; `commandLog: true` tells it the entry came from here. */ onDriverCreated(args: Record) { @@ -114,13 +114,11 @@ export default class TestHubModule extends BaseModule { return } this.commandLogBrowsers.add(browser) - if (this.testFramework?.capturesHttpCommandLogs()) { - browser.on('command', (command: WebDriverCommand) => { - if (shouldProcessEventForTesthub('')) { - this.seenCommands.add(this.commandKey(browser, command)) - } - }) - } + browser.on('command', (command: WebDriverCommand) => { + if (shouldProcessEventForTesthub('')) { + this.seenCommands.add(this.commandKey(browser, command)) + } + }) browser.on('result', (result: WebDriverCommand) => { if (shouldProcessEventForTesthub('')) { void this.onCommandResult(browser, result) @@ -146,7 +144,7 @@ export default class TestHubModule extends BaseModule { commandLog: true, }) } - if (!testFramework.capturesHttpCommandLogs() || !shouldProcessEventForTesthub('LogCreated') || !this.seenCommands.has(this.commandKey(browser, result))) { + if (!shouldProcessEventForTesthub('LogCreated') || !this.seenCommands.has(this.commandKey(browser, result))) { return } const httpResponse = { path: result.endpoint, method: result.method, body: result.body, response: result.result } diff --git a/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts b/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts index 8915cb84..53922779 100644 --- a/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/testHubModule.test.ts @@ -647,15 +647,15 @@ describe('TestHubModule — WebDriver command logs', () => { const screenshot = { method: 'GET', endpoint: '/session/:sessionId/screenshot', body: {} } let module: TestHubModule let browser: EventEmitter & { sessionId: string } - let framework: { trackEvent: Mock, capturesHttpCommandLogs: Mock } + let framework: { trackEvent: Mock } const logEntries = () => framework.trackEvent.mock.calls.map(([state, hook, args]) => { expect([state, hook]).toEqual([TestFrameworkState.LOG, HookState.POST]) expect(args.commandLog).toBe(true) return args.logEntry }) - const register = (httpLogs: boolean) => { - framework = { trackEvent: vi.fn().mockResolvedValue(undefined), capturesHttpCommandLogs: vi.fn().mockReturnValue(httpLogs) } + const register = () => { + framework = { trackEvent: vi.fn().mockResolvedValue(undefined) } module.setTestFramework(framework as unknown as TestFramework) module.onDriverCreated({ browser }) } @@ -682,26 +682,14 @@ describe('TestHubModule — WebDriver command logs', () => { expect(AutomationFramework.registerObserver).toHaveBeenCalledWith(expect.anything(), HookState.POST, expect.any(Function)) }) - it('sends a paired HTTP log when the framework opts in, in the legacy message shape', async () => { - register(true) + it('sends a paired HTTP log in the legacy message shape', async () => { + register() await run(title, { value: 'StackDemo' }) expect(logEntries()).toEqual([{ kind: 'HTTP', message: JSON.stringify({ path: title.endpoint, method: 'GET', body: {}, response: { value: 'StackDemo' } }), timestamp: expect.any(String) }]) }) - it('sends no HTTP log, and listens for no command, when the framework does not opt in', async () => { - register(false) - await run(title, { value: 'StackDemo' }) - expect(browser.listenerCount('command')).toBe(0) - expect(framework.trackEvent).not.toHaveBeenCalled() - }) - - it('sends a screenshot for every framework, then the HTTP log when opted in', async () => { - register(false) - await run(screenshot, { value: 'b64' }) - expect(logEntries()).toEqual([{ kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: expect.any(String) }]) - - browser = Object.assign(new EventEmitter(), { sessionId: 's1' }) - register(true) + it('sends the screenshot, then its HTTP log', async () => { + register() await run(screenshot, { value: 'b64' }) expect(logEntries().map(e => e.kind)).toEqual(['TEST_SCREENSHOT', 'HTTP']) }) @@ -712,13 +700,13 @@ describe('TestHubModule — WebDriver command logs', () => { } else { process.env.BS_TESTOPS_ALLOW_SCREENSHOTS = allow } - register(false) + register() await run(screenshot, { value: 'b64' }) - expect(framework.trackEvent).not.toHaveBeenCalled() + expect(logEntries().map(e => e.kind)).toEqual(['HTTP']) }) it('pairs on the live session id, so a reload pairs its own commands and never an earlier session\'s', async () => { - register(true) + register() browser.emit('command', { ...title }) browser.sessionId = 's2' browser.emit('result', { ...title, result: { value: 'old' } }) @@ -730,7 +718,7 @@ describe('TestHubModule — WebDriver command logs', () => { }) it('pairs every result of concurrent same-key commands', async () => { - register(true) + register() browser.emit('command', { ...title }) browser.emit('command', { ...title }) browser.emit('result', { ...title, result: { value: 'a' } }) @@ -740,7 +728,7 @@ describe('TestHubModule — WebDriver command logs', () => { }) it('registers once per browser, so a reload never doubles the logs', async () => { - register(true) + register() module.onDriverCreated({ browser }) await run(title, { value: 'StackDemo' }) expect(browser.listenerCount('result')).toBe(1) @@ -749,12 +737,12 @@ describe('TestHubModule — WebDriver command logs', () => { it('registers nothing, and later sends nothing, when no product reports events', async () => { delete process.env.BROWSERSTACK_OBSERVABILITY - register(true) + register() expect(browser.listenerCount('result')).toBe(0) process.env.BROWSERSTACK_OBSERVABILITY = 'true' browser = Object.assign(new EventEmitter(), { sessionId: 's1' }) - register(true) + register() delete process.env.BROWSERSTACK_OBSERVABILITY await run(title, { value: 'StackDemo' }) expect(framework.trackEvent).not.toHaveBeenCalled() @@ -764,7 +752,7 @@ describe('TestHubModule — WebDriver command logs', () => { delete process.env.BROWSERSTACK_OBSERVABILITY process.env.BROWSERSTACK_ACCESSIBILITY = 'true' try { - register(true) + register() await run(title, { value: 'StackDemo' }) await run(screenshot, { value: 'b64' }) expect(logEntries().map(e => e.kind)).toEqual(['TEST_SCREENSHOT']) @@ -774,7 +762,7 @@ describe('TestHubModule — WebDriver command logs', () => { }) it('never throws out of a listener when the framework fails', async () => { - register(true) + register() framework.trackEvent.mockRejectedValue(new Error('track blew up')) await expect(run(screenshot, { value: 'b64' })).resolves.toBeUndefined() expect(BStackLogger.error).toHaveBeenCalledWith(expect.stringContaining('track blew up')) diff --git a/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts b/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts index 9c8d6e0e..2b0a2734 100644 --- a/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts +++ b/packages/browserstack-service/tests/cli/wdioCucumberTestFramework.test.ts @@ -64,10 +64,6 @@ describe.each([ describe('cucumber WebDriver command logs', () => { const commandLog = { logEntry: { kind: 'TEST_SCREENSHOT', message: 'b64', timestamp: 't' }, commandLog: true } - it('reports HTTP command logs', () => { - expect(new WdioCucumberTestFramework(['WebdriverIO-cucumber'], { 'WebdriverIO-cucumber': '9.0.0' }, 'bin-1').capturesHttpCommandLogs()).toBe(true) - }) - it('drops command logs until a scenario has started, then attributes them as before', async () => { const framework = new WdioCucumberTestFramework(['WebdriverIO-cucumber'], { 'WebdriverIO-cucumber': '9.0.0' }, 'bin-1') const resolveInstance = vi.spyOn(framework as never, 'resolveInstance').mockReturnValue(null as never) diff --git a/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts b/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts index c942f9f3..ef4a2710 100644 --- a/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts +++ b/packages/browserstack-service/tests/cli/wdioMochaTestFramework.logKind.test.ts @@ -111,10 +111,6 @@ describe('mocha WebDriver command logs follow the hook or test that started last vi.restoreAllMocks() }) - it('reports HTTP command logs', () => { - expect(framework.capturesHttpCommandLogs()).toBe(true) - }) - it('stamps a command log in a hook with that hook\'s id and state, even after the hook finished', async () => { await event(TestFrameworkState.AFTER_ALL, HookState.PRE, { test: {} }) load(http, true) From 72ed1894694db1544b2cce20f5b90cc56c402506 Mon Sep 17 00:00:00 2001 From: AdityaHirapara Date: Fri, 9 Oct 2026 10:49:38 +0530 Subject: [PATCH 24/24] refactor(cli): mark only the live jasmine session at worker end (SDK-7772) Co-Authored-By: Claude Opus 5.5 --- .../src/cli/modules/automateModule.ts | 35 ++++++++----------- .../tests/cli/modules/automateModule.test.ts | 26 +++++++++----- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/packages/browserstack-service/src/cli/modules/automateModule.ts b/packages/browserstack-service/src/cli/modules/automateModule.ts index 7da8dbcd..60759fd5 100644 --- a/packages/browserstack-service/src/cli/modules/automateModule.ts +++ b/packages/browserstack-service/src/cli/modules/automateModule.ts @@ -398,7 +398,7 @@ export default class AutomateModule extends BaseModule { // The binary's config echo carries `testObservabilityOptions` empty, so read the worker's own service options const serviceOptions = BrowserstackCLI.getInstance().options as { testObservabilityOptions?: { ignoreHooksStatus?: boolean } } const ignoreHooksStatus = serviceOptions?.testObservabilityOptions?.ignoreHooksStatus === true - await this.markJasmineSessions(this.liveSessionId(), this.jasmineVerdict(args.sessionVerdictInputs, ignoreHooksStatus)) + await this.markJasmineSession(this.liveSessionId(), this.jasmineVerdict(args.sessionVerdictInputs, ignoreHooksStatus)) this.sessionMap.clear() return } @@ -476,28 +476,23 @@ export default class AutomateModule extends BaseModule { /** * Jasmine: legacy `service.after()` marked only the live session, with the worker's status, the last - * name, and the test and hook failure reasons. A session nothing registered (a beforeAll failed before - * any spec ran) is still marked, and a reloaded one is left to the mark `onReload` already sent. + * name and the test and hook failure reasons. A session nothing registered (a beforeAll failed before + * any spec ran) is still marked; a reloaded one keeps the mark `onReload` already sent. */ - private async markJasmineSessions(liveSessionId: string, verdict: SessionVerdict) { + private async markJasmineSession(liveSessionId: string, verdict: SessionVerdict) { const testContextOptions = this.config.testContextOptions as TestContextOptions - const auth = { user: this.config.userName as string, key: this.config.accessKey as string } - const sessionIds = new Set(this.sessionMap.keys()) - if (liveSessionId && isBrowserstackSession(AutomationFramework.getDriver(AutomationFramework.getTrackedInstance()) as WebdriverIO.Browser)) { - sessionIds.add(liveSessionId) + if (!liveSessionId || testContextOptions.skipSessionStatus) { + return } - - for (const sessionId of sessionIds) { - try { - await this.flushSessionName(sessionId) - if (!liveSessionId || sessionId !== liveSessionId || testContextOptions.skipSessionStatus) { - continue - } - const name = testContextOptions.skipSessionName ? undefined : this.sessionMap.get(sessionId)?.lastTestName || undefined - await this.markSessionStatus(sessionId, verdict.status, verdict.reason, auth, name) - } catch (error) { - this.logger.error(`Failed to process session ${sessionId}: ${error}`) - } + const registered = this.sessionMap.get(liveSessionId) + if (!registered && !isBrowserstackSession(AutomationFramework.getDriver(AutomationFramework.getTrackedInstance()) as WebdriverIO.Browser)) { + return + } + const name = testContextOptions.skipSessionName ? undefined : registered?.lastTestName || undefined + try { + await this.markSessionStatus(liveSessionId, verdict.status, verdict.reason, { user: this.config.userName as string, key: this.config.accessKey as string }, name) + } catch (error) { + this.logger.error(`Failed to process session ${liveSessionId}: ${error}`) } } diff --git a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts index aed20e4e..57649f4b 100644 --- a/packages/browserstack-service/tests/cli/modules/automateModule.test.ts +++ b/packages/browserstack-service/tests/cli/modules/automateModule.test.ts @@ -1105,24 +1105,32 @@ describe('AutomateModule — jasmine session verdict', () => { expect(putBodies()).toEqual([['live.json', { status: 'failed', reason: 'beforeAll hook failure' }]]) }) - it('leaves a reloaded session to its onReload mark, but still names it', async () => { - (automateModule as any).sessionMap.set('old', { lastTestName: 'Suite A', testResults: new Map(), scenariosRan: 0 }) + it('marks only the live session, leaving a reloaded one to its onReload mark', async () => { + (automateModule as any).sessionMap.set('old', { lastTestName: 'Suite A', appliedName: 'Suite A', testResults: new Map(), scenariosRan: 0 }) await automateModule.onAfterExecute({ sessionVerdictInputs: inputs() }) - expect(putBodies()).toEqual([ - ['old.json', { name: 'Suite A' }], - ['live.json', { status: 'passed' }], - ]) + expect(putBodies()).toEqual([['live.json', { status: 'passed' }]]) }) - it('marks no session status when no session is live', async () => { + it('marks nothing when no session is live', async () => { vi.mocked(AutomationFramework.getState).mockReturnValue('' as any) - ;(automateModule as any).sessionMap.set('old', { lastTestName: 'Suite A', testResults: new Map(), scenariosRan: 0 }) + ;(automateModule as any).sessionMap.set('old', { lastTestName: 'Suite A', appliedName: 'Suite A', testResults: new Map(), scenariosRan: 0 }) await automateModule.onAfterExecute({ sessionVerdictInputs: inputs({ result: 1 }) }) - expect(putBodies()).toEqual([['old.json', { name: 'Suite A' }]]) + expect(fetch).not.toHaveBeenCalled() + }) + + it('checks the driver only for an unregistered live session', async () => { + vi.mocked(isBrowserstackSession).mockReturnValue(false) + + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs() }) + expect(fetch).not.toHaveBeenCalled() + + ;(automateModule as any).sessionMap.set('live', { lastTestName: 'Suite', appliedName: 'Suite', testResults: new Map(), scenariosRan: 0 }) + await automateModule.onAfterExecute({ sessionVerdictInputs: inputs() }) + expect(putBodies()).toEqual([['live.json', { status: 'passed', name: 'Suite' }]]) }) it('reads ignoreHooksStatus from the worker\'s service options', async () => {