Skip to content

fix: harden item chooser flow for VoiceOver #47

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cmorten merged 3 commits into from
Aug 16, 2026
Merged

fix: harden item chooser flow for VoiceOver #47

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
938edda
fix: harden item chooser flow for vo
cmorten Aug 16, 2026
2613d7c
fix: full capture on item chooser to find it in spoken phrase
cmorten Aug 16, 2026
6c4252f
fix: also update screen reader test
cmorten Aug 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 106 additions & 24 deletions src/screenReaderTest.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,91 @@ export interface ScreenReaderPlaywright extends ScreenReader {
navigateToWebContent(options?: CaptureCommandOptions): Promise<void>;
}

const closeMenus = async ({
screenReaderPlaywright,
}: {
screenReaderPlaywright: ScreenReaderPlaywright;
}) => {
await screenReaderPlaywright.perform(
{ keyCode: MacOSKeyCodes.Escape },
{ capture: false },
);
};

const cancelCurrentInteraction = async ({
screenReaderPlaywright,
}: {
screenReaderPlaywright: ScreenReaderPlaywright;
}) => {
await screenReaderPlaywright.perform(
{ keyCode: MacOSKeyCodes.Control },
{ capture: false },
);
};

const openItemChooser = async ({
screenReaderPlaywright,
}: {
screenReaderPlaywright: ScreenReaderPlaywright;
}) => {
let lastSpokenPhrase = "";

while (!lastSpokenPhrase.toLowerCase().includes("item chooser")) {
await cancelCurrentInteraction({ screenReaderPlaywright });
await delay(100);
await closeMenus({ screenReaderPlaywright });
await delay(100);

await screenReaderPlaywright.perform(
voiceOverKeyCodeCommands.openItemChooser,
{ capture: true },
);

lastSpokenPhrase = await screenReaderPlaywright.lastSpokenPhrase();
}
};

async function selectWebContentItem({
screenReaderPlaywright,
}: {
screenReaderPlaywright: ScreenReaderPlaywright;
}) {
// eslint-disable-next-line no-constant-condition
while (true) {
let matched = false;

for (const character of "web content") {
await cancelCurrentInteraction({ screenReaderPlaywright });
await delay(100);

await screenReaderPlaywright.type(character, { capture: "initial" });
const lastSpokenPhrase = await screenReaderPlaywright.lastSpokenPhrase();

if (lastSpokenPhrase.toLowerCase().includes("web content")) {
matched = true;

break;
}
}

if (matched) {
break;
}

// Clear the entire Item Chooser search.
await screenReaderPlaywright.perform(
{ keyCode: MacOSKeyCodes.Backspace },
{ capture: false },
);
await delay(100);
}

await screenReaderPlaywright.perform(
{ keyCode: MacOSKeyCodes.Enter },
{ capture: false },
);
}

const screenReaderPlaywright: ScreenReaderPlaywright =
screenReader as ScreenReaderPlaywright;

Expand Down Expand Up @@ -261,14 +346,18 @@ export const screenReaderTest = test.extend<{
screenReaderPlaywright.navigateToWebContent = async ({
capture,
} = {}) => {
const currentSpokenPhraseLog = [
...(await screenReaderPlaywright.spokenPhraseLog()),
];
const currentItemTextLog = [
...(await screenReaderPlaywright.itemTextLog()),
];

// Ensure application is brought to front and focused.
await macOSActivate(applicationName);

// Cancel automatic behaviours/previous commands.
await screenReaderPlaywright.perform(
{ keyCode: MacOSKeyCodes.Control },
{ capture: false },
);
await cancelCurrentInteraction({ screenReaderPlaywright });
await delay(100);

// Ensure the document is ready and focused.
Expand Down Expand Up @@ -301,31 +390,24 @@ export const screenReaderTest = test.extend<{
document.body.prepend(marker);
});

// Open the web item chooser.
await screenReaderPlaywright.perform(
voiceOverKeyCodeCommands.openItemChooser,
{ capture: false },
);
await delay(500);

// Filter by "web content" - currently web content items for all browsers
// are suffixed by "web content".
for (const character of "web content") {
await screenReaderPlaywright.type(character, { capture: false });
await delay(100);
}

// Select the web content window spot.
await screenReaderPlaywright.perform(
{ keyCode: MacOSKeyCodes.Enter },
{ capture: false },
);
await delay(100);
// Open item chooser and select web content.
await openItemChooser({ screenReaderPlaywright });
await selectWebContentItem({ screenReaderPlaywright });

// Navigate into web content.
await screenReaderPlaywright.interact({ capture: false });
await delay(100);

await screenReaderPlaywright.clearSpokenPhraseLog();
await screenReaderPlaywright.clearItemTextLog();

const spokenPhraseLog =
await screenReaderPlaywright.spokenPhraseLog();
const itemTextLog = await screenReaderPlaywright.itemTextLog();

spokenPhraseLog.push(...currentSpokenPhraseLog);
itemTextLog.push(...currentItemTextLog);

// Navigate to the first element of the page using the provided
// capture settings.
await screenReaderPlaywright.next({ capture });
Expand Down
129 changes: 105 additions & 24 deletions src/voiceOverTest.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,91 @@ export interface VoiceOverPlaywright extends VoiceOver {
): Promise<void>;
}

const closeMenus = async ({
voiceOverPlaywright,
}: {
voiceOverPlaywright: VoiceOverPlaywright;
}) => {
await voiceOverPlaywright.perform(
{ keyCode: MacOSKeyCodes.Escape },
{ capture: false },
);
};

const cancelCurrentInteraction = async ({
voiceOverPlaywright,
}: {
voiceOverPlaywright: VoiceOverPlaywright;
}) => {
await voiceOverPlaywright.perform(
{ keyCode: MacOSKeyCodes.Control },
{ capture: false },
);
};

const openItemChooser = async ({
voiceOverPlaywright,
}: {
voiceOverPlaywright: VoiceOverPlaywright;
}) => {
let lastSpokenPhrase = "";

while (!lastSpokenPhrase.toLowerCase().includes("item chooser")) {
await cancelCurrentInteraction({ voiceOverPlaywright });
await delay(100);
await closeMenus({ voiceOverPlaywright });
await delay(100);

await voiceOverPlaywright.perform(
voiceOverPlaywright.keyboardCommands.openItemChooser,
{ capture: true },
);

lastSpokenPhrase = await voiceOverPlaywright.lastSpokenPhrase();
}
};

async function selectWebContentItem({
voiceOverPlaywright,
}: {
voiceOverPlaywright: VoiceOverPlaywright;
}) {
// eslint-disable-next-line no-constant-condition
while (true) {
let matched = false;

for (const character of "web content") {
await cancelCurrentInteraction({ voiceOverPlaywright });
await delay(100);

await voiceOverPlaywright.type(character, { capture: "initial" });
const lastSpokenPhrase = await voiceOverPlaywright.lastSpokenPhrase();

if (lastSpokenPhrase.toLowerCase().includes("web content")) {
matched = true;

break;
}
}

if (matched) {
break;
}

// Clear the entire Item Chooser search.
await voiceOverPlaywright.perform(
{ keyCode: MacOSKeyCodes.Backspace },
{ capture: false },
);
await delay(100);
}

await voiceOverPlaywright.perform(
{ keyCode: MacOSKeyCodes.Enter },
{ capture: false },
);
}

const voiceOverPlaywright: VoiceOverPlaywright =
voiceOver as VoiceOverPlaywright;

Expand Down Expand Up @@ -96,14 +181,18 @@ export const voiceOverTest = test.extend<{
}

voiceOverPlaywright.navigateToWebContent = async ({ capture } = {}) => {
const currentSpokenPhraseLog = [
...(await voiceOverPlaywright.spokenPhraseLog()),
];
const currentItemTextLog = [
...(await voiceOverPlaywright.itemTextLog()),
];

// Ensure application is brought to front and focused.
await macOSActivate(applicationName);

// Cancel automatic behaviours/previous commands.
await voiceOverPlaywright.perform(
{ keyCode: MacOSKeyCodes.Control },
{ capture: false },
);
await cancelCurrentInteraction({ voiceOverPlaywright });
await delay(100);

// Ensure the document is ready and focused.
Expand Down Expand Up @@ -136,31 +225,23 @@ export const voiceOverTest = test.extend<{
document.body.prepend(marker);
});

// Open the web item chooser.
await voiceOverPlaywright.perform(
voiceOverPlaywright.keyboardCommands.openItemChooser,
{ capture: false },
);
await delay(500);

// Filter by "web content" - currently web content items for all browsers
// are suffixed by "web content".
for (const character of "web content") {
await voiceOverPlaywright.type(character, { capture: false });
await delay(100);
}

// Select the web content window spot.
await voiceOverPlaywright.perform(
{ keyCode: MacOSKeyCodes.Enter },
{ capture: false },
);
await delay(100);
// Open item chooser and select web content.
await openItemChooser({ voiceOverPlaywright });
await selectWebContentItem({ voiceOverPlaywright });

// Navigate into web content.
await voiceOverPlaywright.interact({ capture: false });
await delay(100);

await voiceOverPlaywright.clearSpokenPhraseLog();
await voiceOverPlaywright.clearItemTextLog();

const spokenPhraseLog = await voiceOverPlaywright.spokenPhraseLog();
const itemTextLog = await voiceOverPlaywright.itemTextLog();

spokenPhraseLog.push(...currentSpokenPhraseLog);
itemTextLog.push(...currentItemTextLog);

// Navigate to the first element of the page using the provided
// capture settings.
await voiceOverPlaywright.next({ capture });
Expand Down
Loading