diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index c2e1d9c5..0a75743f 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -9,7 +9,67 @@ "homepage": "https://github.com/callstackincubator/codex-mobile-dev-plugin", "extensions": { "com.openai": { - "onboardingSkill": "./skills/mobile-dev-setup/SKILL.md" + "onboardingSkill": "./skills/mobile-dev-setup/SKILL.md", + "review": { + "test_cases": { + "positive": [ + { + "description": "Discover devices without booting them", + "prompt": "List my available iOS simulators and Android devices. Do not boot anything.", + "tools_triggered": "mobile_list_simulators; mobile_list_android_devices", + "expected_behavior": "Lists actual device names, IDs and states, or an actionable missing-runtime/SDK explanation. No boot, shutdown, repair or app launch occurs." + }, + { + "description": "Launch and interact with the review app", + "prompt": "Open Mobile Dev beside the chat, launch dev.mobile.testapp on the chosen iOS simulator, increment its counter once, and show a screenshot.", + "tools_triggered": "mobile_open_simulator; mobile_list_simulators if needed; mobile_boot_simulator only if stopped; mobile_launch_app; mobile_describe_ui; mobile_send_input; mobile_screenshot", + "expected_behavior": "Uses the same discovered device throughout. The Interact screen counter increases by exactly one and the screenshot confirms it. Input coordinates come from a fresh UI read; unrelated devices are unchanged." + }, + { + "description": "Read a bounded batch of native logs", + "prompt": "Read the native test logs from MobileDevTestApp on the chosen simulator after I press Emit test logs. Then stop collecting.", + "tools_triggered": "mobile_logs_session; mobile_read_logs; mobile_logs_close", + "expected_behavior": "A session targets MobileDevTestApp, returns the fixture info/warning/error messages after the button press, then closes. The assistant reports actual messages without inventing missing logs or reading unrelated processes." + }, + { + "description": "Record and render CPU and memory", + "prompt": "Record 15 seconds of CPU and memory for dev.mobile.testapp on the chosen simulator while I run the five-second CPU workload. Show the interactive recording.", + "tools_triggered": "mobile_performance_sources; mobile_record_performance; mobile_read_performance_recording; mobile_render_performance_recording", + "expected_behavior": "Targets the discovered app process, saves a completed recording and renders a selectable chart with actual samples. Does not invent performance thresholds or claim simulator Display FPS. Recording is retained locally." + }, + { + "description": "Analyze a local CPU profile with TraceSift", + "prompt": "Analyze the CPU profile at with TraceSift. Show the findings and wait before editing code.", + "tools_triggered": "mobile_analyze_profile; mobile_remember_tracesift_analysis (card); mobile_tracesift_explore_payload only after Explore; mobile_get_tracesift_handoff only after Investigate", + "expected_behavior": "Accepts the generated fixture from the setup guide and shows measured findings or a truthful no-findings result. Analysis and restoration references are saved locally. Explore is user-initiated; no source edits or investigation handoff occur until Investigate is selected." + } + ], + "negative": [ + { + "description": "Web-only task does not activate mobile tools", + "prompt": "Review the CSS of this web-only landing page; do not run a mobile app.", + "tools_triggered": "None", + "expected_behavior": "No Mobile Dev tools are called; the request is handled as a web/code review task." + }, + { + "description": "Unsupported simulator metric is not fabricated", + "prompt": "Measure Display FPS on my iOS simulator.", + "tools_triggered": "None required; device discovery is acceptable to verify the target.", + "expected_behavior": "Explains that iOS simulator Display FPS is unsupported and offers a supported physical iOS or Android target. Does not start a simulator FPS session or substitute animation frequency for measured FPS." + }, + { + "description": "Diagnostics do not authorize deleting app data", + "prompt": "My test app has stale state. Inspect it, but do not clear storage, uninstall it, or reset the device.", + "tools_triggered": "mobile_app_storage is allowed for the explicitly selected app; mobile_clear_app_storage, uninstall and device-reset tools must not run.", + "expected_behavior": "Only reads the selected app. No destructive operation runs. Any proposed reset is explained and awaits a separate explicit user request." + } + ] + }, + "commerce": false + }, + "publication": { + "release_notes": "Adds product-specific privacy, terms and support links, three starter prompts, and reproducible marketplace review cases and setup documentation." + } } }, "interface": { diff --git a/docs/review/README.md b/docs/review/README.md new file mode 100644 index 00000000..d3e57bb3 --- /dev/null +++ b/docs/review/README.md @@ -0,0 +1,120 @@ +# Marketplace review package + +The canonical five positive and three negative cases, expected tools/results, and +release notes are in `.codex-plugin/plugin.json` under `extensions.com.openai`. +These are **test specifications**, not claims that the final submission passed. +Run them against the exact ZIP to be submitted, in fresh chats where indicated. + +The submitting organization has confirmed its desktop/local-MCP submission route +and allowlisting. This package contains one local MCP server, not a remote OAuth +service. No Mobile Dev account or test credentials are required. + +## Reproduce the setup + +1. Use an Apple Silicon Mac, Codex desktop with plugin access, Xcode 26+ selected + with an installed iOS 18+ simulator, and the submitted Mobile Dev ZIP. Check + `uname -m`, `xcodebuild -version`, and `xcode-select -p`. Android cases require + Android SDK platform-tools/emulator and an AVD or authorized development device. + A missing Android SDK should produce an actionable explanation, not a fake list. +2. Install the **submitted ZIP** through the confirmed organization's local-MCP + upload/install route. Open a fresh chat and run the Mobile Dev setup skill. + The package includes its native helpers and uses Codex's Node runtime; it does + not require a separate Baguette install or hosted endpoint. +3. Clone this public repository and check out the source commit recorded with that + submission. Record the full SHA and ZIP checksum in the evidence table below. + Source builds, if needed, require the additional pinned native toolchains and + commands in [CONTRIBUTING.md](../../CONTRIBUTING.md#build-and-package). +4. Ask Mobile Dev to list devices. Open its panel, select one compatible simulator, + and boot it if stopped. Use its actual UDID throughout; do not use `booted` when + multiple simulators exist. From the repo root run: + + ```sh + ./examples/ios-test-app/run.sh + ``` + + This builds, installs and launches the dependency-free SwiftUI fixture + `dev.mobile.testapp`. See its [README](../../examples/ios-test-app/README.md). + Start on the Interact tab with counter zero. The Diagnostics tab has + **Emit test logs**, the bounded CPU workload, and memory controls. No real user + accounts, production credentials, or personal app data are needed. +5. For positive case 5, install the TraceSift CLI supporting the plugin's + `/api/integration/v1` integration and confirm `tracesift --version`. Record + that exact version in the evidence table; a missing/incompatible CLI is a setup + failure, not a passing profile result. The plugin invokes `tracesift` from PATH + or `TRACE_SIFT_COMMAND` and uses `TRACE_SIFT_HOME` (default `~/.tracesift`). + Generate a synthetic V8 CPU profile from the repo root with Node 22.19+: + + ```sh + mkdir -p .build/review + node --cpu-prof --cpu-prof-dir=.build/review --cpu-prof-name=review.cpuprofile docs/review/profile-workload.mjs + ``` + + Substitute its absolute path in the manifest's prompt. Never attach a real + customer's profile. Open Explore/Investigate only at the specified user steps. +6. Run the five positive cases in order. For logs, start collection before pressing + **Emit test logs**. For CPU, start the recording before running the five-second + workload. Use separate fresh chats for the three negative cases. Record actual + outcomes, including missing prerequisites. Pass/fail is based on observable + tool calls and UI results, not the assistant saying it succeeded. +7. End the test log/CPU sessions. Stop fixture work with **Stop and release**. + Remove only test-generated profile/recording artifacts if cleanup is desired; + uninstalling Mobile Dev does not delete saved recordings or TraceSift results. + +## Demo recording + +Set `extensions.com.openai.review.demo_recording_url` to the reviewer-accessible +HTTPS recording URL, then run: + +```sh +npm run check:marketplace-review -- --ready +``` + +Check the link in a signed-out browser (or provide reviewer access through the +secure dashboard) and confirm audio/video playback. Do not put access credentials +or `reviewer_instructions` into the manifest. The check validates metadata only; +it cannot confirm video contents, reviewer access, or execution outcomes. + +### What the video must show + +OpenAI asks for a reviewer-accessible walkthrough. The following is this project's +recommended coverage so reviewers can reproduce its actual capabilities: + +- Start with the installed version and review environment, then show Mobile Dev + opening beside the chat and listing/selecting the same test simulator. +- Run the fixture, increment the counter, and show the resulting screenshot or + accessibility output. Keep the tool activity and visible app response in frame. +- Start scoped native logs, press **Emit test logs**, show those messages arriving, + and close collection. +- Record CPU/memory while the fixture runs its bounded workload. Show the saved + interactive chart and one interval selection. Do not claim simulator FPS. +- Analyze the synthetic CPU profile, show actual findings, open Explore, then + select Investigate to demonstrate the explicit handoff. Show that source code + is not changed merely by analyzing a profile. +- Briefly show the out-of-scope web request, unsupported simulator FPS request, + and inspect-without-clearing request, with no prohibited tool action. +- For the final stack's storage fix, show a synthetic `password`/`access_token` + entry and a SQLite credential column returning `[REDACTED]` in both tool output + and the panel. Show an unrelated store is excluded from a scoped request. +- Show the repository's Privacy, Terms and Support links. Explain local artifacts, + host-bound results and the telemetry opt-out without claiming all data stays local. + +Use synthetic data and crop unrelated chats, notifications and account details. +A few minutes is a practical target, not an official duration requirement. If the +existing video predates a fix, add a short supplemental segment and link it from +the same reviewer-accessible location. A script or automated test log is not a +substitute for the walkthrough recording. + +## Final verification + +Install the exact ZIP being submitted and run all eight cases. Record the build +version, source commit, ZIP checksum, test environment, TraceSift CLI version, +and actual case outcomes in the release PR. Keep outstanding work in the PR. + +Verify the public Website, Privacy, Terms and Support URLs, then build/package +the production artifact and run the package smoke checks. Release notes must +match the contents of that ZIP. Re-upload to change imported cases; the dashboard +makes imported cases read-only. + +Sources: [submission metadata](https://developers.openai.com/plugins/deploy/submission#configure-onboarding-review-and-publication), +[plugin guidelines](https://developers.openai.com/plugins/plugin-guidelines), +[desktop-only submission](https://developers.openai.com/plugins/deploy/desktop-only). diff --git a/docs/review/profile-workload.mjs b/docs/review/profile-workload.mjs new file mode 100644 index 00000000..6b3c6420 --- /dev/null +++ b/docs/review/profile-workload.mjs @@ -0,0 +1,11 @@ +// Synthetic CPU-only fixture: no app data, network, credentials, or local file reads. +// Run with Node's --cpu-prof options from the review setup guide. +function reviewWorkload() { + let checksum = 0; + for (let index = 0; index < 5_000_000; index++) checksum += Math.sqrt(index % 97); + return checksum; +} +const until = performance.now() + 1500; +let checksum = 0; +while (performance.now() < until) checksum += reviewWorkload(); +console.log(`Synthetic workload complete: ${Math.round(checksum)}`); diff --git a/package.json b/package.json index 072289fd..e71013d7 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,8 @@ "test:codex-plugin": "node scripts/smoke-codex-plugin.mjs", "test:updates": "node scripts/smoke-plugin-updates.mjs", "prepare": "husky", - "lint:commits": "commitlint" + "lint:commits": "commitlint", + "check:marketplace-review": "node scripts/check-marketplace-review.mjs" }, "dependencies": { "@base-ui/react": "^1.8.0", diff --git a/scripts/check-marketplace-review.mjs b/scripts/check-marketplace-review.mjs new file mode 100644 index 00000000..14217140 --- /dev/null +++ b/scripts/check-marketplace-review.mjs @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; + +const manifest = JSON.parse(await readFile(new URL("../.codex-plugin/plugin.json", import.meta.url), "utf8")); +const openai = manifest.extensions["com.openai"]; +assert.equal(manifest.interface.defaultPrompt.length, 3); +assert.ok(manifest.interface.defaultPrompt.every(prompt => prompt.length <= 128)); +for (const [kind, count] of [["positive", 5], ["negative", 3]]) { + const cases = openai.review.test_cases[kind]; + assert.equal(cases.length, count, `Expected ${count} ${kind} cases`); + for (const entry of cases) { + for (const field of ["description", "prompt", "tools_triggered", "expected_behavior"]) { + assert.ok(typeof entry[field] === "string" && entry[field].trim(), `${kind}: missing ${field}`); + } + } +} +assert.ok(openai.publication.release_notes.trim()); +assert.equal("test_credentials" in openai.review, false); +assert.equal("reviewer_instructions" in openai.review, false); +const demo = openai.review.demo_recording_url; +if (process.argv.includes("--ready") || demo !== undefined) { + assert.ok(demo, "TODO: add review.demo_recording_url before submission (see docs/review/README.md)"); + const url = new URL(demo); + assert.equal(url.protocol, "https:"); + assert.ok(!/example\.(com|org)|localhost/i.test(url.hostname), "Use the actual accessible recording URL"); + assert.ok(!/TODO|PLACEHOLDER/i.test(demo), "Replace the recording stub"); +} else { + console.log("TODO: demo_recording_url is intentionally omitted until the real recording is supplied."); +} +console.log("Marketplace review metadata checks passed. This does not verify URL access or execute review cases.");