Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ tests/
parse-repo-url.test.ts # GH / GL / BB parsing + edge cases
scorer.test.ts # scoreRepo, topImprovements
badge-adoption.test.ts # detectBadgeEmbed — README badge-embed detection
path-resolution.test.ts # firstExisting / resolveRelative / resolveAllRelative — case-insensitive lookup
signals/ # one *.test.ts per signal
tasks/
README.md
Expand Down Expand Up @@ -142,6 +143,7 @@ Keep it that way when adding features. If a component needs data, fetch in the p
- **Server components** unless interactivity requires client. Prefer `<Link>` + query params over client state.
- **All SQL** lives in `lib/db.ts`. Don't scatter `db.prepare(...)` elsewhere.
- **Signal IDs** are stable strings (`agents_md`, `tests`, etc.). Changing one = migration.
- **Repo path lookups** in `lib/scoring/signals/` go through `firstExisting` / `resolveRelative` / `resolveAllRelative` in `helpers.ts` — never a raw `existsSync(join(repo, …))`. They match case-insensitively because README / LICENSE / CONTRIBUTING casing varies in the wild (`readme.md`, `Readme.md`, `README.MD`); an exact-match lookup scores those files as missing on case-sensitive filesystems, so Linux CI and a macOS dev box disagree on the same commit. `resolveAllRelative` dedupes by resolved path — a candidate list carrying two spellings of one file must not count as two hits.
- **Tailwind first**, then `@theme` tokens. Avoid inline styles; avoid custom classes unless the pattern is truly repeatable.
- **No comments** explaining _what_ the code does. Only comment _why_ — the shallow-clone rationale in `lib/clients/git.ts` is the model.
- **Brand on UI**: "Agent Friendly Code" (no hyphen). Repo/package slug + GitHub `User-Agent` string: `agent-friendly-code`.
Expand Down Expand Up @@ -170,7 +172,7 @@ If either sibling isn't present locally, flag it; never silently skip the propag

## Adding a signal

1. New file at `lib/scoring/signals/<kebab-id>.ts` implementing `Signal` (including `improveSuggestion`).
1. New file at `lib/scoring/signals/<kebab-id>.ts` implementing `Signal` (including `improveSuggestion`). Use the `helpers.ts` resolvers for every path lookup (see Conventions).
2. Import and add to the `SIGNALS` array in `lib/scoring/signals/index.ts`.
3. Add a weight entry to **every** model in `lib/scoring/weights.ts` — missing weights default to 0, decide deliberately.
4. Re-score: `bun run seed` is idempotent.
Expand Down
8 changes: 7 additions & 1 deletion app/methodology/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ export const metadata: Metadata = {
const FAQ = [
{
q: "How is the agent-friendliness score computed?",
a: "Each repository is shallow-cloned and evaluated against sixteen static signals — twelve cross-agent (AGENTS.md / CLAUDE.md, CI, tests, README, linter, type config, license, contributing guide, reproducible dev environment, pre-commit hooks, dependency manifest, codebase size) plus four agent-specific instruction files (`.cursor/rules/*.mdc`, `GEMINI.md`, `.openhands/setup.sh`, `.aider.conf.yml`). Per-model score = Σ(signal.pass × model.weight[signal]) / Σ(model.weight) × 100. Overall score = mean of per-model scores.",
a: "Each repository is shallow-cloned and evaluated against sixteen static signals — twelve cross-agent (AGENTS.md / CLAUDE.md, CI, tests, README, linter, type config, license, contributing guide, reproducible dev environment, pre-commit hooks, dependency manifest, codebase size) plus four agent-specific instruction files (`.cursor/rules/*.mdc`, `GEMINI.md`, `.openhands/setup.sh`, `.aider.conf.yml`). File names are matched case-insensitively, so `readme.md` and `README.MD` count exactly like `README.md`. Per-model score = Σ(signal.pass × model.weight[signal]) / Σ(model.weight) × 100. Overall score = mean of per-model scores.",
},
{
q: "Why score per model instead of giving one overall number?",
Expand Down Expand Up @@ -133,6 +133,12 @@ improvement = closing a gap unlocks (1 - pass) × weight / Σweight × 100
<Panel>
<PanelHeading>Signals ({SIGNALS.length})</PanelHeading>

<p className="mt-2 mb-1 text-[13.5px] leading-relaxed text-ink-dim">
File names are matched case-insensitively —{" "}
<code className="font-mono text-[12.5px] text-ink">readme.md</code> and{" "}
<code className="font-mono text-[12.5px] text-ink">README.MD</code> both count as a README.
</p>

<ul className="m-0 list-none p-0">
{SIGNALS.map((s) => (
<li
Expand Down
38 changes: 22 additions & 16 deletions lib/scoring/signals/ci.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { existsSync, readdirSync, statSync } from "node:fs";
import { readdirSync, statSync } from "node:fs";
import { join } from "node:path";

import { firstExisting } from "./helpers";
import { firstExisting, resolveRelative } from "./helpers";
import type { Signal } from "./types";

const OTHER_CI = [
Expand All @@ -20,20 +20,26 @@ export const ci: Signal = {
improveSuggestion:
"Add a CI workflow (e.g. .github/workflows/ci.yml or .gitlab-ci.yml) that runs tests + linter on every PR.",
check: (repo) => {
const ghWf = join(repo, ".github", "workflows");

if (existsSync(ghWf) && statSync(ghWf).isDirectory()) {
const files = readdirSync(ghWf).filter((f) => /\.ya?ml$/.test(f));

if (files.length > 0) {
return {
pass: 1,
id: "ci",
label: "CI configuration",
matchedPath: ".github/workflows",
detail: `${files.length} GitHub Actions workflow(s)`,
};
}
const ghWf = resolveRelative(repo, ".github/workflows");

if (ghWf) {
const abs = join(repo, ghWf);

try {
if (statSync(abs).isDirectory()) {
const files = readdirSync(abs).filter((f) => /\.ya?ml$/i.test(f));

if (files.length > 0) {
return {
pass: 1,
id: "ci",
matchedPath: ghWf,
label: "CI configuration",
detail: `${files.length} GitHub Actions workflow(s)`,
};
}
}
} catch {}
}

const m = firstExisting(repo, OTHER_CI);
Expand Down
21 changes: 12 additions & 9 deletions lib/scoring/signals/cursor-rules.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { existsSync, readdirSync, statSync } from "node:fs";
import { readdirSync, statSync } from "node:fs";
import { join } from "node:path";

import { resolveRelative } from "./helpers";
import type { Signal } from "./types";

const LABEL = "Cursor rules (.cursor/rules)";
Expand All @@ -12,33 +13,35 @@ export const cursorRules: Signal = {
improveSuggestion:
"Add `.cursor/rules/*.mdc` files describing how Cursor should work in this repo (architecture, conventions, naming). The legacy `.cursorrules` file is still read but is deprecated.",
check: (repo) => {
const dir = join(repo, ".cursor", "rules");
const dir = resolveRelative(repo, ".cursor/rules");

if (dir) {
const abs = join(repo, dir);

if (existsSync(dir)) {
try {
if (statSync(dir).isDirectory()) {
const mdc = readdirSync(dir).filter((f) => f.endsWith(".mdc"));
if (statSync(abs).isDirectory()) {
const mdc = readdirSync(abs).filter((f) => f.toLowerCase().endsWith(".mdc"));

if (mdc.length > 0) {
return {
pass: 1,
label: LABEL,
id: "cursor_rules",
matchedPath: `.cursor/rules/${mdc[0]}`,
matchedPath: `${dir}/${mdc[0]}`,
detail: `${mdc.length} .mdc file${mdc.length === 1 ? "" : "s"} in .cursor/rules/`,
};
}
}
} catch {}
}

const legacy = join(repo, ".cursorrules");
if (existsSync(legacy)) {
const legacy = resolveRelative(repo, ".cursorrules");
if (legacy) {
return {
pass: 0.5,
label: LABEL,
id: "cursor_rules",
matchedPath: ".cursorrules",
matchedPath: legacy,
detail: "Legacy .cursorrules — Cursor still reads it, but `.cursor/rules/*.mdc` is preferred",
};
}
Expand Down
2 changes: 1 addition & 1 deletion lib/scoring/signals/deps-manifest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ const CANDIDATES = [
"vcpkg.json",
];

const GLOB_MANIFESTS: RegExp[] = [/\.(csproj|fsproj|vbproj|sln)$/, /\.cabal$/, /\.nimble$/];
const GLOB_MANIFESTS: RegExp[] = [/\.(csproj|fsproj|vbproj|sln)$/i, /\.cabal$/i, /\.nimble$/i];

function findGlobManifest(repo: string): string | null {
try {
Expand Down
14 changes: 6 additions & 8 deletions lib/scoring/signals/dev-env.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
import { existsSync } from "node:fs";
import { join } from "node:path";

import { readSafe } from "./helpers";
import { readSafe, resolveAllRelative, resolveRelative } from "./helpers";
import type { Signal } from "./types";

const ARTIFACTS = [
"Makefile",
"makefile",
".devcontainer/devcontainer.json",
".devcontainer.json",
"flake.nix",
Expand All @@ -30,7 +28,7 @@ export const devEnv: Signal = {
description: "One-command setup the agent can run (Makefile / devcontainer / Nix / Docker).",
improveSuggestion: "Add a Makefile or devcontainer or Dockerfile so the agent can set up the project in one command.",
check: (repo) => {
const matches = ARTIFACTS.filter((c) => existsSync(join(repo, c)));
const matches = resolveAllRelative(repo, ARTIFACTS);

if (matches.length >= 2) {
return {
Expand All @@ -52,16 +50,16 @@ export const devEnv: Signal = {
};
}

const pkg = join(repo, "package.json");
if (existsSync(pkg)) {
const pkg = resolveRelative(repo, "package.json");
if (pkg) {
try {
const j = JSON.parse(readSafe(pkg));
const j = JSON.parse(readSafe(join(repo, pkg)));

if (j.scripts && Object.keys(j.scripts).length >= 3) {
return {
pass: 0.6,
id: "dev_env",
matchedPath: "package.json",
matchedPath: pkg,
label: "Reproducible dev env",
detail: `package.json has ${Object.keys(j.scripts).length} scripts`,
};
Expand Down
25 changes: 3 additions & 22 deletions lib/scoring/signals/gemini-md.ts
Original file line number Diff line number Diff line change
@@ -1,37 +1,18 @@
import { readdirSync } from "node:fs";
import { join } from "node:path";

import { readSafe } from "./helpers";
import { readSafe, resolveRelative } from "./helpers";
import type { Signal } from "./types";

const LABEL = "GEMINI.md";

function findGeminiMd(repo: string): string | null {
let entries: string[] = [];

try {
entries = readdirSync(repo);
} catch {
return null;
}

for (const e of entries) {
if (e.toLowerCase() === "gemini.md") {
return join(repo, e);
}
}

return null;
}

export const geminiMd: Signal = {
label: LABEL,
id: "gemini_md",
description: "Gemini CLI's canonical hierarchical instructions file — read at every prompt.",
improveSuggestion:
"Add a GEMINI.md at the repo root covering project goals, layout, setup commands, and conventions. Aim for 800+ chars of real guidance (not boilerplate).",
check: (repo) => {
const matched = findGeminiMd(repo);
const matched = resolveRelative(repo, "GEMINI.md");

if (!matched) {
return {
Expand All @@ -42,7 +23,7 @@ export const geminiMd: Signal = {
};
}

const len = readSafe(matched).trim().length;
const len = readSafe(join(repo, matched)).trim().length;
if (len === 0) {
return {
pass: 0.2,
Expand Down
103 changes: 99 additions & 4 deletions lib/scoring/signals/helpers.ts
Original file line number Diff line number Diff line change
@@ -1,12 +1,107 @@
import { existsSync, readdirSync, readFileSync, type Stats, statSync } from "node:fs";
import { readdirSync, readFileSync, type Stats, statSync } from "node:fs";
import { join } from "node:path";

// Path lookups are case-insensitive: README / LICENSE / CONTRIBUTING casing
// varies in the wild (`readme.md`, `Readme.md`, `README.MD`), and exact
// matching scores those files as missing on case-sensitive filesystems.
type DirIndex = {
exact: Set<string>;
lower: Map<string, string>;
};

type DirCache = Map<string, DirIndex | null>;

function indexDir(dir: string, cache: DirCache): DirIndex | null {
const cached = cache.get(dir);

if (cached !== undefined) {
return cached;
}

let index: DirIndex | null = null;

try {
const entries = readdirSync(dir);
const lower = new Map<string, string>();

// Sorted so that when both README.md and readme.md exist, the pick is
// stable across runs instead of following readdir order.
for (const e of [...entries].sort()) {
const key = e.toLowerCase();

if (!lower.has(key)) {
lower.set(key, e);
}
}

index = { lower, exact: new Set(entries) };
} catch {}

cache.set(dir, index);
return index;
}

function resolveWith(repo: string, rel: string, cache: DirCache): string | null {
const parts: string[] = [];
let current = repo;

for (const segment of rel.split("/")) {
if (!segment) {
return null;
}

const index = indexDir(current, cache);
if (!index) {
return null;
}

// Exact spelling wins so it is never shadowed by a differently-cased sibling.
const actual = index.exact.has(segment) ? segment : index.lower.get(segment.toLowerCase());
if (!actual) {
return null;
}

parts.push(actual);
current = join(current, actual);
}

return parts.join("/");
}

// Returns the path as spelled on disk, not as spelled in the candidate list —
// callers surface it as `matchedPath`, so it has to be the real name.
export function resolveRelative(repo: string, rel: string): string | null {
return resolveWith(repo, rel, new Map());
}

// Deduped by resolved path: a candidate list carrying two spellings of one
// file (Makefile / makefile) must not count as two hits.
export function resolveAllRelative(repo: string, candidates: string[]): string[] {
const cache: DirCache = new Map();
const hits = new Set<string>();

for (const c of candidates) {
const hit = resolveWith(repo, c, cache);

if (hit) {
hits.add(hit);
}
}

return [...hits];
}

// Absolute, unlike the resolve* helpers above: callers feed the result straight
// to readSafe, and scorer.ts relativises it before it is stored or rendered.
export function firstExisting(repo: string, candidates: string[]): string | null {
// One directory listing serves every candidate rooted in the same place.
const cache: DirCache = new Map();

for (const c of candidates) {
const p = join(repo, c);
const hit = resolveWith(repo, c, cache);

if (existsSync(p)) {
return p;
if (hit) {
return join(repo, hit);
}
}

Expand Down
9 changes: 4 additions & 5 deletions lib/scoring/signals/linter.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { existsSync } from "node:fs";
import { join } from "node:path";

import { firstExisting, readSafe } from "./helpers";
import { firstExisting, readSafe, resolveRelative } from "./helpers";
import type { Signal } from "./types";

const CANDIDATES = [
Expand Down Expand Up @@ -74,12 +73,12 @@ export const linter: Signal = {
};
}

const pyproject = join(repo, "pyproject.toml");
if (existsSync(pyproject) && PYPROJECT_RE.test(readSafe(pyproject))) {
const pyproject = resolveRelative(repo, "pyproject.toml");
if (pyproject && PYPROJECT_RE.test(readSafe(join(repo, pyproject)))) {
return {
pass: 1,
id: "linter",
matchedPath: "pyproject.toml",
matchedPath: pyproject,
label: "Linter / formatter config",
detail: "Configured in pyproject.toml",
};
Expand Down
Loading
Loading