diff --git a/.codedna b/.codedna index 7daf86b..ee8416a 100644 --- a/.codedna +++ b/.codedna @@ -30,14 +30,6 @@ packages: cross_cutting_patterns: {} agent_sessions: - - agent: "gpt-5" - provider: "openai" - date: "2026-08-21" - session_id: "s_20260821_gitbook_docs" - task: "Redesign documentation like GitBook" - changed: ["docs/install.html","docs/site.css",".codedna"] - visited: ["docs/install.html","docs/site.css",".codedna"] - message: "Reworked the documentation shell with a GitBook-inspired three-column layout, searchable hierarchical sidebar, keyboard shortcut, breadcrumb, page metadata, sticky outline, pagination, feedback, and responsive spacing. HTML/CSS checks, Ruff, and 284 tests pass." - agent: "gpt-5" provider: "openai" date: "2026-08-21" @@ -70,3 +62,11 @@ agent_sessions: changed: ["codedna_tool/languages/php.py","codedna_tool/languages/base.py","codedna_tool/languages/vbnet.py","codedna_tool/languages/__init__.py","codedna_tool/cli.py","integrations/.roorules","integrations/install.sh","integrations/README.md","docs/languages.md","README.md","README-it.md","CHANGELOG.md","tests/test_issues_4_6_7.py","tests/test_docs.py"] visited: ["codedna_tool/languages/php.py","codedna_tool/languages/base.py","codedna_tool/languages/csharp.py","codedna_tool/languages/__init__.py","codedna_tool/cli.py","integrations/.clinerules"] message: "Validated #4/#6/#7 still open. Fixed PHP mixed-HTML injection + full-file has_codedna_header; added VbNetAdapter; added Roo --tools roo. 306 tests pass." + - agent: "composer-2.5" + provider: "cursor" + date: "2026-09-08" + session_id: "s_20260908_001" + task: "Challenge honesty + experimental notes" + changed: ["docs/challenge.md","docs/challenge.it.md","docs/challenge.html","challenge/SUBMISSION_TEMPLATE.md","challenge/SUBMISSION_TEMPLATE.it.md","README.md","README-it.md"] + visited: ["docs/challenge.md","docs/challenge.it.md","docs/challenge.html","challenge/SUBMISSION_TEMPLATE.md","challenge/SUBMISSION_TEMPLATE.it.md","README.md","README-it.md"] + message: "Documented experimental agent×language caveat + honesty/Meet review + public lives in challenge rules." diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aa3478..fe6ba70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All notable changes to CodeDNA will be documented in this file. ### Added -- **CodeDNA Challenge (€200)** — bilingual rules, required `metrics.json` (languages + frameworks + size/files), docs site page + static board (`docs/challenge.html`, `docs/challenge-board.json`). PR opens enrollment (no separate signup). Separate from historical SWE-bench benchmarks. +- **CodeDNA Challenge (€200)** — bilingual rules, required `metrics.json` (languages + frameworks + size/files ≥25 + `install` + `setup.layout` + `bugs_reported`), optional file-localization metrics + optional judge-agent comparison, real-project bar (no toy sites), same-task control vs CodeDNA via two branches/checkouts/twin projects, copy-paste agent/judge prompts + task list, Meet presentation verification, docs site page + live public board (IT default) updated as valid PRs arrive. PR opens enrollment (no separate signup). Separate from historical SWE-bench benchmarks. ### Added diff --git a/README-it.md b/README-it.md index f7bc644..9f0ba38 100644 --- a/README-it.md +++ b/README-it.md @@ -48,9 +48,9 @@ Nessuna infrastruttura. Nessuna pipeline di retrieval. Nessuna memoria esterna. ## CodeDNA Challenge — €200 -Valutazione pubblica sul **tuo** progetto (non SWE-bench). Almeno 10 task con/senza CodeDNA, stack L0/L1/L2 a pari livello, PR con metriche — anche se vanno contro CodeDNA. +Valutazione pubblica sul **tuo** progetto (non SWE-bench). Progetto reale funzionante (≥25 file), stack tecnologico + install CodeDNA dichiarati, stessa metodologia per tutti (due branch / checkout / progetti gemelli), ≥10 **stessi** task con/senza CodeDNA, [prompt da copiare per l’agente](docs/challenge-agent-prompt.it.md), PR con metriche — anche se vanno contro CodeDNA. I bug vanno segnalati (sperimentale tra agenti). Possibile presentazione Meet; la bacheca pubblica si aggiorna quando arrivano PR valide. -- Bacheca: [larens94.github.io/codedna/challenge.html](https://larens94.github.io/codedna/challenge.html) +- Classifica / bacheca: [larens94.github.io/codedna/challenge.html](https://larens94.github.io/codedna/challenge.html) - Regolamento: [`docs/challenge.it.md`](docs/challenge.it.md) (IT) · [`docs/challenge.md`](docs/challenge.md) (EN) - Submission: `challenge//metrics.json` ([esempio](challenge/metrics.example.json)) — **aprire la PR ti iscrive** diff --git a/README.md b/README.md index 1642ae5..f308de8 100644 --- a/README.md +++ b/README.md @@ -48,9 +48,9 @@ No infrastructure. No retrieval pipeline. No external memory. The code carries i ## CodeDNA Challenge — €200 -Public **own-project** evaluation (not SWE-bench). ≥10 tasks with/without CodeDNA, fair L0/L1/L2 stack parity, metrics PR — even if results go against CodeDNA. +Public **own-project** evaluation (not SWE-bench). Real working project (≥25 files), declared tech stack + CodeDNA install, same methodology for everyone (two branches / checkouts / twin projects), ≥10 **same** tasks with/without CodeDNA, copy-paste [agent prompt](docs/challenge-agent-prompt.md), metrics PR — even if results go against CodeDNA. Bugs must be reported (experimental across agents). Meet presentation may be required; public board updates as valid PRs arrive. -- Board: [larens94.github.io/codedna/challenge.html](https://larens94.github.io/codedna/challenge.html) +- Ranking / board: [larens94.github.io/codedna/challenge.html](https://larens94.github.io/codedna/challenge.html) - Rules: [`docs/challenge.md`](docs/challenge.md) (EN) · [`docs/challenge.it.md`](docs/challenge.it.md) (IT) - Submission: `challenge//metrics.json` ([example](challenge/metrics.example.json)) — **opening the PR enrolls you** diff --git a/challenge/AGENT_PROMPT.md b/challenge/AGENT_PROMPT.md new file mode 100644 index 0000000..9899e2e --- /dev/null +++ b/challenge/AGENT_PROMPT.md @@ -0,0 +1,13 @@ +# Agent prompts (copy-paste) + +Canonical prompts (also served on GitHub Pages): + +- Italiano: [`docs/challenge-agent-prompt.it.md`](../docs/challenge-agent-prompt.it.md) +- English: [`docs/challenge-agent-prompt.md`](../docs/challenge-agent-prompt.md) + +Task checklist: + +- Italiano: [`TASKS_TEMPLATE.it.md`](./TASKS_TEMPLATE.it.md) +- English: [`TASKS_TEMPLATE.md`](./TASKS_TEMPLATE.md) + +On the public page use **Copia per il tuo agente** → https://larens94.github.io/codedna/challenge.html diff --git a/challenge/README.md b/challenge/README.md index 09a75e3..a617008 100644 --- a/challenge/README.md +++ b/challenge/README.md @@ -4,19 +4,30 @@ ```text challenge// - metrics.json # REQUIRED + metrics.json # REQUIRED (stack + install + setup.layout + same-task control/codedna results) README.md # optional notes.md # optional ``` -No official signup. Public board (maintainer-updated): [`docs/challenge.html`](../docs/challenge.html) · data [`docs/challenge-board.json`](../docs/challenge-board.json) +No official signup. Public ranking (updated as valid PRs arrive): [`docs/challenge.html`](../docs/challenge.html) · data [`docs/challenge-board.json`](../docs/challenge-board.json) + +**Copy for your agent** + +- Italiano: [`docs/challenge-agent-prompt.it.md`](../docs/challenge-agent-prompt.it.md) +- English: [`docs/challenge-agent-prompt.md`](../docs/challenge-agent-prompt.md) +- Judge (optional, after both sessions): [`docs/challenge-judge-prompt.it.md`](../docs/challenge-judge-prompt.it.md) · [EN](../docs/challenge-judge-prompt.md) +- Task lists: [`TASKS_TEMPLATE.it.md`](./TASKS_TEMPLATE.it.md) · [`TASKS_TEMPLATE.md`](./TASKS_TEMPLATE.md) +- Pointer: [`AGENT_PROMPT.md`](./AGENT_PROMPT.md) + +Methodology (same for everyone): same ≥10 tasks with vs without CodeDNA via `two_branches` / `two_checkouts` / `two_projects`. +File-localization fields (`files_expected`, F1, …) are **optional** — use when known; otherwise a judge agent can still compare sessions. **Metrics format** - Example: [`metrics.example.json`](./metrics.example.json) - Schema: [`metrics.schema.json`](./metrics.schema.json) -Required project fields: `languages`, `frameworks`, `approx_source_files`, `size_band` (`S`/`M`/`L`/`XL`). +Required: `languages`, `frameworks`, `approx_source_files` (≥25), `size_band`, `install`, `setup.layout`, `bugs_reported`. **Rules** diff --git a/challenge/SUBMISSION_TEMPLATE.it.md b/challenge/SUBMISSION_TEMPLATE.it.md index 2e8c30b..b529cf7 100644 --- a/challenge/SUBMISSION_TEMPLATE.it.md +++ b/challenge/SUBMISSION_TEMPLATE.it.md @@ -7,12 +7,38 @@ ### Partecipante - Handle: -- Issue di iscrizione collegata: - Modalità challenge: - [ ] Parity (stessi L1/L2 su Control e CodeDNA) - [ ] Dichiarata `codedna-only` vs stack superiore -### Stack sotto test +### Stack tecnologico (obbligatorio) + +- Linguaggio/i: +- Framework (obbligatorio — non solo il linguaggio): +- Agente / modello: +- File sorgente approx. (≥ **25**): +- Size band (`S`/`M`/`L`/`XL`): +- Note stack (DB, monorepo, infra — opzionale): +- URL pubblico (opzionale) / “privato — solo metriche”: +- [ ] È un **progetto reale e funzionante** (non sito giocattolo / hello-world / demo throwaway) + +### Install CodeDNA (obbligatorio) + +- Agente usato con CodeDNA: +- Passi esatti di install / init (incolla i comandi): +- Valore di `codedna install --tools` (se usato): +- Install + annotation ok? sì / no — note: + +### Setup Control vs CodeDNA (obbligatorio) + +- [ ] `two_branches` (es. `challenge/control` + `challenge/codedna`) +- [ ] `two_checkouts` (due cartelle) +- [ ] `two_projects` (progetti gemelli) +- Ref Control / CodeDNA: +- [ ] Prompt agente usato: [`docs/challenge-agent-prompt.it.md`](../docs/challenge-agent-prompt.it.md) +- [ ] Lista task congelata: [`TASKS_TEMPLATE.it.md`](./TASKS_TEMPLATE.it.md) + +### Stack sotto test (L0/L1/L2) | Livello | Tool / file | Presente in Control? | Presente in CodeDNA? | |---|---|---|---| @@ -20,23 +46,20 @@ | L1 | | | | | L2 | | | | -### Progetto - -- Linguaggio/i: -- Framework (obbligatorio — non solo il linguaggio): -- File sorgente approx.: -- Size band (`S`/`M`/`L`/`XL`): -- URL pubblico (opzionale) / “privato — solo metriche”: - ### Metrics JSON (obbligatorio) - [ ] Presente `challenge//metrics.json` - [ ] Partito da [`metrics.example.json`](./metrics.example.json) / conforme a [`metrics.schema.json`](./metrics.schema.json) - [ ] `schema_version` = `"1.0"` -- [ ] Compilati `project.languages` + `project.frameworks` + `approx_source_files` + `size_band` +- [ ] Compilati `project.languages` + `project.frameworks` + `approx_source_files` (≥25) + `size_band` +- [ ] Compilati `install.agent` + `install.steps` +- [ ] Compilato `setup.layout` (`two_branches` | `two_checkouts` | `two_projects`) +- [ ] Presente `bugs_reported` (array vuoto OK se nessuno) - [ ] ≥10 task con mix `difficulty` (easy ≥3, medium ≥3, hard ≥2) -- [ ] Ogni task ha risultati `control` e `codedna` +- [ ] Gli **stessi task** hanno risultati `control` e `codedna` - [ ] `summary.favors` impostato (`codedna` | `control` | `tie` | `inconclusive`) +- [ ] (Opzionale) `files_expected` / file F1 solo se conosciuti — non inventati +- [ ] (Opzionale) giudice usato: [`docs/challenge-judge-prompt.it.md`](../docs/challenge-judge-prompt.it.md) → `judge` / `tasks[].judge` - [ ] I risultati possono favorire **oppure** sfavorire CodeDNA (onestà OK) ### Opzionale @@ -47,9 +70,13 @@ ### Bug trovati in CodeDNA - Link (oppure “nessuno”) — elencarli anche in `metrics.json` → `bugs_reported`: +- [ ] Se qualcosa si è rotto su agente/linguaggio, ho aperto issue o PR di fix ### Checklist - [ ] Nessun secret / sorgente proprietaria in questa PR - [ ] Solo log redatti - [ ] Modalità + parity di stack dichiarate nel JSON (`mode`, `stack`) +- [ ] Metriche e narrativa sono veritiere (non inventate) +- [ ] Posso presentare progetto e processo di test in una call di review (Meet / simile) se richiesto +- [ ] Capisco che claim inventati o progetti finti = squalifica diff --git a/challenge/SUBMISSION_TEMPLATE.md b/challenge/SUBMISSION_TEMPLATE.md index 71b2333..8745274 100644 --- a/challenge/SUBMISSION_TEMPLATE.md +++ b/challenge/SUBMISSION_TEMPLATE.md @@ -7,12 +7,38 @@ ### Entrant - Handle: -- Linked entry issue: - Challenge mode: - [ ] Parity (same L1/L2 on Control and CodeDNA) - [ ] Declared `codedna-only` vs higher stack -### Stack under test +### Tech stack (required) + +- Language(s): +- Framework(s) (required — not language alone): +- Agent / model: +- Approx. source files (≥ **25**): +- Size band (`S`/`M`/`L`/`XL`): +- Tech stack notes (DB, monorepo, infra — optional): +- Public URL (optional) / “private — metrics only”: +- [ ] This is a **real working project** (not a toy site / hello-world / throwaway demo) + +### CodeDNA install (required) + +- Agent used with CodeDNA: +- Exact install / init steps (paste commands): +- `codedna install --tools` value (if any): +- Did install + annotation work? yes / no — notes: + +### Control vs CodeDNA setup (required) + +- [ ] `two_branches` (e.g. `challenge/control` + `challenge/codedna`) +- [ ] `two_checkouts` (two folders) +- [ ] `two_projects` (twin projects) +- Control / CodeDNA refs: +- [ ] Agent prompt used: [`docs/challenge-agent-prompt.md`](../docs/challenge-agent-prompt.md) +- [ ] Frozen task list: [`TASKS_TEMPLATE.md`](./TASKS_TEMPLATE.md) + +### Stack under test (L0/L1/L2) | Level | Tools / files | Present in Control? | Present in CodeDNA? | |---|---|---|---| @@ -20,23 +46,20 @@ | L1 | | | | | L2 | | | | -### Project - -- Language(s): -- Framework(s) (required — not language alone): -- Approx. source files: -- Size band (`S`/`M`/`L`/`XL`): -- Public URL (optional) / “private — metrics only”: - ### Metrics JSON (required) - [ ] `challenge//metrics.json` present - [ ] Copied from [`metrics.example.json`](./metrics.example.json) / matches [`metrics.schema.json`](./metrics.schema.json) - [ ] `schema_version` = `"1.0"` -- [ ] `project.languages` + `project.frameworks` + `approx_source_files` + `size_band` filled +- [ ] `project.languages` + `project.frameworks` + `approx_source_files` (≥25) + `size_band` filled +- [ ] `install.agent` + `install.steps` filled +- [ ] `setup.layout` filled (`two_branches` | `two_checkouts` | `two_projects`) +- [ ] `bugs_reported` present (empty array OK if none) - [ ] ≥10 tasks with `difficulty` mix (easy ≥3, medium ≥3, hard ≥2) -- [ ] Each task has both `control` and `codedna` results +- [ ] **Same tasks** each have both `control` and `codedna` results - [ ] `summary.favors` set (`codedna` | `control` | `tie` | `inconclusive`) +- [ ] (Optional) `files_expected` / file F1 only when known — never invented +- [ ] (Optional) judge used: [`docs/challenge-judge-prompt.md`](../docs/challenge-judge-prompt.md) → `judge` / `tasks[].judge` - [ ] Results may favor **or** disfavor CodeDNA (honesty OK) ### Optional @@ -47,9 +70,13 @@ ### Bugs found in CodeDNA - Links (or “none”) — also list them in `metrics.json` → `bugs_reported`: +- [ ] If something broke on my agent/language, I filed an issue or fix PR ### Checklist - [ ] No secrets / proprietary source in this PR - [ ] Redacted logs only - [ ] Mode + stack parity declared in JSON (`mode`, `stack`) +- [ ] Metrics and narrative are truthful (not invented) +- [ ] I can present the project and test process on a review call (Meet / similar) if asked +- [ ] I understand fabricated claims or fake projects = disqualification diff --git a/challenge/TASKS_TEMPLATE.it.md b/challenge/TASKS_TEMPLATE.it.md new file mode 100644 index 0000000..ac391c9 --- /dev/null +++ b/challenge/TASKS_TEMPLATE.it.md @@ -0,0 +1,35 @@ +# CodeDNA Challenge — template lista task + +> Congela questa lista **prima** delle run cronometrate. Esegui gli **stessi** ID su Control e su CodeDNA. +> Prompt completo per l’agente: [`docs/challenge-agent-prompt.it.md`](../docs/challenge-agent-prompt.it.md) + +## Setup (scegline uno) + +- [ ] `two_branches` — es. `challenge/control` + `challenge/codedna` +- [ ] `two_checkouts` — due cartelle dello stesso progetto +- [ ] `two_projects` — progetti gemelli dallo stesso codice di partenza + +Agente / modello: _______________ +Note layout: _______________ + +## Task (≥10 — easy ≥3, medium ≥3, hard ≥2) + +| ID | Difficoltà | Titolo (specifico del progetto) | Control ok? | CodeDNA ok? | Note | +|---|---|---|---|---|---| +| E1 | easy | | | | | +| E2 | easy | | | | | +| E3 | easy | | | | | +| M1 | medium | | | | | +| M2 | medium | | | | | +| M3 | medium | | | | | +| H1 | hard | | | | | +| H2 | hard | | | | | +| T9 | easy/medium/hard | | | | | +| T10 | easy/medium/hard | | | | | + +Extra opzionali: E4, M4, H3… + +## Dopo le run + +- [ ] `metrics.json` compilato (`install`, `setup.layout`, `bugs_reported`, `summary.favors`) +- [ ] PR aperta con submission template diff --git a/challenge/TASKS_TEMPLATE.md b/challenge/TASKS_TEMPLATE.md new file mode 100644 index 0000000..7e4a0b6 --- /dev/null +++ b/challenge/TASKS_TEMPLATE.md @@ -0,0 +1,35 @@ +# CodeDNA Challenge — task list template + +> Freeze this list **before** timed runs. Run the **same** IDs on Control and on CodeDNA. +> Italiano: adatta i titoli al tuo repo; scheletro anche in [`docs/challenge-agent-prompt.it.md`](../docs/challenge-agent-prompt.it.md). + +## Setup (pick one) + +- [ ] `two_branches` — e.g. `challenge/control` + `challenge/codedna` +- [ ] `two_checkouts` — two folders of the same project +- [ ] `two_projects` — twin projects from the same starting code + +Agent / model: _______________ +Layout notes: _______________ + +## Tasks (≥10 — easy ≥3, medium ≥3, hard ≥2) + +| ID | Difficulty | Title (project-specific) | Control pass? | CodeDNA pass? | Notes | +|---|---|---|---|---|---| +| E1 | easy | | | | | +| E2 | easy | | | | | +| E3 | easy | | | | | +| M1 | medium | | | | | +| M2 | medium | | | | | +| M3 | medium | | | | | +| H1 | hard | | | | | +| H2 | hard | | | | | +| T9 | easy/medium/hard | | | | | +| T10 | easy/medium/hard | | | | | + +Optional extras: E4, M4, H3… + +## After runs + +- [ ] `metrics.json` filled (`install`, `setup.layout`, `bugs_reported`, `summary.favors`) +- [ ] PR opened with submission template diff --git a/challenge/metrics.example.json b/challenge/metrics.example.json index fb036ff..7f3be9e 100644 --- a/challenge/metrics.example.json +++ b/challenge/metrics.example.json @@ -44,7 +44,8 @@ "NestJS", "FastAPI" ], - "size_band": "M" + "size_band": "M", + "tech_stack_notes": "NestJS API + FastAPI workers, Postgres, monorepo apps/api + apps/worker" }, "success_definition": "Task passes if the agent produced a correct change without the human rewriting the core logic.", "tasks": [ @@ -60,7 +61,30 @@ "tool_calls": 12, "wrong_file_or_module": false, "human_interventions": 0, - "confidence_1_to_5": 4 + "confidence_1_to_5": 4, + "outcome": "pass", + "files_opened": [ + "apps/api/src/billing/formatAmount.ts", + "apps/api/src/billing/invoice.service.ts", + "apps/api/src/users/user.service.ts", + "apps/api/src/billing/invoice.service.spec.ts" + ], + "files_edited": [ + "apps/api/src/billing/formatAmount.ts", + "apps/api/src/billing/invoice.service.ts" + ], + "files_missed": [ + "apps/api/src/billing/invoice.service.spec.ts" + ], + "files_extra": [], + "file_precision": 1.0, + "file_recall": 0.667, + "file_f1": 0.8, + "first_relevant_file_turn": 3, + "opened_before_first_edit": 2, + "tests_green": false, + "rules_followed": true, + "human_rewrote_core": false }, "codedna": { "passed": true, @@ -69,9 +93,46 @@ "tool_calls": 8, "wrong_file_or_module": false, "human_interventions": 0, - "confidence_1_to_5": 5 + "confidence_1_to_5": 5, + "outcome": "pass", + "files_opened": [ + "apps/api/src/billing/formatAmount.ts", + "apps/api/src/billing/invoice.service.ts", + "apps/api/src/billing/invoice.service.spec.ts" + ], + "files_edited": [ + "apps/api/src/billing/formatAmount.ts", + "apps/api/src/billing/invoice.service.ts", + "apps/api/src/billing/invoice.service.spec.ts" + ], + "files_missed": [], + "files_extra": [], + "file_precision": 1.0, + "file_recall": 1.0, + "file_f1": 1.0, + "first_relevant_file_turn": 1, + "opened_before_first_edit": 1, + "tests_green": true, + "rules_followed": true, + "human_rewrote_core": false }, - "notes": "" + "notes": "", + "files_expected": [ + "apps/api/src/billing/formatAmount.ts", + "apps/api/src/billing/invoice.service.ts", + "apps/api/src/billing/invoice.service.spec.ts" + ], + "judge": { + "agent": "cursor", + "prompt_ref": "docs/challenge-judge-prompt.it.md", + "favors": "codedna", + "score_control_1_to_5": 3, + "score_codedna_1_to_5": 5, + "navigation_notes": "Control opened an unrelated user service file first.", + "correctness_notes": "Control missed updating the unit test.", + "files_assessment": "CodeDNA edited all three expected files; Control missed the spec.", + "raw_verdict": "CodeDNA clearer navigation and complete caller/test updates." + } }, { "id": "T02", @@ -311,7 +372,44 @@ "control_total_turns": 212, "codedna_total_turns": 125, "favors": "codedna", - "one_line_verdict": "CodeDNA reduced wrong-file mistakes on hard tasks; still honest about one medium task needing a steer." + "one_line_verdict": "CodeDNA reduced wrong-file mistakes on hard tasks; still honest about one medium task needing a steer.", + "control_pass_rate": 0.7, + "codedna_pass_rate": 0.9, + "delta_pass_rate": 0.2, + "control_file_f1_avg": 0.8, + "codedna_file_f1_avg": 1.0, + "delta_file_f1": 0.2, + "control_wrong_file_rate": 0.3, + "codedna_wrong_file_rate": 0.0, + "control_total_interventions": 8, + "codedna_total_interventions": 2, + "favors_basis": "judge + pass_rate + file_f1" }, - "bugs_reported": [] + "bugs_reported": [], + "install": { + "agent": "cursor", + "tools_flag": "cursor", + "steps": "pipx install git+https://github.com/Larens94/codedna.git\ncodedna install --path . --tools cursor\ncodedna init . --no-llm", + "worked": true, + "notes": "Install OK on Cursor; no adapter issues on this NestJS/FastAPI stack." + }, + "setup": { + "layout": "two_branches", + "control_ref": "challenge/control", + "codedna_ref": "challenge/codedna", + "notes": "Same commit base; CodeDNA branch has codedna init annotations" + }, + "judge": { + "used": true, + "agent": "cursor", + "model": "claude-sonnet", + "prompt_ref": "docs/challenge-judge-prompt.it.md", + "method": "per_task", + "favors": "codedna", + "tasks_judged": 10, + "codedna_wins": 7, + "control_wins": 1, + "ties": 2, + "notes": "Judge compared diffs/logs after both sessions; did not need files_expected on every task." + } } diff --git a/challenge/metrics.schema.json b/challenge/metrics.schema.json index 61e5692..8f9e3a3 100644 --- a/challenge/metrics.schema.json +++ b/challenge/metrics.schema.json @@ -10,8 +10,11 @@ "mode", "stack", "project", + "install", + "setup", "tasks", - "summary" + "summary", + "bugs_reported" ], "properties": { "schema_version": { @@ -154,8 +157,8 @@ }, "approx_source_files": { "type": "integer", - "minimum": 1, - "description": "Approximate number of source files in the project under test." + "minimum": 25, + "description": "Approximate number of source files. Minimum 25. Toy landing pages / hello-world demos are rejected; must be a real working project (small is OK)." }, "visibility": { "type": "string", @@ -193,6 +196,13 @@ "XL" ], "description": "S=<50 files, M=50-199, L=200-999, XL=1000+" + }, + "tech_stack_notes": { + "type": [ + "string", + "null" + ], + "description": "Optional free-text stack notes (DB, infra, monorepo layout, etc.)." } } }, @@ -241,6 +251,90 @@ }, "notes": { "type": "string" + }, + "files_expected": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "description": "Optional ground-truth files for this task. Omit if unknown before/during the run. Needed only to compute precision/recall/F1." + }, + "judge": { + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "description": "Optional judge-agent comparison of Control vs CodeDNA for this task.", + "properties": { + "agent": { + "type": "string", + "description": "Judge agent/model used." + }, + "prompt_ref": { + "type": [ + "string", + "null" + ], + "description": "Link or path to the judge prompt used." + }, + "favors": { + "type": "string", + "enum": [ + "codedna", + "control", + "tie", + "inconclusive" + ] + }, + "score_control_1_to_5": { + "type": [ + "integer", + "null" + ], + "minimum": 1, + "maximum": 5 + }, + "score_codedna_1_to_5": { + "type": [ + "integer", + "null" + ], + "minimum": 1, + "maximum": 5 + }, + "navigation_notes": { + "type": [ + "string", + "null" + ] + }, + "correctness_notes": { + "type": [ + "string", + "null" + ] + }, + "files_assessment": { + "type": [ + "string", + "null" + ], + "description": "Judge notes on right/wrong/missed files even without a predeclared expected list." + }, + "raw_verdict": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "favors" + ] } } } @@ -314,6 +408,89 @@ }, "one_line_verdict": { "type": "string" + }, + "control_pass_rate": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1 + }, + "codedna_pass_rate": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1 + }, + "delta_pass_rate": { + "type": [ + "number", + "null" + ] + }, + "control_file_f1_avg": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1, + "description": "Optional average file_f1 over tasks where measurable." + }, + "codedna_file_f1_avg": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1 + }, + "delta_file_f1": { + "type": [ + "number", + "null" + ] + }, + "control_wrong_file_rate": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1, + "description": "Optional share of tasks with missed/extra or wrong_file_or_module." + }, + "codedna_wrong_file_rate": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1 + }, + "control_total_interventions": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "codedna_total_interventions": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "favors_basis": { + "type": [ + "string", + "null" + ], + "description": "Optional note: human / judge / pass_rate+file_f1 / etc." } } }, @@ -334,7 +511,164 @@ } } }, - "default": [] + "default": [], + "description": "Bugs found in CodeDNA during the challenge. Empty array if none. If you hit agent/language issues you MUST file issues or fix PRs and list them here." + }, + "install": { + "type": "object", + "additionalProperties": false, + "required": [ + "agent", + "steps" + ], + "description": "How CodeDNA was installed for this run (required \u2014 publish your install path).", + "properties": { + "agent": { + "type": "string", + "minLength": 1, + "description": "AI coding agent used (e.g. cursor, claude-code, opencode, codex)." + }, + "tools_flag": { + "type": [ + "string", + "null" + ], + "description": "Value passed to `codedna install --tools`, if any." + }, + "steps": { + "type": "string", + "minLength": 1, + "description": "Exact install/init commands and steps you ran." + }, + "worked": { + "type": "boolean", + "description": "Whether install + annotation completed successfully on this agent/stack." + }, + "notes": { + "type": "string", + "description": "Problems, workarounds, agent-specific gaps." + } + } + }, + "setup": { + "type": "object", + "additionalProperties": false, + "required": [ + "layout" + ], + "description": "How Control vs CodeDNA workspaces were separated.", + "properties": { + "layout": { + "type": "string", + "enum": [ + "two_branches", + "two_checkouts", + "two_projects" + ], + "description": "two_branches | two_checkouts | two_projects" + }, + "control_ref": { + "type": [ + "string", + "null" + ], + "description": "Branch name, path, or URL for Control" + }, + "codedna_ref": { + "type": [ + "string", + "null" + ], + "description": "Branch name, path, or URL for CodeDNA" + }, + "notes": { + "type": "string" + } + } + }, + "judge": { + "type": [ + "object", + "null" + ], + "additionalProperties": false, + "description": "Optional overall judge-agent review across sessions (Control vs CodeDNA).", + "properties": { + "used": { + "type": "boolean" + }, + "agent": { + "type": "string" + }, + "model": { + "type": [ + "string", + "null" + ] + }, + "prompt_ref": { + "type": [ + "string", + "null" + ] + }, + "method": { + "type": "string", + "enum": [ + "per_task", + "batch", + "spot_check" + ], + "description": "How the judge reviewed the runs." + }, + "favors": { + "type": "string", + "enum": [ + "codedna", + "control", + "tie", + "inconclusive" + ] + }, + "tasks_judged": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "codedna_wins": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "control_wins": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "ties": { + "type": [ + "integer", + "null" + ], + "minimum": 0 + }, + "notes": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "used", + "favors" + ] } }, "$defs": { @@ -385,6 +719,127 @@ ], "minimum": 1, "maximum": 5 + }, + "outcome": { + "description": "Optional finer outcome; if set, passed should match outcome==pass.", + "anyOf": [ + { + "type": "string", + "enum": [ + "pass", + "partial", + "fail", + "inconclusive" + ] + }, + { + "type": "null" + } + ] + }, + "files_opened": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "description": "Optional. Files opened/read during the run." + }, + "files_edited": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "description": "Optional. Files actually modified." + }, + "files_missed": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "description": "Optional. Expected files not edited (only if files_expected known)." + }, + "files_extra": { + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "description": "Optional. Edited files not in expected (only if files_expected known)." + }, + "file_precision": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1, + "description": "Optional. |edited \u2229 expected| / |edited|." + }, + "file_recall": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1, + "description": "Optional. |edited \u2229 expected| / |expected|." + }, + "file_f1": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 1, + "description": "Optional. Harmonic mean of file_precision and file_recall." + }, + "first_relevant_file_turn": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "description": "Optional. Turn when the first relevant/expected file was opened." + }, + "opened_before_first_edit": { + "type": [ + "integer", + "null" + ], + "minimum": 0, + "description": "Optional. Files opened before the first useful edit." + }, + "tests_green": { + "type": [ + "boolean", + "null" + ], + "description": "Optional. Relevant tests green after the change." + }, + "rules_followed": { + "type": [ + "boolean", + "null" + ], + "description": "Optional. Applicable CodeDNA Rules: constraints followed." + }, + "human_rewrote_core": { + "type": [ + "boolean", + "null" + ], + "description": "Optional. Human had to rewrite core logic after the agent." } } } diff --git a/docs/challenge-agent-prompt.it.md b/docs/challenge-agent-prompt.it.md new file mode 100644 index 0000000..fafaf3b --- /dev/null +++ b/docs/challenge-agent-prompt.it.md @@ -0,0 +1,107 @@ +# Prompt agente — CodeDNA Challenge (€200) + +> Incolla questo intero messaggio nel tuo agente AI (Cursor, Claude Code, OpenCode, Codex, …). +> Pagina classifica: https://larens94.github.io/codedna/challenge.html +> Regolamento: https://github.com/Larens94/codedna/blob/main/docs/challenge.it.md +> English: [challenge-agent-prompt.md](challenge-agent-prompt.md) + +--- + +## Ruolo + +Aiutami a eseguire la **CodeDNA Challenge** sul **mio progetto reale** in modo onesto e riproducibile. Non inventare metriche. Se qualcosa non è misurabile, marca `inconclusive`. + +## Metodologia (uguale per tutti) + +Confrontiamo gli **stessi ≥10 task** in due condizioni: + +| Condizione | Cosa | +|---|---| +| **A — Control** | Workflow AI **senza** CodeDNA (niente header / senza affidarsi a CodeDNA) | +| **B — CodeDNA** | Stesso agente/modello **con** CodeDNA installato e annotato | + +### Setup ammesso (scegline uno e dichiaralo) + +1. **Due branch** nello stesso repo — es. `challenge/control` e `challenge/codedna` +2. **Due checkout / due cartelle** dello stesso progetto +3. **Due progetti gemelli** (stesso codice di partenza) + +Regole: + +- Gli **stessi task** in A e in B (stessa lista, stesso ordine se possibile). +- Stesso agente, stesso modello, stessi layer L1/L2 (wiki/skills/Graphify) salvo modalità `codedna_only` dichiarata. +- Progetto **reale e funzionante** (≥25 file sorgente). Niente sito vetrina / hello-world. + +## Stack da dichiarare + +Annota e poi metti in `metrics.json`: + +- linguaggi, framework (non solo il linguaggio), approx file, size band S/M/L/XL +- agente + modello +- come hai installato CodeDNA (`install.steps`) +- layout usato: `two_branches` | `two_checkouts` | `two_projects` + +## Install CodeDNA (condizione B) + +```bash +pipx install git+https://github.com/Larens94/codedna.git +codedna install --path . --tools +codedna init . --no-llm +``` + +Se fallisce: riproduci, apri issue o PR di fix su Larens94/codedna, elenca in `bugs_reported`. È sperimentale — gap agente×linguaggio sono attesi. + +## Lista attività (adattale al mio repo, poi congelale) + +Crea **≥10 task** sul mio codice. Mix obbligatorio: easy ≥3, medium ≥3, hard ≥2. + +Usa questa checklist come scheletro (sostituisci i titoli con task **reali** del mio progetto): + +### Easy (≥3) + +- [ ] **E1** — Rename di un simbolo + aggiornare tutti i caller +- [ ] **E2** — Aggiungere un campo/DTO/prop con validazione +- [ ] **E3** — Fix bug chiaro con file già noto +- [ ] **E4** (opz.) — Aggiornare un test esistente dopo un rename + +### Medium (≥3) + +- [ ] **M1** — Feature piccola cross-file (API + service + test) +- [ ] **M2** — Refactor con invariante da non rompere +- [ ] **M3** — Cambio firma / contratto e aggiornamento caller +- [ ] **M4** (opz.) — Aggiungere logging/metriche in un percorso esistente + +### Hard (≥2) + +- [ ] **H1** — Bug multi-modulo (“dove cambio questo in sicurezza?”) +- [ ] **H2** — Vincolo architetturale / auth / multi-tenant / confini di package +- [ ] **H3** (opz.) — Migrazione o cambio schema con impatto su più layer + +Per **ogni** task registra Control e CodeDNA: `passed`, `minutes`/`turns`/`tool_calls` se disponibili, `wrong_file_or_module`, `human_interventions`, note. + +**Opzionale (non obbligatorio):** + +- `files_expected` solo se li conosci; altrimenti ometti (niente precision/recall inventate) +- `files_opened` / `files_edited` se riesci a traccarli +- Dopo entrambe le sessioni: usa un **agente giudice** con [`challenge-judge-prompt.it.md`](challenge-judge-prompt.it.md) e salva `tasks[].judge` + eventuale `judge` top-level + +## Deliverable + +1. Congela la lista task **prima** delle run cronometrate. +2. Esegui tutti i task in **A**, poi gli **stessi** in **B** (o interleaved, ma stessi ID). +3. (Consigliato) Fai giudicare le due sessioni da un agente separato col prompt giudice. +4. Compila `challenge//metrics.json` da + https://github.com/Larens94/codedna/blob/main/challenge/metrics.example.json + (schema: `metrics.schema.json`). +5. Apri PR su `Larens94/codedna` con titolo: + `challenge: — CodeDNA Challenge submission` + Checklist: `challenge/SUBMISSION_TEMPLATE.it.md` +6. Onestà: niente risultati inventati. Possibile Meet di verifica. + +## Cosa fare ora + +1. Ispeziona il mio repo e proponi la lista ≥10 task adattata (easy/medium/hard). +2. Chiedimi quale setup uso: **due branch** / **due checkout** / **due progetti**. +3. Prepara i comandi di install per la condizione B. +4. Solo dopo conferma mia: esegui i task e compila le metriche. +5. Dopo le run: proponi il passaggio giudice (prompt già pronto) se voglio confrontare le sessioni. diff --git a/docs/challenge-agent-prompt.md b/docs/challenge-agent-prompt.md new file mode 100644 index 0000000..30ee114 --- /dev/null +++ b/docs/challenge-agent-prompt.md @@ -0,0 +1,107 @@ +# Agent prompt — CodeDNA Challenge (€200) + +> Paste this whole message into your AI coding agent (Cursor, Claude Code, OpenCode, Codex, …). +> Ranking page: https://larens94.github.io/codedna/challenge.html +> Rules: https://github.com/Larens94/codedna/blob/main/docs/challenge.md +> Italiano: [challenge-agent-prompt.it.md](challenge-agent-prompt.it.md) + +--- + +## Role + +Help me run the **CodeDNA Challenge** on **my real project** honestly and reproducibly. Do not invent metrics. If something is not measurable, mark it `inconclusive`. + +## Methodology (same for everyone) + +We compare the **same ≥10 tasks** under two conditions: + +| Condition | What | +|---|---| +| **A — Control** | Normal AI workflow **without** CodeDNA (no headers / do not rely on CodeDNA) | +| **B — CodeDNA** | Same agent/model **with** CodeDNA installed and annotated | + +### Allowed setup (pick one and declare it) + +1. **Two branches** in the same repo — e.g. `challenge/control` and `challenge/codedna` +2. **Two checkouts / two folders** of the same project +3. **Two twin projects** (same starting code) + +Rules: + +- The **same tasks** in A and B (same list, same order if possible). +- Same agent, same model, same L1/L2 layers (wiki/skills/Graphify) unless `codedna_only` mode is declared. +- **Real working project** (≥25 source files). No toy landing page / hello-world. + +## Stack to declare + +Capture, then put in `metrics.json`: + +- languages, frameworks (not language alone), approx files, size band S/M/L/XL +- agent + model +- how CodeDNA was installed (`install.steps`) +- layout used: `two_branches` | `two_checkouts` | `two_projects` + +## Install CodeDNA (condition B) + +```bash +pipx install git+https://github.com/Larens94/codedna.git +codedna install --path . --tools +codedna init . --no-llm +``` + +If it fails: reproduce, open an issue or fix PR on Larens94/codedna, list it in `bugs_reported`. Experimental — agent×language gaps are expected. + +## Activity list (adapt to my repo, then freeze) + +Create **≥10 tasks** on my code. Required mix: easy ≥3, medium ≥3, hard ≥2. + +Use this checklist as a skeleton (replace titles with **real** tasks from my project): + +### Easy (≥3) + +- [ ] **E1** — Rename a symbol and update all callers +- [ ] **E2** — Add a field/DTO/prop with validation +- [ ] **E3** — Fix a clear bug with a known file +- [ ] **E4** (opt.) — Update an existing test after a rename + +### Medium (≥3) + +- [ ] **M1** — Small cross-file feature (API + service + test) +- [ ] **M2** — Refactor with an invariant that must not break +- [ ] **M3** — Signature / contract change and update callers +- [ ] **M4** (opt.) — Add logging/metrics on an existing path + +### Hard (≥2) + +- [ ] **H1** — Multi-module bug (“where do I change this safely?”) +- [ ] **H2** — Architectural / auth / multi-tenant / package-boundary constraint +- [ ] **H3** (opt.) — Migration or schema change across layers + +For **each** task record Control and CodeDNA: `passed`, `minutes`/`turns`/`tool_calls` when available, `wrong_file_or_module`, `human_interventions`, notes. + +**Optional (not required):** + +- `files_expected` only if known; otherwise omit (do not invent precision/recall) +- `files_opened` / `files_edited` if you can track them +- After both sessions: run a **judge agent** with [`challenge-judge-prompt.md`](challenge-judge-prompt.md) and save `tasks[].judge` + optional top-level `judge` + +## Deliverable + +1. Freeze the task list **before** timed runs. +2. Run every task in **A**, then the **same** ones in **B** (or interleaved, but same IDs). +3. (Recommended) Have a separate agent judge both sessions with the judge prompt. +4. Fill `challenge//metrics.json` from + https://github.com/Larens94/codedna/blob/main/challenge/metrics.example.json + (schema: `metrics.schema.json`). +5. Open a PR on `Larens94/codedna` titled: + `challenge: — CodeDNA Challenge submission` + Checklist: `challenge/SUBMISSION_TEMPLATE.md` +6. Honesty: no invented results. A Meet verification may be requested. + +## What to do now + +1. Inspect my repo and propose an adapted ≥10 task list (easy/medium/hard). +2. Ask which setup I will use: **two branches** / **two checkouts** / **two projects**. +3. Prepare install commands for condition B. +4. Only after I confirm: run the tasks and fill the metrics. +5. After the runs: offer the judge pass (prompt ready) if I want a structured comparison. diff --git a/docs/challenge-board.README.md b/docs/challenge-board.README.md index a3478b5..be93b12 100644 --- a/docs/challenge-board.README.md +++ b/docs/challenge-board.README.md @@ -1,11 +1,12 @@ # Updating the public challenge board -The live board is [`docs/challenge.html`](../docs/challenge.html). -It reads [`docs/challenge-board.json`](../docs/challenge-board.json) (static — no backend). +The live board / ranking is [`docs/challenge.html`](../docs/challenge.html). +It reads [`docs/challenge-board.json`](../docs/challenge-board.json) (static — no backend). +**Update it as soon as valid PRs arrive** so the ranking stays publicly visible. ## When a metrics PR is accepted -1. Merge the PR (`challenge//metrics.json`). +1. Merge the PR (`challenge//metrics.json`) after checking: real project (≥25 files), stack declared, `install` present, same-task control/codedna pairs, bugs filed when relevant. 2. Append a row to `docs/challenge-board.json` → `submissions` using fields from that JSON: ```json @@ -21,12 +22,19 @@ It reads [`docs/challenge-board.json`](../docs/challenge-board.json) (static — "tasks_total": 10, "control_passed": 7, "codedna_passed": 9, + "control_file_f1_avg": 0.72, + "codedna_file_f1_avg": 0.91, + "delta_file_f1": 0.19, + "judge_favors": "codedna", "favors": "codedna", + "favors_basis": "judge + pass_rate", "merged": true } ``` +Optional board fields (`control_file_f1_avg`, `codedna_file_f1_avg`, `delta_file_f1`, `judge_favors`, `favors_basis`) are shown when present — file lists / F1 are **not** required from entrants. 3. Bump `updated_at` (ISO date). 4. Commit + push `main` so GitHub Pages refreshes. -Enrollment does **not** require an issue — the metrics PR is enough. +Enrollment does **not** require an issue — the metrics PR is enough. +Optional: invite the entrant to a Meet presentation before or after the board row if verification is needed. diff --git a/docs/challenge-board.json b/docs/challenge-board.json index a62d9bf..a46e423 100644 --- a/docs/challenge-board.json +++ b/docs/challenge-board.json @@ -7,7 +7,7 @@ "window": { "start": null, "end": null, - "note": "Dates TBD — set before public launch" + "note": "In arrivo — date annunciate con il video di lancio" }, "submissions": [] } diff --git a/docs/challenge-judge-prompt.it.md b/docs/challenge-judge-prompt.it.md new file mode 100644 index 0000000..f36dc0a --- /dev/null +++ b/docs/challenge-judge-prompt.it.md @@ -0,0 +1,81 @@ +# Prompt giudice — CodeDNA Challenge + +> Usa questo prompt **dopo** le due sessioni (Control e CodeDNA), non durante. +> Serve a confrontare le run in modo strutturato. +> English: [challenge-judge-prompt.md](challenge-judge-prompt.md) + +--- + +## Ruolo + +Sei un **agente giudice imparziale**. Confronti due esecuzioni dello **stesso task** (o dello stesso set di task): + +- **A — Control:** senza CodeDNA +- **B — CodeDNA:** con CodeDNA + +Non inventare file o esiti. Se l’evidenza manca, marca `inconclusive`. + +## Input che ti fornisco + +Per ogni task: + +1. Titolo / descrizione del task +2. Diff / file modificati della sessione Control +3. Diff / file modificati della sessione CodeDNA +4. (Opzionale) log turni, tool calls, interventi umani +5. (Opzionale) `files_expected` — **solo se li conosco**; altrimenti valuta senza ground truth + +## Cosa valutare (in ordine) + +1. **Completezza** — ha risolto il task? (pass / partial / fail) +2. **File toccati** — ha modificato i posti giusti? Ha dimenticato caller/test? Ha toccato file irrilevanti? +3. **Navigazione** — ha girato a vuoto o è andato subito ai file rilevanti? +4. **Vincoli** — ha rispettato regole di dominio / `Rules:` se evidenti dal diff o dal contesto +5. **Intervento umano** — quanto ha dovuto correggere la persona + +Se **non** c’è `files_expected`: + +- Non inventare precision/recall/F1 +- Descrivi comunque `files_assessment` in linguaggio naturale (completo / incompleto / fuori target) +- Puoi confrontare le due liste `files_edited` tra loro (quale sembra più coerente col task) + +Se **c’è** `files_expected`: + +- Calcola o stima missed / extra / precision / recall / F1 per entrambe le condizioni + +## Output richiesto (per task) + +```json +{ + "task_id": "T01", + "favors": "codedna | control | tie | inconclusive", + "score_control_1_to_5": 1, + "score_codedna_1_to_5": 1, + "navigation_notes": "", + "correctness_notes": "", + "files_assessment": "", + "raw_verdict": "" +} +``` + +## Output complessivo (opzionale) + +```json +{ + "used": true, + "method": "per_task | batch | spot_check", + "favors": "codedna | control | tie | inconclusive", + "tasks_judged": 10, + "codedna_wins": 0, + "control_wins": 0, + "ties": 0, + "notes": "" +} +``` + +## Regole + +- Non favorire CodeDNA a prescindere: se Control è meglio, dillo. +- Non usare conoscenza esterna inventata sul repo. +- Se confronti solo un campione di task, usa `method: spot_check` e dichiaralo. +- Il giudice **non sostituisce** `metrics.json`: arricchisce `tasks[].judge` e/o `judge` top-level. diff --git a/docs/challenge-judge-prompt.md b/docs/challenge-judge-prompt.md new file mode 100644 index 0000000..ddb2123 --- /dev/null +++ b/docs/challenge-judge-prompt.md @@ -0,0 +1,81 @@ +# Judge prompt — CodeDNA Challenge + +> Use this prompt **after** both sessions (Control and CodeDNA), not during them. +> It structures a comparison of the two runs. +> Italiano: [challenge-judge-prompt.it.md](challenge-judge-prompt.it.md) + +--- + +## Role + +You are an **impartial judge agent**. Compare two executions of the **same task** (or task set): + +- **A — Control:** without CodeDNA +- **B — CodeDNA:** with CodeDNA + +Do not invent files or outcomes. If evidence is missing, mark `inconclusive`. + +## Inputs I will provide + +Per task: + +1. Task title / description +2. Diff / edited files from Control +3. Diff / edited files from CodeDNA +4. (Optional) turn logs, tool calls, human interventions +5. (Optional) `files_expected` — **only if known**; otherwise judge without ground truth + +## What to score (in order) + +1. **Completeness** — did it solve the task? (pass / partial / fail) +2. **Files touched** — right places? missed callers/tests? irrelevant files? +3. **Navigation** — wandered vs went to relevant files quickly? +4. **Constraints** — domain / `Rules:` respected when visible from diff/context +5. **Human steering** — how much did the person have to fix + +If there is **no** `files_expected`: + +- Do **not** invent precision/recall/F1 +- Still write a natural-language `files_assessment` (complete / incomplete / off-target) +- You may compare the two `files_edited` lists for coherence with the task + +If there **is** `files_expected`: + +- Compute or estimate missed / extra / precision / recall / F1 for both conditions + +## Required per-task output + +```json +{ + "task_id": "T01", + "favors": "codedna | control | tie | inconclusive", + "score_control_1_to_5": 1, + "score_codedna_1_to_5": 1, + "navigation_notes": "", + "correctness_notes": "", + "files_assessment": "", + "raw_verdict": "" +} +``` + +## Optional overall output + +```json +{ + "used": true, + "method": "per_task | batch | spot_check", + "favors": "codedna | control | tie | inconclusive", + "tasks_judged": 10, + "codedna_wins": 0, + "control_wins": 0, + "ties": 0, + "notes": "" +} +``` + +## Rules + +- Do not bias toward CodeDNA: if Control is better, say so. +- Do not invent external repo knowledge. +- If you only sample tasks, use `method: spot_check` and declare it. +- The judge **does not replace** `metrics.json`: it fills `tasks[].judge` and/or top-level `judge`. diff --git a/docs/challenge.html b/docs/challenge.html index c9e57a9..8f9b2ef 100644 --- a/docs/challenge.html +++ b/docs/challenge.html @@ -1,10 +1,16 @@ - + -CodeDNA Challenge — €200 - +CodeDNA Challenge — €200 · Classifica pubblica + + + + + + + @@ -85,10 +91,14 @@ display: inline-flex; align-items: center; gap: 0.5rem; padding: 0.75rem 1.5rem; border-radius: var(--radius); font-size: 0.875rem; font-weight: 600; text-decoration: none; border: 1px solid var(--black); transition: opacity .15s, border-color .15s; + cursor: pointer; font-family: inherit; background: transparent; color: var(--black); } .btn-primary { background: var(--black); color: #fff; } .btn-secondary { background: transparent; color: var(--black); border-color: var(--gray-200); } .btn-secondary:hover { border-color: var(--black); } +.copy-status { + margin-top: 0.75rem; font-size: 0.85rem; color: var(--gray-600); +} .section { max-width: 1100px; margin: 0 auto; padding: 64px 28px; } .section + .section { border-top: 1px solid var(--gray-200); } @@ -150,10 +160,21 @@ .favors-control { color: #d97706; font-weight: 600; } .favors-tie, .favors-inconclusive { color: var(--gray-600); font-weight: 600; } .empty { - padding: 3rem 1.5rem; text-align: center; color: var(--gray-600); + padding: 3.25rem 1.75rem; text-align: center; color: var(--gray-600); border: 1px dashed var(--gray-200); border-radius: var(--radius); + background: + linear-gradient(180deg, #fafafa 0%, #fff 100%); +} +.empty strong { display: block; color: var(--black); font-size: 1.05rem; margin-bottom: 0.55rem; } +.empty span { display: block; max-width: 420px; margin: 0 auto 1.25rem; line-height: 1.65; } +.empty-meta { + display: inline-flex; flex-wrap: wrap; gap: 0.5rem; justify-content: center; + font-family: 'JetBrains Mono', monospace; font-size: 11px; color: var(--gray-600); +} +.empty-meta span { + display: inline-block; margin: 0; padding: 0.35rem 0.65rem; + border: 1px solid var(--gray-200); border-radius: var(--radius); background: var(--white); } -.empty strong { display: block; color: var(--black); margin-bottom: 0.4rem; } .note { margin-top: 1.25rem; font-size: 0.85rem; color: var(--gray-600); @@ -172,17 +193,80 @@ .meta-cell { border-right: none; border-bottom: 1px solid var(--gray-200); } } @media (max-width: 720px) { - .nav-links { display: none; position: absolute; top: 60px; left: 0; right: 0; flex-direction: column; gap: 0; background: var(--white); border-bottom: 1px solid var(--gray-200); padding: 0.75rem 1rem 1rem; } + .nav-links { display: none; position: absolute; top: 68px; left: 0; right: 0; flex-direction: column; gap: 0.5rem; background: var(--white); border-bottom: 1px solid var(--gray-200); padding: 0.85rem 1.1rem 1.15rem; } .nav-links.open { display: flex; } - .nav-links a { padding: 0.65rem 0; } + .nav-links a { padding: 0.7rem 0; } + .nav-links a.nav-cta { margin-top: 0.35rem; margin-bottom: 0.35rem; } + .nav-links .lang-toggle { margin-top: 0.15rem; align-self: stretch; } .nav-hamburger { display: flex; } .meta-strip, .steps, .levels { grid-template-columns: 1fr; } .page-hero h1 { max-width: none; } + .page-hero, .section, footer.site-footer { padding-left: 20px; padding-right: 20px; } } + - +