diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json new file mode 100644 index 0000000..155d55f --- /dev/null +++ b/.config/dotnet-tools.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "docfx": { + "version": "2.81.0", + "commands": [ + "docfx" + ] + } + } +} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3e67f37..8a14ba1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,9 @@ jobs: - uses: actions/setup-dotnet@v4 with: - dotnet-version: "10.0.x" + dotnet-version: | + 8.0.x + 10.0.x - name: Restore run: dotnet restore Unsga3.slnx @@ -53,3 +55,20 @@ jobs: - name: Smoke (mock) run: python tools/typesafe-pareto/score_pareto.py --smoke --force-mock --metrics metrics/typesafe-runs.jsonl + + api-docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: | + 8.0.x + 10.0.x + + - name: Restore DocFX + run: dotnet tool restore + + - name: Build API docs + run: dotnet docfx docfx.json diff --git a/.github/workflows/publish-github-packages.yml b/.github/workflows/publish-github-packages.yml index f33287d..a505502 100644 --- a/.github/workflows/publish-github-packages.yml +++ b/.github/workflows/publish-github-packages.yml @@ -21,7 +21,9 @@ jobs: - uses: actions/setup-dotnet@v4 with: - dotnet-version: "10.0.x" + dotnet-version: | + 8.0.x + 10.0.x source-url: https://nuget.pkg.github.com/${{ github.repository_owner }}/index.json env: NUGET_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index a6bb1fe..3102aef 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,7 @@ metrics/typesafe-runs.jsonl appsettings.*.local.json .env .env.* + +# DocFX (regenerate with `dotnet tool restore && dotnet docfx docfx.json`) +_site/ +/api/ diff --git a/.grok/skills/unsga3-oracle/SKILL.md b/.grok/skills/unsga3-oracle/SKILL.md index a29b05f..523cad5 100644 --- a/.grok/skills/unsga3-oracle/SKILL.md +++ b/.grok/skills/unsga3-oracle/SKILL.md @@ -19,13 +19,13 @@ Repo root: Unsga3. Confirm `Unsga3.slnx` / `tools/OracleCompare` exist before ru | zdt2 | 12 | 52 | **250** | `--pymoo-mode` (`PymooCompatible`) | | dtlz2 | 12 | 92 | 150 | `--pymoo-mode` (`PymooCompatible`) | -DTLZ2: C# `Dtlz2Problem(k: 10)` ⇒ **n_var=12**. `run_pymoo_oracle.py` passes `n_var=12`. pymoo’s own default is n_var=10 (k=8). Do not treat the published 15-seed pymoo column as that matched run, and do not rewrite `docs/WILCOXON-RESULTS.md` until the seeds are re-run. +DTLZ2: C# `Dtlz2Problem(k: 10)` ⇒ **n_var=12**. `run_pymoo_oracle.py` and `run_multiseed_wilcoxon.py` pass `n_var=12`. pymoo’s own default is n_var=10 (k=8). The checked-in `docs/WILCOXON-RESULTS.md` DTLZ2 columns are that matched re-run. Rewrite that file only by re-running the harness. Do not paste a remembered n_var=10 column back in. ZDT2 **gens=100** is an early-stress snapshot (collapse on Bend, C#, and pymoo), not the quality bar. Quality protocol matches unsga3-bend A/B (gens=250, PymooCompatible). `RankNicheDistance` is an optional unpublished Wilcoxon ZDT2 mating mode — do not silently switch all ZDT defaults to it. ZDT1 and DTLZ2 unchanged. IGD = **mean** nearest Euclidean distance (pymoo-compatible). C# scores the full non-dominated front; `run_pymoo_oracle.py` scores pymoo `res.F` (niche optimum). Compare them only on a shared front definition and a shared reference set. Docs: `docs/EQUIVALENCE.md`, `docs/RESEARCH-STANDARDS.md`. -Requires: .NET 10 SDK; Python 3 + `pip install pymoo` for pymoo side / multi-seed. +Requires: .NET 10 SDK to run `tools/OracleCompare`. A full `dotnet test` of both library TFMs also needs the .NET 8 SDK. Python 3 + `pip install pymoo` for the pymoo side / multi-seed. ## Modes diff --git a/CHANGELOG.md b/CHANGELOG.md index bd5e717..5a4e953 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,11 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/). ### Added -- Measured 15-seed tables for OSY, TNK, and C1-DTLZ1 (feasible non-dominated IGD vs pymoo `UNSGA3`) and for mating opt-ins (`TwoShuffledPasses`, `Algorithm2`, and both) in `docs/NEW-SURFACES-RESULTS.md`. Protocol is in `docs/EQUIVALENCE.md`. Package defaults are unchanged. `docs/WILCOXON-RESULTS.md` is not rewritten. +- `PerformanceIndicators.GenerationalDistancePlus` (Ishibuchi GD+): mean modified distance from each obtained point to the nearest reference point. Hand-case unit tests. No new benchmark IGD table. +- API reference via DocFX. From a clean checkout: `dotnet tool restore && dotnet docfx docfx.json`. CI job `api-docs` builds `_site/`. +- Measured 15-seed tables for OSY, TNK, and C1-DTLZ1 (feasible non-dominated IGD vs pymoo `UNSGA3`) and for mating opt-ins (`TwoShuffledPasses`, `Algorithm2`, and both) in `docs/NEW-SURFACES-RESULTS.md`. Protocol is in `docs/EQUIVALENCE.md`. Package defaults are unchanged. - Optional **TypeSafe / Jev System One** pass (`tools/typesafe-pareto`) for Score + Choice judgments over a small Pareto candidate sample. Additive semantic layer — does not replace NSGA-III / U-NSGA-III objectives. Live calls only when `TYPESAFE_API_KEY` is set; CI uses mocks. Metrics append to `metrics/typesafe-runs.jsonl`. ### Changed +- The library, tests, and BasicUsage sample multi-target `net8.0` and `net10.0`. `tools/OracleCompare` stays `net10.0`. GitHub Packages is still the only publish target. +- `docs/WILCOXON-RESULTS.md` is the matched DTLZ2 re-run (`n_var=12` on C# and pymoo). ZDT2 stays out of that table (quality bar is gens=250 + `PymooCompatible`). No invented ZDT2 Wilcoxon numbers. - Docs and XML comments describe `initialPopulation` and hybrid loops in generic terms (domain-adapter warm-start / grid-seed). No product-repo names. ## [0.2.0] — 2026-09-30 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 20f7b5a..d8c7ed5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,8 +10,9 @@ Thanks for your interest. This library aims to be a **faithful, well-tested** U- ## Development setup -- **.NET 10 SDK** ([download](https://dotnet.microsoft.com/download)) +- **.NET 8 SDK and .NET 10 SDK** ([download](https://dotnet.microsoft.com/download)). The library, tests, and sample target both. `tools/OracleCompare` is `net10.0` only. - Optional (oracle / multi-seed stats): Python 3.10+ with `pip install pymoo` +- Optional (API docs): `dotnet tool restore` then `dotnet docfx docfx.json` (DocFX is pinned in `.config/dotnet-tools.json`) - Optional (TypeSafe / Jev Pareto scores): Python 3.10+; live calls need `TYPESAFE_API_KEY` (never commit it) ```bash @@ -19,7 +20,7 @@ git clone https://github.com/AppSprout-dev/Unsga3.git cd Unsga3 dotnet build Unsga3.slnx -c Release dotnet test Unsga3.slnx -c Release -dotnet run --project samples/BasicUsage -c Release +dotnet run --project samples/BasicUsage -c Release -f net10.0 ``` ### Oracle / equivalence (optional) diff --git a/README.md b/README.md index d76baf6..46dc600 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,12 @@ [![ci](https://github.com/AppSprout-dev/Unsga3/actions/workflows/ci.yml/badge.svg)](https://github.com/AppSprout-dev/Unsga3/actions/workflows/ci.yml) [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) -[![.NET](https://img.shields.io/badge/.NET-10.0-purple.svg)](https://dotnet.microsoft.com/) +[![.NET](https://img.shields.io/badge/.NET-8.0%20%7C%2010.0-purple.svg)](https://dotnet.microsoft.com/) **U-NSGA-III** (Unified NSGA-III) for .NET — single-, multi-, and many-objective evolutionary optimization with Das–Dennis reference directions, SBX crossover, polynomial mutation, and **niching-based tournament selection** ([Seada & Deb, 2016](https://ieeexplore.ieee.org/document/7271063)). -> **v0.2.0** — opt-in two-shuffle mating, Algorithm 2 tournament, odd-N pairing, feasible-only niching, equal-CV Deb fix, two-layer Das–Dennis, and OSY/TNK/C1-DTLZ1. Defaults stay unconstrained `RankNicheDistance`, independent mating, and single-layer Das–Dennis. The published 15-seed table is not rewritten. -> **15-seed IGD vs pymoo `UNSGA3`:** ZDT1 **median 0.053 vs 0.070** (MWU *p*≈0.05) compares the full C# non-dominated front with pymoo `res.F`. DTLZ2 **median 0.0045 vs 0.0028** (~1.6×) compares C# n_var=12 with pymoo's default n_var=10. Neither pair is a same-set, same-problem ranking. Notes: [`docs/ORACLE-RESULTS.md`](docs/ORACLE-RESULTS.md). +> **v0.2.0** — opt-in two-shuffle mating, Algorithm 2 tournament, odd-N pairing, feasible-only niching, equal-CV Deb fix, two-layer Das–Dennis, and OSY/TNK/C1-DTLZ1. Defaults stay unconstrained `RankNicheDistance`, independent mating, and single-layer Das–Dennis. Unreleased work adds `net8.0`+`net10.0`, GD+, and DocFX. +> **15-seed IGD vs pymoo `UNSGA3`:** ZDT1 **median 0.053173 vs 0.0695884** (MWU *p*=0.0512394) compares the full C# non-dominated front with pymoo `res.F`. DTLZ2 **median 0.00451168 vs 0.00388327** (ratio 1.16183, MWU *p*=0.0225311) uses **n_var=12** on both sides. ZDT2 has no Wilcoxon table. Notes: [`docs/WILCOXON-RESULTS.md`](docs/WILCOXON-RESULTS.md). > Details: [`docs/WILCOXON-RESULTS.md`](docs/WILCOXON-RESULTS.md) · single-seed notes: [`docs/ORACLE-RESULTS.md`](docs/ORACLE-RESULTS.md) · constrained + mating opt-ins: [`docs/NEW-SURFACES-RESULTS.md`](docs/NEW-SURFACES-RESULTS.md) ```text @@ -76,7 +76,7 @@ var algo = new Unsga3Algorithm(dirs, populationSize: 92, seed: 1, | `ReferenceDirections.DasDennis` | Single-layer reference points (default) | | `ReferenceDirections.TwoLayerDasDennis` | Outer layer plus an inside layer for larger M | | `SimulatedBinaryCrossover` / `PolynomialMutation` | Variation operators | -| `PerformanceIndicators` | IGD, GD, 2-D hypervolume | +| `PerformanceIndicators` | IGD, IGD+, GD, GD+, 2-D hypervolume | | `TournamentMode` | Default rank→niche, `PymooCompatible` (distance-tie coin flip), or opt-in `Algorithm2` (keep the second parent) | | `MatingPoolMode` | Default independent tournaments; opt-in `TwoShuffledPasses` (paper / pymoo 0.6.2) | @@ -87,10 +87,19 @@ Built-in problems: ZDT1–4/6, DTLZ1–4/7, Sphere, Ackley, Rosenbrock, and cons ```bash dotnet build Unsga3.slnx -c Release dotnet test Unsga3.slnx -c Release -dotnet run --project samples/BasicUsage -c Release +dotnet run --project samples/BasicUsage -c Release -f net10.0 ``` -Requires **.NET 10** SDK. Optional oracle: Python 3 + `pip install pymoo` (see [CONTRIBUTING.md](CONTRIBUTING.md)). +The library, tests, and sample target **`net8.0` and `net10.0`**. Building both needs the .NET 8 and .NET 10 SDKs. `tools/OracleCompare` is `net10.0` only. Optional oracle: Python 3 + `pip install pymoo` (see [CONTRIBUTING.md](CONTRIBUTING.md)). + +API reference (DocFX, local tool): + +```bash +dotnet tool restore +dotnet docfx docfx.json +``` + +That writes `_site/` and does not publish a package. ## TypeSafe / Jev Pareto scoring (optional) @@ -141,6 +150,7 @@ Unsga3/ ├── tools/typesafe-pareto/ # optional TypeSafe / Jev Score+Choice on a front sample ├── metrics/ # typesafe-runs.jsonl (local; gitignored) ├── docs/ +├── docfx.json # API docs (`dotnet docfx docfx.json`) └── .github/workflows/ # CI + GitHub Packages publish ``` diff --git a/docfx.json b/docfx.json new file mode 100644 index 0000000..fad9e41 --- /dev/null +++ b/docfx.json @@ -0,0 +1,47 @@ +{ + "metadata": [ + { + "src": [ + { + "src": ".", + "files": [ + "src/Unsga3/Unsga3.csproj" + ] + } + ], + "dest": "api", + "includePrivateMembers": false, + "properties": { + "TargetFramework": "net10.0" + } + } + ], + "build": { + "content": [ + { + "files": [ + "toc.yml", + "docs/docfx/index.md", + "docs/docfx/toc.yml" + ] + }, + { + "files": [ + "api/**.yml", + "api/index.md" + ] + } + ], + "output": "_site", + "template": [ + "default", + "modern" + ], + "globalMetadata": { + "_appName": "Unsga3", + "_appTitle": "Unsga3", + "_enableSearch": true, + "_disableContribution": true + } + } +} diff --git a/docs/EQUIVALENCE.md b/docs/EQUIVALENCE.md index 2cab62a..f6a8f3b 100644 --- a/docs/EQUIVALENCE.md +++ b/docs/EQUIVALENCE.md @@ -12,7 +12,7 @@ See also **[RESEARCH-STANDARDS.md](RESEARCH-STANDARDS.md)** for the literature + - [pymoo `UNSGA3`](https://pymoo.org/algorithms/moo/unsga3.html) - pymoo `nsga3.py` — `HyperplaneNormalization`, `associate_to_niches`, `niching` - pymoo `Survival.filter_infeasible` — feasible niching, then ascending-CV fill (Jain & Deb, NSGA-III Part II) -- Indicators: [pymoo performance indicators](https://pymoo.org/misc/indicators.html) (GD, IGD, IGD+, HV) +- Indicators: [pymoo performance indicators](https://pymoo.org/misc/indicators.html) (GD, GD+, IGD, IGD+, HV) ## Protocol @@ -24,7 +24,7 @@ See also **[RESEARCH-STANDARDS.md](RESEARCH-STANDARDS.md)** for the literature + Score the **same front definition** and the **same reference set**. C# reports the full non-dominated front. pymoo `res.F` is the survival niche set (about one point per filled direction). `ReferenceDirectionThinning.OnePerDirection` can match cardinality; it does not reproduce `res.F` and is not a parity claim. 6. **Tolerance:** do not read the published table as median IGD within 1–2% of pymoo. ZDT1 median ratio 0.764107, Mann–Whitney U = 65, p = 0.0512394 (the pymoo column is `res.F`). - DTLZ2 median ratio 1.58638, U = 222, p = 6.15164×10⁻⁶ (the pymoo column is n_var=10). That comparison rejects equal distributions at α = 0.05. + DTLZ2 median ratio 1.16183, U = 168, p = 0.0225311 (both columns are n_var=12). That comparison rejects equal distributions at α = 0.05. The CI guard is 2× the published mismatched DTLZ2 scalar 0.00350, which fails a regression to about 2.9×. It is not a same-problem equivalence claim. Published A/B budgets (ZDT1 / DTLZ2 unchanged; ZDT2 matches unsga3-bend protocol honesty): @@ -55,7 +55,7 @@ ZDT2 **gens=100** is an early-stress snapshot (collapse on Bend, C#, and pymoo), - **Normalization intercepts / ASF axis extremes** (`NormalizationTests`) - Tournament pressure - `populationSize: null` ⇒ `|refs|` -- IGD/HV formulas on hand-checked fronts (`MetricsTests`) +- IGD/GD/GD+/HV formulas on hand-checked fronts (`MetricsTests`) ## Automated IGD smoke (CI) @@ -126,7 +126,7 @@ Same populations and generation counts as the quality protocols above. ZDT1 gens | algorithm2 | `Algorithm2` | `IndependentWithReplacement` | | both | `Algorithm2` | `TwoShuffledPasses` | -The ZDT2 and DTLZ2 default columns in that table are `RankNicheDistance`. The published DTLZ2 Wilcoxon column is `PymooCompatible`. ZDT2 has no published Wilcoxon column. Neither published table is rewritten. +The ZDT2 and DTLZ2 default columns in that table are `RankNicheDistance`. The published DTLZ2 Wilcoxon column is `PymooCompatible` at n_var=12. ZDT2 has no published Wilcoxon column. `run_new_surfaces.py` does not write `docs/WILCOXON-RESULTS.md`. ## DTLZ2 gap history @@ -136,4 +136,4 @@ The ZDT2 and DTLZ2 default columns in that table are `RankNicheDistance`. The pu | ASF + persistent ideal | 0.0052 | ~1.5× | | + duplicate elimination | **0.0040** | **~1.15× vs pymoo n_var=10** | -The ~1.15× denominator is pymoo at **n_var=10** (k=8), not the C# problem (n_var=12, k=10). A matched seed-1 pair is recorded in [ORACLE-RESULTS.md](ORACLE-RESULTS.md). The 15-seed table is still the mismatched-k run. +The ~1.15× denominator is pymoo at **n_var=10** (k=8), not the C# problem (n_var=12, k=10). That history row is not the current 15-seed table. The current table is the matched n_var=12 re-run in [WILCOXON-RESULTS.md](WILCOXON-RESULTS.md) (median ratio 1.16183). A matched seed-1 pair is also in [ORACLE-RESULTS.md](ORACLE-RESULTS.md). diff --git a/docs/NEW-SURFACES-RESULTS.md b/docs/NEW-SURFACES-RESULTS.md index 1e434d0..1029fb3 100644 --- a/docs/NEW-SURFACES-RESULTS.md +++ b/docs/NEW-SURFACES-RESULTS.md @@ -61,7 +61,7 @@ IGD is the C# feasible non-dominated front against `ParetoFronts` (ZDT: 500 poin | algorithm2 | Algorithm2 | IndependentWithReplacement | | both | Algorithm2 | TwoShuffledPasses | -ZDT2 uses gens=250. Its default column is `RankNicheDistance`, not `PymooCompatible`. There is no published ZDT2 Wilcoxon column. DTLZ2's default column is also `RankNicheDistance`. The published DTLZ2 column is `PymooCompatible` and is not rewritten here. +ZDT2 uses gens=250. Its default column is `RankNicheDistance`, not `PymooCompatible`. There is no published ZDT2 Wilcoxon column. DTLZ2's default column is also `RankNicheDistance`. The published DTLZ2 Wilcoxon column is `PymooCompatible` at n_var=12. This file does not rewrite it. Paired tests compare each opt-in with the default column (same seed index). Lower IGD is better. p-values are two-sided and not multiplicity-adjusted. diff --git a/docs/ORACLE-RESULTS.md b/docs/ORACLE-RESULTS.md index bd90ddc..501694f 100644 --- a/docs/ORACLE-RESULTS.md +++ b/docs/ORACLE-RESULTS.md @@ -74,11 +74,11 @@ On the shared 100-point PF, the full-front pair is C# 0.05119280568479224 and py | 5 | 0.00466 | | **mean** | **~0.00485** | -Those five C# seeds are `Dtlz2Problem(k: 10)` (n_var=12). The 0.0035 figure they were compared with is pymoo at **n_var=10** (k=8). That is not a same-problem band. The 15-seed file is unchanged until a matched re-run (see below). +Those five C# seeds are `Dtlz2Problem(k: 10)` (n_var=12). The 0.0035 figure they were compared with is pymoo at **n_var=10** (k=8). That is not a same-problem band. The current 15-seed file is the matched n_var=12 re-run (see below). ### DTLZ2 n_var (known mismatch, seed 1 remeasured) -`Dtlz2Problem(nObjectives: 3, k: 10)` builds **n = 12**. Deb et al. suggest k = 10. pymoo 0.6.2 `get_problem("dtlz2", n_obj=3)` defaults to **n_var=10** (k = 8). The harness used to omit `n_var`, so the published seed-1 pair and `docs/WILCOXON-RESULTS.md` compare those two dimensions. `tools/oracle/run_pymoo_oracle.py` now passes **n_var=12**. +`Dtlz2Problem(nObjectives: 3, k: 10)` builds **n = 12**. Deb et al. suggest k = 10. pymoo 0.6.2 `get_problem("dtlz2", n_obj=3)` defaults to **n_var=10** (k = 8). The harness used to omit `n_var`, so the historical seed-1 pair below compares those two dimensions. `tools/oracle/run_pymoo_oracle.py` and `tools/oracle/run_multiseed_wilcoxon.py` pass **n_var=12**. [WILCOXON-RESULTS.md](WILCOXON-RESULTS.md) is that matched 15-seed re-run. Published mismatched seed 1 (already in the Wilcoxon table; not re-interpreted as parity): @@ -94,7 +94,7 @@ Seed 1 remeasured **2026-09-22** with pymoo 0.6.2 after the oracle passes `n_var | C# `PymooCompatible` | 12 | 10 | 0.00403168 | 0.004031675764658275 | 92 | | pymoo `n_var=12` | 12 | 10 | 0.00308392 | 0.003083921253245871 | 91 | -Ratio of the two meta IGDs: 0.004031675764658275 / 0.003083921253245871 = **1.30732**. That is one seed, and the fronts still differ by one point (92 vs 91). It is not a 15-seed ranking and it does not replace the Wilcoxon table. +Ratio of the two meta IGDs: 0.004031675764658275 / 0.003083921253245871 = **1.30732**. That is one seed, and the fronts still differ by one point (92 vs 91). Seed 1 of the matched 15-seed table is this console pair. The 15-seed median ratio is **1.16183** ([WILCOXON-RESULTS.md](WILCOXON-RESULTS.md)). ## Root cause of the old ~5× DTLZ2 gap (fixed) diff --git a/docs/RESEARCH-STANDARDS.md b/docs/RESEARCH-STANDARDS.md index b26e1b1..e155234 100644 --- a/docs/RESEARCH-STANDARDS.md +++ b/docs/RESEARCH-STANDARDS.md @@ -27,7 +27,7 @@ Default dimensions (Deb / pymoo convention): - ZDT1–3: n=30; ZDT4: n=10; ZDT6: n=10 - DTLZ: n = M + k − 1 with k=5 (DTLZ1) or k=10 (DTLZ2–4), k=20 (DTLZ7) -- C# `Dtlz2Problem` uses that k=10, so M=3 ⇒ **n_var=12**. pymoo 0.6.2 `get_problem("dtlz2", n_obj=3)` defaults to **n_var=10** (k=8). Oracle runs pass `n_var=12`. The published 15-seed pymoo column is the default-10 run and is not a same-k comparison. +- C# `Dtlz2Problem` uses that k=10, so M=3 ⇒ **n_var=12**. pymoo 0.6.2 `get_problem("dtlz2", n_obj=3)` defaults to **n_var=10** (k=8). Oracle runs pass `n_var=12`. The published 15-seed pymoo column is that matched re-run. ## 2. Algorithm hyperparameters (match paper + pymoo) @@ -56,8 +56,9 @@ Default dimensions (Deb / pymoo convention): | Metric | Direction | Needs true PF? | Notes | |--------|-----------|----------------|-------| | **IGD** | ↓ | Yes | **Primary** — pymoo = **mean** nearest Euclidean distance (not √Σd²/n) | -| **IGD+** | ↓ | Yes | Weakly Pareto compliant (Ishibuchi) | -| **GD** | ↓ | Yes | Convergence only (can miss spread) | +| **IGD+** | ↓ | Yes | Weakly Pareto compliant (Ishibuchi). Mean modified distance from each reference point to the nearest obtained point | +| **GD** | ↓ | Yes | Convergence only (can miss spread). Mean Euclidean distance from each obtained point to the nearest reference point | +| **GD+** | ↓ | Yes | Ishibuchi modified distance, same direction as GD. `max(a_j − z_j, 0)` so a weakly better point contributes 0 | | **HV** | ↑ | No (needs ref point) | Prefer when PF unknown; 2-D closed form here | Definitions implemented in `Unsga3.Metrics.PerformanceIndicators` follow **pymoo’s formulas** (p=2 Euclidean averages). @@ -87,15 +88,15 @@ Typical ZDT: r = (1.1, 1.1). Always document r; never compare HV across differen 5. Compare IGD (and HV for M=2) to pymoo `UNSGA3` on the **same front definition**. C# uses the full non-dominated front; pymoo's harness value is `res.F` (the niche optimum). A gap between those two reporters is a set-definition gap until both sides are reduced the same way. 6. Shipping bar: the published 15-seed table is **not** “median IGD within ~1–2% of pymoo.” ZDT1: Mann–Whitney U = 65, p = 0.0512394, median ratio 0.764107 (pymoo column is `res.F`). - DTLZ2: U = 222, p = 6.15164×10⁻⁶, median ratio 1.58638 (pymoo column is n_var=10). That test rejects equal distributions at α = 0.05. - ZDT2 has no published C# Wilcoxon table; the quality budget is 250 generations. A matched 15-seed re-run has not replaced the table. + DTLZ2: U = 168, p = 0.0225311, median ratio 1.16183 (both columns are n_var=12). That test rejects equal distributions at α = 0.05. + ZDT2 has no published C# Wilcoxon table; the quality budget is 250 generations. Do not invent one. Export path: dump final `F` as CSV from both sides; compute IGD in this library. ## 5. What we implemented after this research - Full ZDT1–4, ZDT6; DTLZ1–4, DTLZ7; Sphere, Ackley, Rosenbrock -- GD / IGD / IGD+ / HV₂ +- GD / GD+ / IGD / IGD+ / HV₂ - Analytic PF samplers - Fixed-seed smoke IGD bounds (loose) + unit metric tests against hand-checked values - GitHub Packages publish workflow diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 57183d9..c2e7d74 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -14,13 +14,13 @@ Living plan for Unsga3. Issues track concrete work; this page is the narrative. ## Near term (0.2) -- [ ] Multi-seed Wilcoxon results checked in / refreshed on release -- [ ] `net8.0` (+ `net10.0`) multi-target for broader NuGet consumers +- [x] Multi-seed Wilcoxon results checked in / refreshed on release +- [x] `net8.0` (+ `net10.0`) multi-target for broader NuGet consumers - [ ] nuget.org publish (in addition to GitHub Packages) -- [ ] GD+ indicator (IGD+ has a hand-case test; GD and IGD were already implemented) +- [x] GD+ indicator (IGD+ has a hand-case test; GD and IGD were already implemented) - [x] Constrained demos (OSY, TNK, C1-DTLZ1) with formulation and survival self-tests -- [x] Constrained IGD / Wilcoxon vs pymoo — [NEW-SURFACES-RESULTS.md](NEW-SURFACES-RESULTS.md). The published ZDT/DTLZ Wilcoxon file is unchanged. -- [ ] API docs site (DocFX or similar) +- [x] Constrained IGD / Wilcoxon vs pymoo — [NEW-SURFACES-RESULTS.md](NEW-SURFACES-RESULTS.md). Those tables do not replace the ZDT/DTLZ Wilcoxon file. +- [x] API docs site (DocFX or similar) ## Medium term (0.3+) diff --git a/docs/WILCOXON-RESULTS.md b/docs/WILCOXON-RESULTS.md index 9d3c57c..a04ade7 100644 --- a/docs/WILCOXON-RESULTS.md +++ b/docs/WILCOXON-RESULTS.md @@ -4,15 +4,15 @@ Generated by `tools/oracle/run_multiseed_wilcoxon.py`. IGD = mean nearest Euclid ## Protocol -| Problem | Pop | Gens | Partitions | C# tournament | Seeds | -|---------|-----|------|------------|---------------|-------| -| zdt1 | 52 | 100 | 12 | RankNicheDistance | 15 | -| zdt2 (quality A/B; not in this table) | 52 | **250** | 12 | PymooCompatible | — | -| dtlz2 | 92 | 150 | 12 | PymooCompatible | 15 | +| Problem | Pop | Gens | Partitions | n_var | C# tournament | Seeds | +|---------|-----|------|------------|-------|---------------|-------| +| zdt1 | 52 | 100 | 12 | problem default | RankNicheDistance | 15 | +| dtlz2 | 92 | 150 | 12 | 12 | PymooCompatible | 15 | +| zdt2 (quality A/B; not in this table) | 52 | **250** | 12 | problem default | PymooCompatible | — | -ZDT2 **gens=100** is an early-stress snapshot, not the quality bar. C# never published a hard ZDT2 Wilcoxon table; the unpublished harness used gens=100 + `RankNicheDistance` (optional mating mode — do not silently switch all ZDT defaults to it). Quality protocol matches [unsga3-bend](https://github.com/AppSprout-dev/unsga3-bend) A/B honesty: gens=250 + `PymooCompatible`. ZDT1 and DTLZ2 numbers below are unchanged. Do not invent a ZDT2 IGD table here. +ZDT2 **gens=100** is an early-stress snapshot, not the quality bar. C# has no published ZDT2 Wilcoxon table. The unpublished harness used gens=100 + `RankNicheDistance` (optional mating mode — do not silently switch all ZDT defaults to it). Quality protocol matches [unsga3-bend](https://github.com/AppSprout-dev/unsga3-bend) A/B honesty: gens=250 + `PymooCompatible`. Do not invent a ZDT2 IGD table here. -**DTLZ2 dimension (do not refresh this file in place).** The pymoo column below was produced with pymoo’s default `n_var=10` (k=8). C# used `Dtlz2Problem(k: 10)` (`n_var=12`). Seed 1 of that mismatched pair is C# 0.00403168 / pymoo 0.00349879. New oracle runs pass `n_var=12`. A matched seed-1 pair is in [ORACLE-RESULTS.md](ORACLE-RESULTS.md). **Leave this table as published until the 15 seeds are actually re-run.** +**DTLZ2 dimension.** Both columns below use **n_var=12** (k=10): C# `Dtlz2Problem(k: 10)` and pymoo `get_problem("dtlz2", n_obj=3, n_var=12)`. pymoo's own default remains n_var=10 (k=8). This file replaces that older pymoo column. A matched seed-1 pair is also in [ORACLE-RESULTS.md](ORACLE-RESULTS.md). Hypothesis tests (α = 0.05, two-sided): @@ -68,42 +68,43 @@ Lower IGD is better. | Impl | n | mean | std | median | Q1 | Q3 | min | max | |------|---|------|-----|--------|----|----|-----|-----| | **Unsga3** | 15 | 0.0044803 | 0.000658452 | **0.00451168** | 0.00397447 | 0.00477695 | 0.00361016 | 0.005666 | -| **pymoo** | 15 | 0.00301684 | 0.000461866 | **0.00284401** | 0.00258435 | 0.00345514 | 0.00246865 | 0.00387676 | +| **pymoo** | 15 | 0.00383266 | 0.000580361 | **0.00388327** | 0.00323491 | 0.00421612 | 0.0030682 | 0.00479208 | ### Hypothesis tests | Test | Statistic | p-value | Verdict (α=0.05) | |------|-----------|---------|------------------| -| Mann–Whitney U (U₁) | 222 | 6.15164e-06 | reject H₀ (differs; better median = pymoo) | -| Wilcoxon signed-rank | 0 | 0.000726514 | reject H₀ (differs; better median = pymoo) | +| Mann–Whitney U (U₁) | 168 | 0.0225311 | reject H₀ (differs; better median = pymoo) | +| Wilcoxon signed-rank | 15 | 0.0114901 | reject H₀ (differs; better median = pymoo) | -- Median ratio (C# / pymoo): **1.58638** -- Mean ratio (C# / pymoo): **1.4851** +- Median ratio (C# / pymoo): **1.16183** +- Mean ratio (C# / pymoo): **1.16898** - Better median: **pymoo** ### Per-seed IGD | Seed | Unsga3 | pymoo | Δ (C#−pymoo) | |------|--------|-------|--------------| -| 1 | 0.00403168 | 0.00349879 | 0.00053289 | -| 2 | 0.005666 | 0.00276702 | 0.00289898 | -| 3 | 0.00513016 | 0.00269731 | 0.00243285 | -| 4 | 0.00477695 | 0.00348147 | 0.00129548 | -| 5 | 0.00466321 | 0.00325574 | 0.00140747 | -| 6 | 0.00545999 | 0.00258435 | 0.00287564 | -| 7 | 0.00361016 | 0.00258159 | 0.00102857 | -| 8 | 0.00511116 | 0.00387676 | 0.0012344 | -| 9 | 0.0046699 | 0.00346408 | 0.00120582 | -| 10 | 0.00381559 | 0.00258797 | 0.00122762 | -| 11 | 0.00397447 | 0.00284401 | 0.00113046 | -| 12 | 0.00451168 | 0.00250194 | 0.00200974 | -| 13 | 0.00398436 | 0.00246865 | 0.00151571 | -| 14 | 0.00417245 | 0.00345514 | 0.000717311 | -| 15 | 0.00362671 | 0.00318779 | 0.000438915 | +| 1 | 0.00403168 | 0.00308392 | 0.000947759 | +| 2 | 0.005666 | 0.00465811 | 0.00100789 | +| 3 | 0.00513016 | 0.00386095 | 0.00126921 | +| 4 | 0.00477695 | 0.0030682 | 0.00170875 | +| 5 | 0.00466321 | 0.00479208 | -0.000128866 | +| 6 | 0.00545999 | 0.00436049 | 0.0010995 | +| 7 | 0.00361016 | 0.00323491 | 0.000375246 | +| 8 | 0.00511116 | 0.00316287 | 0.00194829 | +| 9 | 0.0046699 | 0.00430283 | 0.000367069 | +| 10 | 0.00381559 | 0.00388327 | -6.7676e-05 | +| 11 | 0.00397447 | 0.00345229 | 0.00052218 | +| 12 | 0.00451168 | 0.00331282 | 0.00119886 | +| 13 | 0.00398436 | 0.00421612 | -0.000231758 | +| 14 | 0.00417245 | 0.00408783 | 8.46218e-05 | +| 15 | 0.00362671 | 0.00401321 | -0.0003865 | ## Notes -- ZDT1 sets differ. The Unsga3 column is the full non-dominated front. The pymoo column is `res.F` (about one point per reference direction). The median ratio **0.764107** inherits that asymmetry. A shared-front seed-1 note is in [ORACLE-RESULTS.md](ORACLE-RESULTS.md). Do not replace this table until both columns use one front definition. +- ZDT1 sets differ. The Unsga3 column is the full non-dominated front. The pymoo column is `res.F` (about one point per reference direction). The median ratio **0.764107** inherits that asymmetry. A shared-front seed-1 note is in [ORACLE-RESULTS.md](ORACLE-RESULTS.md). +- DTLZ2 uses **n_var=12** (k=10) on both C# `Dtlz2Problem(k: 10)` and pymoo. This replaces the earlier pymoo column at n_var=10 (k=8). - Not bit-identical: different RNG implementations and minor operator ordering. -- These tests are not a 1–2% equivalence claim. ZDT1 U = 65, p = 0.0512394, median ratio 0.764107, and its pymoo column is `res.F`. DTLZ2 U = 222, p = 6.15164e-06, median ratio 1.58638, and its pymoo column is n_var=10. DTLZ2 rejects equal distributions at α = 0.05. +- These tests are not a 1–2% equivalence claim. ZDT1 U = 65, p = 0.0512394, median ratio 0.764107. DTLZ2 U = 168, p = 0.0225311, median ratio 1.16183. - Reproduce: `python tools/oracle/run_multiseed_wilcoxon.py` diff --git a/docs/docfx/index.md b/docs/docfx/index.md new file mode 100644 index 0000000..c6c19e5 --- /dev/null +++ b/docs/docfx/index.md @@ -0,0 +1,10 @@ +# Unsga3 API + +Reference pages are generated from the public surface of `src/Unsga3`. + +```bash +dotnet tool restore +dotnet docfx docfx.json +``` + +Output is `_site/` (gitignored). Metadata YAML lands in `api/` (also gitignored). The library targets `net8.0` and `net10.0`; DocFX extracts the `net10.0` build. diff --git a/docs/docfx/toc.yml b/docs/docfx/toc.yml new file mode 100644 index 0000000..1ba183c --- /dev/null +++ b/docs/docfx/toc.yml @@ -0,0 +1,2 @@ +- name: Overview + href: index.md diff --git a/samples/BasicUsage/BasicUsage.csproj b/samples/BasicUsage/BasicUsage.csproj index cd04a30..7ab7842 100644 --- a/samples/BasicUsage/BasicUsage.csproj +++ b/samples/BasicUsage/BasicUsage.csproj @@ -2,7 +2,7 @@ Exe - net10.0 + net8.0;net10.0 false Unsga3.Samples.BasicUsage diff --git a/src/Unsga3/Metrics/PerformanceIndicators.cs b/src/Unsga3/Metrics/PerformanceIndicators.cs index d960e7e..e981be3 100644 --- a/src/Unsga3/Metrics/PerformanceIndicators.cs +++ b/src/Unsga3/Metrics/PerformanceIndicators.cs @@ -2,7 +2,7 @@ namespace Unsga3.Metrics; /// /// Industry-standard MOEA performance indicators aligned with pymoo / Coello / Ishibuchi: -/// GD, IGD (p=2 Euclidean), IGD+, and 2-D hypervolume. +/// GD, GD+, IGD (p=2 Euclidean), IGD+, and 2-D hypervolume. /// Formulas match pymoo docs (https://pymoo.org/misc/indicators.html). /// public static class PerformanceIndicators @@ -22,6 +22,36 @@ public static double GenerationalDistance(IReadOnlyList obtained, IRea return sum / obtained.Count; } + /// + /// GD+ (Ishibuchi et al.): mean modified distance from each obtained point + /// to the nearest reference point (minimization). The distance is the Euclidean + /// norm of max(a_j − z_j, 0), the same one used by + /// . A point that is weakly better + /// than a reference point in every objective contributes 0 toward that reference. + /// + public static double GenerationalDistancePlus( + IReadOnlyList obtained, + IReadOnlyList referenceFront) + { + ValidateFronts(obtained, referenceFront); + if (obtained.Count == 0 || referenceFront.Count == 0) + return double.PositiveInfinity; + + double sum = 0; + for (int i = 0; i < obtained.Count; i++) + { + double best = double.PositiveInfinity; + var a = obtained[i]; + for (int j = 0; j < referenceFront.Count; j++) + { + double d = ModifiedDistance(a, referenceFront[j]); + if (d < best) best = d; + } + sum += best; + } + return sum / obtained.Count; + } + /// /// Inverted Generational Distance as used by pymoo: mean Euclidean distance /// from each true PF point Z to the nearest obtained point A (lower is better). diff --git a/src/Unsga3/Unsga3.csproj b/src/Unsga3/Unsga3.csproj index 9983470..8b34068 100644 --- a/src/Unsga3/Unsga3.csproj +++ b/src/Unsga3/Unsga3.csproj @@ -1,7 +1,7 @@ - net10.0 + net8.0;net10.0 Unsga3 Unsga3 Unsga3 diff --git a/tests/Unsga3.Tests/Unit/MetricsTests.cs b/tests/Unsga3.Tests/Unit/MetricsTests.cs index 739fadc..2761d57 100644 --- a/tests/Unsga3.Tests/Unit/MetricsTests.cs +++ b/tests/Unsga3.Tests/Unit/MetricsTests.cs @@ -62,8 +62,42 @@ public void IgdPlus_and_gd_match_the_hand_case() var reference = new[] { new[] { 0.0, 0.0 }, new[] { 1.0, 0.0 } }; double igdPlus = PerformanceIndicators.InvertedGenerationalDistancePlus(obtained, reference); double gd = PerformanceIndicators.GenerationalDistance(obtained, reference); + double gdPlus = PerformanceIndicators.GenerationalDistancePlus(obtained, reference); Assert.Equal(1.0, igdPlus, 9); Assert.Equal(1.0, gd, 9); + Assert.Equal(1.0, gdPlus, 9); + } + + [Fact] + public void GdPlus_uses_only_the_worse_objectives() + { + // (0, 0) weakly dominates (1, 1). Euclidean GD is √2. GD+ is 0. + var dominating = new[] { new[] { 0.0, 0.0 } }; + var worseReference = new[] { new[] { 1.0, 1.0 } }; + double gdPlus = PerformanceIndicators.GenerationalDistancePlus(dominating, worseReference); + double gd = PerformanceIndicators.GenerationalDistance(dominating, worseReference); + Assert.Equal(0.0, gdPlus, 9); + Assert.Equal(Math.Sqrt(2), gd, 9); + + // (0.5, 0.5) against {(0, 1), (1, 0)}. + // Each modified distance is 0.5. Euclidean nearest distance is √0.5. + var obtained = new[] { new[] { 0.5, 0.5 } }; + var reference = new[] { new[] { 0.0, 1.0 }, new[] { 1.0, 0.0 } }; + gdPlus = PerformanceIndicators.GenerationalDistancePlus(obtained, reference); + gd = PerformanceIndicators.GenerationalDistance(obtained, reference); + Assert.Equal(0.5, gdPlus, 9); + Assert.Equal(Math.Sqrt(0.5), gd, 9); + } + + [Fact] + public void GdPlus_is_infinite_when_a_front_is_empty() + { + var point = new[] { new[] { 0.0, 0.0 } }; + var empty = Array.Empty(); + Assert.True(double.IsPositiveInfinity( + PerformanceIndicators.GenerationalDistancePlus(empty, point))); + Assert.True(double.IsPositiveInfinity( + PerformanceIndicators.GenerationalDistancePlus(point, empty))); } [Fact] @@ -77,6 +111,8 @@ public void Indicators_reject_mismatched_objective_lengths() PerformanceIndicators.GenerationalDistance(shortVector, wide)); Assert.Throws(() => PerformanceIndicators.InvertedGenerationalDistancePlus(wide, shortVector)); + Assert.Throws(() => + PerformanceIndicators.GenerationalDistancePlus(wide, shortVector)); } [Fact] diff --git a/tests/Unsga3.Tests/Unsga3.Tests.csproj b/tests/Unsga3.Tests/Unsga3.Tests.csproj index 3fd0c49..e0fe88d 100644 --- a/tests/Unsga3.Tests/Unsga3.Tests.csproj +++ b/tests/Unsga3.Tests/Unsga3.Tests.csproj @@ -1,7 +1,7 @@ - net10.0 + net8.0;net10.0 false Unsga3.Tests diff --git a/toc.yml b/toc.yml new file mode 100644 index 0000000..e0e1dfa --- /dev/null +++ b/toc.yml @@ -0,0 +1,4 @@ +- name: Overview + href: docs/docfx/index.md +- name: API + href: api/ diff --git a/tools/oracle/run_multiseed_wilcoxon.py b/tools/oracle/run_multiseed_wilcoxon.py index 52ca71e..3073c3b 100644 --- a/tools/oracle/run_multiseed_wilcoxon.py +++ b/tools/oracle/run_multiseed_wilcoxon.py @@ -56,8 +56,7 @@ class Protocol: # Optional: csharp_pymoo_mode=False (RankNicheDistance) + gens=100 is the # unpublished Wilcoxon ZDT2 mating snapshot — do not treat it as the default. "zdt2": Protocol("zdt2", partitions=12, pop=52, gens=250, n_obj=2, csharp_pymoo_mode=True), - # n_var=12 matches C#. The checked-in WILCOXON-RESULTS.md pymoo column is the - # older n_var=10 run and must not be regenerated until those seeds are re-run. + # n_var=12 matches C# Dtlz2Problem(k: 10). pymoo's own default is n_var=10 (k=8). "dtlz2": Protocol( "dtlz2", partitions=12, @@ -264,27 +263,56 @@ def fmt(x: float) -> str: return f"{x:.6g}" +def _tournament_name(csharp_pymoo_mode: bool) -> str: + return "PymooCompatible" if csharp_pymoo_mode else "RankNicheDistance" + + def markdown_report(results: dict) -> str: + problems = results["problems"] lines = [ "# Multi-seed Wilcoxon: Unsga3 vs pymoo UNSGA3", "", - f"Generated by `tools/oracle/run_multiseed_wilcoxon.py`. " - f"IGD = mean nearest Euclidean distance (pymoo-compatible).", + "Generated by `tools/oracle/run_multiseed_wilcoxon.py`. " + "IGD = mean nearest Euclidean distance (pymoo-compatible).", "", "## Protocol", "", - "| Problem | Pop | Gens | Partitions | C# tournament | Seeds |", - "|---------|-----|------|------------|---------------|-------|", + "| Problem | Pop | Gens | Partitions | n_var | C# tournament | Seeds |", + "|---------|-----|------|------------|-------|---------------|-------|", ] - for name, block in results["problems"].items(): + for name, block in problems.items(): p = block["protocol"] + n_var = p.get("n_var") + n_var_cell = str(n_var) if n_var is not None else "problem default" lines.append( - f"| {name} | {p['pop']} | {p['gens']} | {p['partitions']} | " - f"{'PymooCompatible' if p['csharp_pymoo_mode'] else 'RankNicheDistance'} | " - f"{results['n_seeds']} |" + f"| {name} | {p['pop']} | {p['gens']} | {p['partitions']} | {n_var_cell} | " + f"{_tournament_name(p['csharp_pymoo_mode'])} | {results['n_seeds']} |" ) + if "zdt2" not in problems: + lines.append( + "| zdt2 (quality A/B; not in this table) | 52 | **250** | 12 | problem default | " + "PymooCompatible | — |" + ) + lines.append("") + if "zdt2" not in problems: + lines += [ + "ZDT2 **gens=100** is an early-stress snapshot, not the quality bar. " + "C# has no published ZDT2 Wilcoxon table. The unpublished harness used gens=100 + " + "`RankNicheDistance` (optional mating mode — do not silently switch all ZDT defaults to it). " + "Quality protocol matches [unsga3-bend](https://github.com/AppSprout-dev/unsga3-bend) A/B honesty: " + "gens=250 + `PymooCompatible`. Do not invent a ZDT2 IGD table here.", + "", + ] + dtlz2 = problems.get("dtlz2") + if dtlz2 is not None and dtlz2["protocol"].get("n_var") == 12: + lines += [ + "**DTLZ2 dimension.** Both columns below use **n_var=12** (k=10): " + "C# `Dtlz2Problem(k: 10)` and pymoo `get_problem(\"dtlz2\", n_obj=3, n_var=12)`. " + "pymoo's own default remains n_var=10 (k=8). This file replaces that older pymoo column. " + "A matched seed-1 pair is also in [ORACLE-RESULTS.md](ORACLE-RESULTS.md).", + "", + ] lines += [ - "", "Hypothesis tests (α = 0.05, two-sided):", "", "- **Mann–Whitney U** (Wilcoxon rank-sum): independent samples, H₀: same IGD distribution.", @@ -349,16 +377,50 @@ def markdown_report(results: dict) -> str: lines.append(f"| {i} | {fmt(a)} | {fmt(b)} | {fmt(a - b)} |") lines.append("") + lines += _closing_notes(results) + return "\n".join(lines) + + +def _closing_notes(results: dict) -> list[str]: + problems = results["problems"] + lines = ["## Notes", ""] + zdt1 = problems.get("zdt1") + if zdt1 is not None and "tests" in zdt1: + ratio = fmt(zdt1["tests"]["median_ratio"]) + lines.append( + "- ZDT1 sets differ. The Unsga3 column is the full non-dominated front. " + "The pymoo column is `res.F` (about one point per reference direction). " + f"The median ratio **{ratio}** inherits that asymmetry. " + "A shared-front seed-1 note is in [ORACLE-RESULTS.md](ORACLE-RESULTS.md)." + ) + dtlz2 = problems.get("dtlz2") + if dtlz2 is not None and dtlz2["protocol"].get("n_var") == 12: + lines.append( + "- DTLZ2 uses **n_var=12** (k=10) on both C# `Dtlz2Problem(k: 10)` and pymoo. " + "This replaces the earlier pymoo column at n_var=10 (k=8)." + ) + lines.append( + "- Not bit-identical: different RNG implementations and minor operator ordering." + ) + bits = [] + for name, block in problems.items(): + tests = block.get("tests") + if not tests: + continue + bits.append( + f"{name.upper()} U = {fmt(tests['mannwhitney']['U'])}, " + f"p = {fmt(tests['mannwhitney']['p'])}, " + f"median ratio {fmt(tests['median_ratio'])}" + ) + if bits: + lines.append( + "- These tests are not a 1–2% equivalence claim. " + ". ".join(bits) + "." + ) lines += [ - "## Notes", - "", - "- Not bit-identical: different RNG implementations and minor operator ordering.", - "- These tests are not a 1–2% equivalence claim. Report the Mann–Whitney result as computed. " - "DTLZ2 pymoo runs in this script use n_var=12; a table generated before that change is the n_var=10 column.", "- Reproduce: `python tools/oracle/run_multiseed_wilcoxon.py`", "", ] - return "\n".join(lines) + return lines def main() -> int: