diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..468dfaa --- /dev/null +++ b/.editorconfig @@ -0,0 +1,16 @@ +root = true + +[*] +charset = utf-8 +insert_final_newline = true +indent_style = space +indent_size = 4 + +[*.{json,yaml,yml}] +indent_size = 2 + +[*.cs] +csharp_style_namespace_declarations = file_scoped:suggestion + +[*.md] +trim_trailing_whitespace = false diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index ef0b61a..13152d5 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,4 +1,2 @@ -# These are supported funding model platforms - github: [magiccodingman] custom: ['https://sayou.biz/support', 'https://paypal.me/lancewr'] diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 0000000..3006923 --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,3 @@ +self-hosted-runner: + labels: + - magicquant-smoke diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..0b69677 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,7 @@ +## Change + +Describe the problem and resulting behavior. Note changes to numerical policy, configuration, paths, or persisted formats. + +## Validation + +List relevant tests and any model/hardware checks. State material limitations. diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml new file mode 100644 index 0000000..9889be7 --- /dev/null +++ b/.github/workflows/dotnet.yml @@ -0,0 +1,47 @@ +name: .NET checks +on: + push: + branches: [main] + pull_request: +permissions: + contents: read +jobs: + test: + runs-on: ${{ matrix.os }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + configuration: [Debug, Release] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m unittest discover -s scripts -p "test_*.py" + - run: dotnet restore MagicQuant.sln --locked-mode -warnaserror + - run: dotnet build MagicQuant.sln --configuration ${{ matrix.configuration }} --no-restore -warnaserror + - run: dotnet test MagicQuant.sln --configuration ${{ matrix.configuration }} --no-build --blame-hang-timeout 2m --blame-hang-dump-type none --logger trx --logger "console;verbosity=normal" --results-directory TestResults + - run: dotnet pack src/MagicQuant --configuration ${{ matrix.configuration }} --no-restore -p:Version=0.0.0-ci -o artifacts -warnaserror + - run: python scripts/package_smoke.py artifacts/MagicQuant.0.0.0-ci.nupkg + - uses: actions/upload-artifact@v4 + if: always() + with: + name: test-results-${{ matrix.os }}-${{ matrix.configuration }} + path: TestResults/ + secrets: + name: Secret scan + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python scripts/scan_secrets.py diff --git a/.github/workflows/model-smoke.yml b/.github/workflows/model-smoke.yml new file mode 100644 index 0000000..ae587fc --- /dev/null +++ b/.github/workflows/model-smoke.yml @@ -0,0 +1,41 @@ +name: Model smoke (manual) +on: + workflow_dispatch: + inputs: + model_path: + description: Complete source model on the trusted runner + required: true + llama_root: + description: Existing llama.cpp checkout on the trusted runner + required: true + runtime_root: + description: Existing MagicQuant runtime with Python dependencies + required: true + output_root: + description: Dedicated smoke output directory on the trusted runner + required: true +permissions: + contents: read +jobs: + smoke: + # Never trigger self-hosted execution automatically from untrusted pull requests. + runs-on: [self-hosted, magicquant-smoke] + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + - run: dotnet restore MagicQuant.sln --locked-mode + - run: dotnet test tests/MagicQuant.Tests -c Release --no-restore --filter Category=ModelSmoke --logger trx --results-directory TestResults + env: + MQ_RUN_MODEL_SMOKE: '1' + MQ_SMOKE_MODEL: ${{ inputs.model_path }} + MQ_SMOKE_LLAMA_ROOT: ${{ inputs.llama_root }} + MQ_SMOKE_RUNTIME_ROOT: ${{ inputs.runtime_root }} + MQ_SMOKE_OUTPUT: ${{ inputs.output_root }} + - uses: actions/upload-artifact@v4 + if: always() + with: + name: smoke-test-results + path: TestResults/*.trx diff --git a/.github/workflows/publish-nuget.yml b/.github/workflows/publish-nuget.yml new file mode 100644 index 0000000..74d242f --- /dev/null +++ b/.github/workflows/publish-nuget.yml @@ -0,0 +1,84 @@ +name: Publish NuGet +on: + push: + branches: [release] + workflow_dispatch: +permissions: + contents: read +jobs: + validate: + if: github.ref == 'refs/heads/release' + runs-on: ${{ matrix.os }} + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: dotnet restore MagicQuant.sln --locked-mode -warnaserror + - run: dotnet build MagicQuant.sln -c Release --no-restore -warnaserror + - run: dotnet test MagicQuant.sln -c Release --no-build --blame-hang-timeout 2m --blame-hang-dump-type none + - run: python -m unittest discover -s scripts -p "test_*.py" + - run: dotnet pack src/MagicQuant -c Release --no-restore -p:Version=0.0.0-ci -o artifacts -warnaserror + - run: python scripts/package_smoke.py artifacts/MagicQuant.0.0.0-ci.nupkg + publish: + needs: validate + runs-on: ubuntu-latest + timeout-minutes: 20 + environment: release + permissions: + contents: write + id-token: write + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Reserve or reuse the version for this commit + id: version + run: | + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + python scripts/release_version.py --reserve + - run: dotnet restore MagicQuant.sln --locked-mode -warnaserror + - name: Pack exact release version + env: + RELEASE_VERSION: ${{ steps.version.outputs.version }} + run: dotnet pack src/MagicQuant -c Release --no-restore -p:Version="$RELEASE_VERSION" -p:ContinuousIntegrationBuild=true -o artifacts -warnaserror + - name: Validate the release package + env: + RELEASE_VERSION: ${{ steps.version.outputs.version }} + run: python scripts/package_smoke.py "artifacts/MagicQuant.$RELEASE_VERSION.nupkg" + - uses: actions/upload-artifact@v4 + with: + name: nuget-${{ steps.version.outputs.version }} + path: artifacts/*.nupkg + - name: NuGet login through trusted publishing + uses: NuGet/login@v1 + id: login + with: + user: ${{ secrets.NUGET_USER }} + - name: Publish tested package + env: + NUGET_API_KEY: ${{ steps.login.outputs.NUGET_API_KEY }} + run: dotnet nuget push artifacts/*.nupkg --source https://api.nuget.org/v3/index.json --api-key "$NUGET_API_KEY" --skip-duplicate + - name: Create release record + env: + GH_TOKEN: ${{ github.token }} + RELEASE_VERSION: ${{ steps.version.outputs.version }} + run: | + if ! gh release view "v$RELEASE_VERSION" >/dev/null 2>&1; then + gh release create "v$RELEASE_VERSION" --verify-tag --title "MagicQuant $RELEASE_VERSION" --generate-notes + fi diff --git a/.gitignore b/.gitignore index f6b6248..df057d6 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,81 @@ .obsidian/ + +# Build results +bin/ +obj/ + +# Rider / JetBrains +.idea/ +*.sln.iml + +# Visual Studio user settings +*.user +*.userosscache +*.suo +*.cache +*.dbmdl +*.bak +*.ncb +*.opendb +*.VC.db + +# Other common C# stuff +*.log +*.vs/ + +# Local campaigns and generated model/runtime artifacts +config.local.yaml +*.local.yaml +*.dev.yaml +**/MagicQuant_SQLite.db* +*.duckdb +*.duckdb.wal +*.gguf +*.safetensors +.MagicQuant_tmp/ +TestResults/ +artifacts/ + +__pycache__/ + +# Visual Studio / Rider / VS Code per-user and machine state +.vs/ +.idea/ +.vscode/* +!.vscode/extensions.json +!.vscode/settings.example.json +*.DotSettings.user +*.sln.DotSettings.user +*.slnx.user +*.rsuser +*.sln.docstates +_ReSharper*/ +*.ncrunch* +_NCrunch*/ +[Bb]enchmark[Dd]ot[Nn]et.[Aa]rtifacts/ +[Tt]est[Rr]esults/ +coverage/ +*.coverage +*.coveragexml +*.testlog + +# Local secrets, package output and OS/editor temporary files +.env +.env.* +!.env.example +*.pfx +*.p12 +*.key +*.pem +*.nupkg +*.snupkg +.DS_Store +Thumbs.db +Desktop.ini +*~ +*.swp +*.swo + +# Local campaign configuration and tool installs +config.yaml +.tool-install/ diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 0000000..dcbd30a --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,10 @@ +# Retain every default detector. This exact tensor fixture is not an API key. +[extend] +useDefault = true + +[[allowlists]] +description = "Exact native/external tensor-name fixture, including its historical path" +condition = "AND" +paths = ['''(^|/)MagicQuant.Tests/ExternalBaselineTensorParityTests\.cs$'''] +regexTarget = "line" +regexes = ['''^\s*var (native|external) = Metadata\(nextnLayers: 1, "token_embd\.weight", "blk\.0\.attn_q\.weight", "blk\.64\.nextn\.eh_proj\.weight"\);$'''] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..08172ab --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,50 @@ +# Contributing + +Start with the [architecture map](docs/architecture.md), [configuration rules](docs/configuration.md), and the [research wiki](https://github.com/magiccodingman/MagicQuant). This repository implements benchmark-driven discovery; the `evolution` name survives as a compatibility alias. + +## Local workflow + +```sh +dotnet restore MagicQuant.sln --locked-mode +dotnet build MagicQuant.sln -c Debug --no-restore +dotnet test MagicQuant.sln -c Debug --no-build +dotnet build MagicQuant.sln -c Release --no-restore +dotnet test MagicQuant.sln -c Release --no-build +``` + +CI runs both configurations on Linux and Windows with warnings treated as errors and locked package restores. Use `--filter FullyQualifiedName~YourTestClass` to focus a test run during development. Tests run serially because configuration and runtime registries are global. Source-contract regression tests assume the normal repository/build layout; run the suite from the checkout rather than copying the test DLL elsewhere. + +Keep personal settings in an ignored `config.local.yaml` and pass `--config` explicitly. Do not add machine paths or automatic DEBUG campaigns to `Program.cs`. Use IDE run arguments for your campaign. Never commit weights, runtime databases, exported GGUFs, credentials, or local logs. + +## Making a change + +- Keep commands focused on orchestration; extract cohesive policy or path logic into services when it can be tested independently. +- Use existing baseline identity, effective-state, and path helpers. Do not duplicate database filenames, tensor-slot ordering, or custom-baseline normalization. +- Explain why a non-obvious constraint exists in a comment. Avoid comments that merely restate a method call or retain obsolete blocks of disabled implementation. +- When adding a config option, update the typed model, loader override if needed, commented default YAML, documentation, and a behavior test. Distinguish C# defaults from the distributed YAML profile. +- Test observable behavior: boundary cases, context scoping, cache reuse/invalidation, ranking/tie rules, path resolution, or failure propagation. Avoid tests that only repeat the implementation's constants without exercising a contract. +- Treat persisted IDs, database schemas, manifests, and artifact names as compatibility contracts. Provide an explicit migration plan for changes to them. + +For a bug fix, add a regression that fails without the fix. Tests that mutate `Config`, `Cache`, or registries must restore prior state in `finally`; use unique temporary roots. Keep unit tests free of network downloads, sudo, model quantization, and persistent changes to a developer's runtime. + +## Hardware integration changes + +Changes to conversion, quantization arguments, benchmark scheduling, or numerical selection need a small-model integration check in addition to unit tests. Record model identity, hardware, imatrix, dependency revisions, command/config, observed outputs, and before/after metrics. Use an isolated output directory. Do not claim full quantization parity from a passing unit suite. + +For documentation or path refactoring, verify examples against actual help and protect historical path rules. Avoid re-running expensive full campaigns when the changed behavior can be checked directly. + +## Pull request expectations + +Describe the concrete problem and resulting behavior, relevant compatibility effects, and validation performed. Separate numerical policy changes from mechanical cleanup when possible. Mention untested hardware/platform paths and any remaining compiler warnings. Prefer focused commits that can be reviewed without reconstructing the conversation that led to them. + +The maintainer still needs to choose a software license before an open-source release; do not infer one from generated model metadata or dependency licenses. + +See [testing and merge checks](docs/testing.md) for the manual small-model workflow, +package lock updates, and required-check setup. [Worked examples](docs/extending.md) +show how to add configuration and test native/process/path changes. + +## Repository layout and release safety + +Application projects are under `src/`; test projects are under `tests/`. Program guides belong in `docs/`; research explanations belong in `wiki/`. Use relative links so documentation remains useful in checkouts and forks. Source paths are separate from model/runtime data paths. + +Run the [installed-package checks](docs/testing.md#installed-package-and-release-checks) when changing paths, packaging, startup, or bundled files. Publication is controlled by [release branch automation](docs/releases.md). Never put NuGet keys or personal configuration into a workflow. Original contributions are accepted under the repository's AGPL-3.0-only license; retain third-party attribution. diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 0000000..f860221 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,5 @@ + + + true + + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..be3f7b2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/MagicQuant.sln b/MagicQuant.sln new file mode 100644 index 0000000..05df0cb --- /dev/null +++ b/MagicQuant.sln @@ -0,0 +1,37 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MagicQuant", "src\MagicQuant\MagicQuant.csproj", "{9259012B-0EB2-4AD8-81E5-807FD4465AA3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MQ.DB", "src\MQ.DB\MQ.DB.csproj", "{A97D6992-2659-47F9-9AC9-99425D2677A4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MagicQuant.Tests", "tests\MagicQuant.Tests\MagicQuant.Tests.csproj", "{D106FC82-5FD7-4C95-BF20-0940C64A234C}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MagicQuant.ProcessFixture", "tests\MagicQuant.ProcessFixture\MagicQuant.ProcessFixture.csproj", "{6F77FCF7-A105-44A9-A708-2B9F9F2B3B6D}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {9259012B-0EB2-4AD8-81E5-807FD4465AA3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9259012B-0EB2-4AD8-81E5-807FD4465AA3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9259012B-0EB2-4AD8-81E5-807FD4465AA3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9259012B-0EB2-4AD8-81E5-807FD4465AA3}.Release|Any CPU.Build.0 = Release|Any CPU + {A97D6992-2659-47F9-9AC9-99425D2677A4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A97D6992-2659-47F9-9AC9-99425D2677A4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A97D6992-2659-47F9-9AC9-99425D2677A4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A97D6992-2659-47F9-9AC9-99425D2677A4}.Release|Any CPU.Build.0 = Release|Any CPU + {D106FC82-5FD7-4C95-BF20-0940C64A234C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D106FC82-5FD7-4C95-BF20-0940C64A234C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D106FC82-5FD7-4C95-BF20-0940C64A234C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D106FC82-5FD7-4C95-BF20-0940C64A234C}.Release|Any CPU.Build.0 = Release|Any CPU + {6F77FCF7-A105-44A9-A708-2B9F9F2B3B6D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6F77FCF7-A105-44A9-A708-2B9F9F2B3B6D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6F77FCF7-A105-44A9-A708-2B9F9F2B3B6D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6F77FCF7-A105-44A9-A708-2B9F9F2B3B6D}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/README.md b/README.md index 0954c99..303aeea 100644 --- a/README.md +++ b/README.md @@ -1,222 +1,105 @@ -# MagicQuant (v2.0) +# MagicQuant -**MagicQuant is a benchmark-driven GGUF evaluation and hybrid-discovery system.** - -> **Which quantized models are actually worth using at each size?** - -Most quant releases give you a pile of files, AKA: Q8, Q6, Q5, Q4, and leave you to guess. MagicQuant replaces that guesswork with benchmarks, tensor-group probing, mixed hybrid GGUF builds when they are worth it, and a final survivor list built around meaningful size/fidelity tradeoffs. +[![NuGet version](https://img.shields.io/nuget/v/MagicQuant.svg)](https://www.nuget.org/packages/MagicQuant/) +[![NuGet downloads](https://img.shields.io/nuget/dt/MagicQuant.svg)](https://www.nuget.org/packages/MagicQuant/) +[![Build and tests](https://github.com/magiccodingman/MagicQuant/actions/workflows/dotnet.yml/badge.svg)](https://github.com/magiccodingman/MagicQuant/actions/workflows/dotnet.yml) +[![License: AGPL v3](https://img.shields.io/badge/License-AGPL_v3-blue.svg)](LICENSE) ---- +**Benchmark-driven GGUF quantization and mixed-precision hybrid discovery for llama.cpp.** -## What MagicQuant Does +MagicQuant helps answer: **which quantized versions of a model are worth keeping at each size?** It measures baseline quantizations, learns tensor-group assignments, explores hybrid combinations, and validates candidates against size and fidelity criteria. The result is a selected set of GGUF artifacts with supporting measurements, rather than an unranked collection of quantization levels. -MagicQuant takes the messy quantization space and turns it into a judged survivor list. - -It tests standard baselines, learns from external quant strategies, and builds mixed tensor-group hybrids when there may be a better size/fidelity trade hiding between normal quant levels. - -Then it validates the results. - -MagicQuant does not assume hybrids are better. It does not assume baselines are safe. Every option has to earn its slot. - -A final MagicQuant release is meant to show: - -* what is smallest -* what is safest -* what is meaningfully in-between -* what was removed as redundant or not worth the damage -* and what the real benchmark numbers say - -If a model survives MagicQuant, it survived because the trade was worth showing. +It is a .NET command-line application that orchestrates llama.cpp and Python tooling. It does not invent a new quantization format or use evolutionary search. Hybrids must earn their place: a standard baseline can be the better result. ---- +## How it works -## Example +1. **Establish baselines.** Read a local source model and measure standard quantization choices. Optionally learn tensor assignments from compatible external GGUFs. +2. **Probe tensor groups.** Measure how changes to groups such as attention, embeddings, and feed-forward tensors affect the model. +3. **Discover hybrids.** Use measured evidence and predictions to explore mixed-precision combinations with promising size/fidelity tradeoffs. +4. **Validate and select.** Measure candidates, reject poor or redundant trades, and export survivors with metadata and local provenance. -The following example is Qwen3-4B-2507-Instruct going through MagicQuants pipeline and the final results: +KLD and perplexity help evaluate fidelity; throughput and file size provide additional context. The results depend on the model, calibration/evaluation data, configuration, and hardware. A smaller KLD in one campaign is not a universal claim about downstream task quality. Read the [research overview](wiki/index.md) for the selection policy and its assumptions. -| Name | Provider | Quant Family | KLD | Size (GB) | -| ----------------------------------------------------------------------------------------- | ---------- | ------------ | -------: | --------: | -| LM-Q8_0 | llama.cpp | Q8_0 | 0.001339 | 3.99 | -| MQ-Q6_K_1 | MagicQuant | Q6_K | 0.001817 | 3.58 | -| UD-Q6_K_XL | Unsloth | UD-Q6_K_XL | 0.002111 | 3.41 | -| LM-Q6_K | llama.cpp | Q6_K | 0.004640 | 3.08 | -| [MQ-Q5_K_1](#winner-notes "Replaced: MQ-Q5_K") | MagicQuant | Q5_K | 0.006632 | 2.88 | -| [UD-Q5_K_XL](#winner-notes "Replaced: LM-Q5_K, LM-Q5_K_S") | Unsloth | UD-Q5_K_XL | 0.009839 | 2.73 | -| [MQ-Q4_K_M_1](#winner-notes "Replaced: MQ-Q4_K_M, UD-Q4_K_XL, LM-Q4_K_M + 1 more") | MagicQuant | Q4_K_M | 0.020346 | 2.44 | -| [LM-Q4_K_S](#winner-notes "Replaced: LM-IQ4_NL") | llama.cpp | Q4_K_S | 0.029803 | 2.22 | -| LM-IQ4_XS | llama.cpp | IQ4_XS | 0.031300 | 2.11 | -| UD-Q3_K_XL | Unsloth | UD-Q3_K_XL | 0.072278 | 1.98 | +## Support the project -The table above includes a mix of standard llama.cpp quantizations, Unsloth Dynamic GGUF models, and MagicQuant hybrids. +I build and maintain MagicQuant on the side, for free. Developing it and experimenting with quantizations has put a frankly ridiculous amount of terabytes written (TBW) on my drives! My Hugging Face storage is also creeping toward its cap, so there will eventually be more storage to fund. If this project helps you, [supporting the work](https://sayou.biz/support) helps with those costs. Anything helps and is always appreciated. ❤️ -In some cases, dominance is absolute. For example, Unsloth’s **Q5_K_XL** fully replaces the standard llama.cpp **Q5_K**, as MagicQuant determined the baseline offered no meaningful tradeoff in comparison. +## Install and run -More interesting are the hybrid outcomes. **MQ-Q4_K_M_1** emerged as a clear dominant variant, replacing multiple candidates simultaneously (_UD-Q4_K_XL, MQ-Q4_K_M, LM-Q4_K_M_). While baseline quants can sometimes achieve similar dominance, this case highlights a hybrid configuration that decisively outperformed across the board. +**Linux is the tested campaign platform.** Windows has automated build, unit-test, and packaged CLI checks; full Windows quantization campaigns have not been validated. No macOS campaign support is claimed. -**MQ-Q5_K_1** is another notable result. It leverages Unsloth’s learned tensor behavior (_Q5_K_XL_) within the `ffn_up_gate`, discovering a middle ground between **UD-Q5_K_XL** and **LM-Q6_K**. The result is a hybrid that achieves a disproportionately large KLD improvement relative to the additional size cost, exceeding a simple linear tradeoff. +Install the [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0), then install the CLI from [NuGet](https://www.nuget.org/packages/MagicQuant/): -The table below breaks down these MagicQuant hybrids by tensor group, showing the assigned quantization for each, whether derived from llama.cpp baselines or Unsloth’s learned tensor mappings. - -| Name | embeddings | attn_q | attn_kv | attn_output | ffn_up_gate | ffn_down | -| ----------- | ---------- | ------ | ------- | ----------- | ----------- | -------- | -| MQ-Q6_K_1 | Q8_0 | Q8_0 | Q8_0 | Q8_0 | Q6_K | Q8_0 | -| MQ-Q5_K_1 | Q8_0 | Q5_K | Q8_0 | Q6_K | UD-Q5_K_XL | Q5_K_S | -| MQ-Q4_K_M_1 | Q8_0 | Q5_K | Q8_0 | Q6_K | IQ4_XS | IQ4_XS | - ---- +```bash +dotnet tool install --global MagicQuant +magicquant --version +magicquant init-config --output config.yaml +``` -## Nonlinear Wins +The package becomes available after the first successful release publication; until then use the [source installation instructions](docs/setup.md#build-from-source). -MagicQuant does not look for simple "winners" in sub space between baselines. Instead it only allows nonlinear trade wins. Documentation presented later goes further into detail on this subject, but here's the TLDR: +Edit the generated config for your source model, architecture identity, export destination, and storage. Initialize the external toolchain, validate the config, then start the campaign: -Imagine a graph like this: -``` -Size → -| -| Q6 -| / -| / -| Q5 -| / -|Q4 -+---------------- +```bash +magicquant initialize-llama-cpp +magicquant pipeline --config ./config.yaml --check-config --strict-config +magicquant pipeline --config ./config.yaml ``` -A nonlinear win looks like: -``` - Q6 - / - / ← MQ-Q5_K_1 (above the line) - Q5 - / -Q4 -``` +Initialization can download/build llama.cpp and install Python dependencies. NuGet installs MagicQuant, not model weights or a complete GPU toolchain. Follow the [installation guide](docs/setup.md) for native prerequisites, GPU setup, custom toolchains, and environment paths. -That hybrid sits above the straight line between Q4 and Q5. +For updates: `dotnet tool update --global MagicQuant`. For reproducible runs, install a particular release with `--version X.Y.Z` and retain your config, model revision, and run provenance. -Meaning: -👉 It’s a **more efficient trade** than the normal step-up +## Configure a campaign -This is what MagicQuant calls a "nonlinear trade/win" when such wordage is used. +A minimal example (replace these paths and the architecture identity): ---- - -## Deeper Understanding - -For a deeper dive into MagicQuant and how it works, the [wiki index](https://github.com/magiccodingman/MagicQuant-Wiki/blob/main/wiki/index.md) is a good place to start. +```yaml +paths: + model_dir: /data/models/my-model + scratch_roots: + - /mnt/nvme-a/magicquant-scratch + - /mnt/nvme-b/magicquant-scratch +identity: + architecture_family_name: my-model-family +output: + output_dir: /data/exports/my-model-MagicQuant + output_name_prefix: MyModel +learning: + confirm_tensor_group_profile: true +``` -When you see a MagicQuant hybrid, it’s not just a “Q4.5” sitting somewhere between Q4 and Q5. It represents a discovered configuration where the **KLD reduction is non-linear relative to the size increase**, a genuinely better trade space. Not universally “better” than everything else, but a variant that earned its place through measurable advantage. +Custom YAML uses typed defaults for omitted values; it does not merge with the bundled tuning profile. Start with `init-config` when you want that complete profile. See [configuration](docs/configuration.md), [examples](examples), and the [command reference](docs/commands.md). -Whether the winner is a hybrid or a pure baseline from llama.cpp or Unsloth, any quant that removes another from the final selection does so because its dominance made the alternative no longer worth considering. +**Plan scratch storage early.** Quantization writes and rereads large intermediate models, and storage can be a major throughput limitation. Fast SSD/NVMe scratch disks, especially separate physical devices, can materially improve throughput when IO is the bottleneck. Multiple folders on the same device still share its bandwidth. Allow space for concurrent intermediate artifacts and keep unrelated data out of managed scratch/export directories. See [storage](docs/storage.md) and [best practices](docs/best-practices.md). -The goal is not to flood the space with near-duplicates offering negligible KLD gains for minimal size differences, nor to claim superiority for the sake of it. In fact, that’s explicitly what MagicQuant avoids. +## Learning from external quantizations -MagicQuant is built around transparency, honesty, maintainability, and most importantly trust. As it evaluates new architectures and quant families, it doesn’t invent quantization schemes in isolation. Instead, it learns from proven tensor assignments provided by trusted sources like llama.cpp and Unsloth. If those baselines are stable, MagicQuant operates within that same safe space, extending rather than reinventing. +External providers are optional. MagicQuant can run using its local baseline choices alone, but compatible external tensor assignments can provide valuable additional evidence. -Historical sources expand that tensor vocabulary; they do not vote on the current winner. MagicQuant pins the source revision, rebuilds the available recipes under current controlled conditions, and relearns their effects rather than replaying an old final mixture. +**Unsloth is the maintainer's recommended starting point** for external GGUF baselines. MagicQuant can learn their tensor-group patterns, rebuild a controlled equivalent from your local source model, and benchmark it in your campaign. It does not simply trust an external file's label or score. Choose the exact matching model and revision, and review its license. See the [Unsloth configuration walkthrough](docs/best-practices.md#optional-unsloth-baselines) and [research explanation](wiki/docs/Learning-From-Existing-Quantizations.md). -That said, the system is designed to adapt. Edge cases can exist, but the architecture is intentionally flexible to handle them. +For the same model, prefer linking to the original provider's surviving baselines. For a compatible variant they do not host, cloning can rebuild the full selected set locally. Learning tensor assignments does not automatically reproduce a provider's other processing techniques. See [publishing and cloning guidance](docs/best-practices.md#link-upstream-for-the-same-model-build-locally-for-variants). -### How MagicQuant Works +## Documentation -``` - ┌────────────────────────────┐ - │ Input Quantized Models │ - │ ───────────────────────── │ - │ llama.cpp / Unsloth / etc │ - └────────────┬──────────────┘ - │ - │ Inspect tensors - ▼ - ┌────────────────────────────┐ - │ Tensor Extraction Layer │ - │ ───────────────────────── │ - │ - Read all tensors │ - │ - Detect quant types │ - │ - Capture F32 / BF16 │ - └────────────┬──────────────┘ - │ - │ Group by role - ▼ - ┌────────────────────────────┐ - │ Tensor Group Mapping │ - │ ───────────────────────── │ - │ embeddings │ - │ attn_q / attn_kv / output │ - │ ffn_up_gate / ffn_down │ - │ lm_head / moe_* │ - └────────────┬──────────────┘ - │ - │ Learn configs - ▼ - ┌────────────────────────────┐ - │ Learned Config Library │ - │ ───────────────────────── │ - │ "Q5_K attn_q pattern" │ - │ "UD-Q5_K_XL ffn pattern" │ - │ etc │ - └────────────┬──────────────┘ - │ - │ Normalize external configs - ▼ - ┌────────────────────────────┐ - │ Controlled Rebuild Layer │ - │ ───────────────────────── │ - │ - Apply configs to BF16 │ - │ - Use MagicQuant imatrix │ - │ - Equal comparison ground │ - └────────────┬──────────────┘ - │ - │ Feed into - ▼ - ┌────────────────────────────┐ - │ Hybrid Construction Engine │ - │ ───────────────────────── │ - │ Mix tensor groups across │ - │ learned configurations │ - └────────────┬──────────────┘ - │ - │ Evaluate candidates - ▼ - ┌────────────────────────────┐ - │ Prediction + Isolation │ - │ ───────────────────────── │ - │ - Group-level testing │ - │ - Rank-safe prediction │ - │ - Controlled context tests │ - └────────────┬──────────────┘ - │ - │ Build real GGUF - ▼ - ┌────────────────────────────┐ - │ Benchmark Layer │ - │ ───────────────────────── │ - │ - KLD (primary) │ - │ - PPL (secondary) │ - │ - Measured GPU scheduling │ - └────────────┬──────────────┘ - │ - │ Final decision - ▼ - ┌────────────────────────────┐ - │ Survivor Selection │ - │ ───────────────────────── │ - │ - Dominance pruning │ - │ - Nonlinear winners │ - │ - Spacing collapse │ - └────────────────────────────┘ -``` +| Start here | What you will find | +| --- | --- | +| [Installation](docs/setup.md) | NuGet, native prerequisites, custom environments, source builds | +| [Configuration](docs/configuration.md) | YAML, overrides, read-only validation, profiles | +| [Commands](docs/commands.md) | Pipeline, setup, cloning, prediction validation | +| [Best practices](docs/best-practices.md) | Scratch disks, Unsloth, reproducibility, first campaigns | +| [Storage](docs/storage.md) | Persistent data, scratch leases, cache and output ownership | +| [Research](wiki/index.md) | Measurements, prediction, pruning, hybrid selection | +| [Contributing](CONTRIBUTING.md) | Development workflow, tests, code boundaries | +| [Releases](docs/releases.md) | Automatic versions and NuGet trusted publishing | -The controlled context tests check whether a promising group choice still behaves the same way when the surrounding model moves from a Q4-or-better regime into more aggressive compression. They are bounded and evidence-driven because exhaustive context testing would recreate the full combinatorial problem. +## Development and history -GPU scheduling is also measured rather than assumed. A large benchmark can use multiple GPUs in one shared process, while batches of smaller candidates can run concurrently on independent GPUs when that produces higher aggregate throughput. +Application code lives in `src/`, tests in `tests/`, operational guides in `docs/`, and research documentation in `wiki/`. Both the former MagicQuant-Wiki and MagicQuant-Pipeline histories are retained. The `evolution` command remains a compatibility alias for `pipeline`; existing database and artifact contracts are preserved. Historical research remains under `archival/` and is not current setup guidance. -The final release is a curated survivor menu. Research campaigns and cross-run audits should preserve the full nondominated evidence frontier before applying spacing, so that a presentation decision does not erase valid results. +## License -## Deep Dive Documentation +MagicQuant's original code and documentation are licensed under **GNU AGPL version 3 only** (`AGPL-3.0-only`). Commercial use is permitted subject to its terms. Distribution and remote interaction with modified versions carry source-availability obligations; the [license text](LICENSE) controls the details. -- [Wiki index](./wiki/index.md) -- [Prediction Engine](./wiki/docs/Prediction-Engine.md) -- [Regime-Aware Tensor Search](./wiki/docs/Regime-Aware-Search.md) -- [GPU Benchmark Scheduling](./wiki/docs/GPU-Benchmark-Scheduling.md) -- [Pareto Archives and Reproducibility](./wiki/docs/Pareto-Archives-And-Reproducibility.md) +This does not automatically relicense model weights or generated GGUFs. Model, dataset, external-provider, and third-party dependency licenses still apply. See [third-party notices](THIRD-PARTY-NOTICES.md). diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md new file mode 100644 index 0000000..e7e5f5b --- /dev/null +++ b/THIRD-PARTY-NOTICES.md @@ -0,0 +1,43 @@ +# Third-party notices + +MagicQuant's original code is AGPL-3.0-only. Dependencies retain their own licenses; they are not relicensed by this repository. The runtime dependency inventory below is taken from the committed application lock file. License texts and bundled notices are retained in [licenses/](licenses/), also included in the tool package. + +LibGit2Sharp is MIT; its native libgit2 component is GPL version 2 with an explicit linking exception permitting combinations with other programs. Keep that exception with its license. SQLitePCLRaw is Apache-2.0; SQLite itself is public domain. Other listed managed components use MIT or BSD-2-Clause. Build/test-only dependencies remain governed by their package notices. + +| Package | Version | License | +| --- | --- | --- | +| Blake3 | 2.2.0 | BSD-2-Clause | +| DuckDB.NET.Data.Full | 1.4.3 | MIT | +| LibGit2Sharp | 0.31.0 | MIT | +| Spectre.Console | 0.54.0 | MIT | +| System.Management | 10.0.11 | MIT | +| YamlDotNet | 17.0.1 | MIT | +| DuckDB.NET.Bindings.Full | 1.4.3 | MIT | +| LibGit2Sharp.NativeBinaries | 2.0.323 | GPL-2.0 with linking exception | +| Microsoft.Data.Sqlite | 10.0.11 | MIT | +| Microsoft.Data.Sqlite.Core | 10.0.11 | MIT | +| Microsoft.EntityFrameworkCore | 10.0.11 | MIT | +| Microsoft.EntityFrameworkCore.Abstractions | 10.0.11 | MIT | +| Microsoft.EntityFrameworkCore.Analyzers | 10.0.11 | MIT | +| Microsoft.EntityFrameworkCore.Relational | 10.0.11 | MIT | +| Microsoft.EntityFrameworkCore.Sqlite | 10.0.11 | MIT | +| Microsoft.EntityFrameworkCore.Sqlite.Core | 10.0.11 | MIT | +| Microsoft.Extensions.Caching.Abstractions | 10.0.11 | MIT | +| Microsoft.Extensions.Caching.Memory | 10.0.11 | MIT | +| Microsoft.Extensions.Configuration.Abstractions | 10.0.11 | MIT | +| Microsoft.Extensions.DependencyInjection | 10.0.11 | MIT | +| Microsoft.Extensions.DependencyInjection.Abstractions | 10.0.11 | MIT | +| Microsoft.Extensions.DependencyModel | 10.0.11 | MIT | +| Microsoft.Extensions.Logging | 10.0.11 | MIT | +| Microsoft.Extensions.Logging.Abstractions | 10.0.11 | MIT | +| Microsoft.Extensions.Options | 10.0.11 | MIT | +| Microsoft.Extensions.Primitives | 10.0.11 | MIT | +| SQLitePCLRaw.bundle_e_sqlite3 | 2.1.12 | Apache-2.0 | +| SQLitePCLRaw.core | 2.1.12 | Apache-2.0 | +| SQLitePCLRaw.lib.e_sqlite3 | 2.1.12 | Apache-2.0 | +| SQLitePCLRaw.provider.e_sqlite3 | 2.1.12 | Apache-2.0 | +| System.CodeDom | 10.0.11 | MIT | + +The native/Python toolchain (including llama.cpp, Python, PyTorch, llama-cpp-python and downloaded packages) is installed separately and retains its own licenses and notices. Model weights, datasets and external GGUF baselines are also separate works; check their specific terms before downloading or redistributing them. MagicQuant does not grant rights to third-party models or training data. + +When updating dependencies, refresh the lock files, review their license metadata and native component notices, and update this inventory. Package source repositories and exact source commits are recorded in their NuGet metadata; retain upstream notices when redistributing binaries. diff --git a/assets/icon.png b/assets/icon.png new file mode 100644 index 0000000..a2ca809 Binary files /dev/null and b/assets/icon.png differ diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..896250d --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,58 @@ +# Architecture and contributor code map + +## Execution flow + +`src/MagicQuant/Program.cs` dispatches through `CommandCatalog`. Help returns before runtime initialization. A normal command reads and validates YAML/CLI/input paths before loading `Config.Current` and run state into `MQ.DB.Cache`. It records provenance, cleans stale scratch, checks dependencies, and invokes an `ICommand`. `--check-config` exits before those runtime changes. + +`src/MagicQuant/Commands/QuantizationPipeline.cs` coordinates full discovery. `Evolution.cs` preserves the historical C# entry point and the CLI registry keeps `evolution` as an alias. The orchestrator should describe stage order; reusable behavior belongs in services. + +1. Validate source model and initialize model-local paths. +2. Obtain model hash; prepare native GGUF and optional projector; review tensor grouping. +3. Resolve architecture family and tensor-group profile; register custom baselines; handle targeted relearn. +4. Acquire imatrix context, rebucket existing tensor truth when appropriate, and load/probe the hardware plan. +5. Establish native benchmark truth and compatibility, then run initial/continuation isolation samples. +6. Apply isolation policy and materialize the remaining candidate space in DuckDB. +7. Fit predictions, investigate contextual evidence, choose candidates, and validate them with real benchmarks. +8. Finalize survivors and write GGUFs, manifests, benchmark summaries, and model cards. + +The [research wiki](https://github.com/magiccodingman/MagicQuant) is the source for the mathematical motivation. This guide maps the implementation, not a new algorithm specification. + +## Where to change things + +| Concern | Main files / services | +| --- | --- | +| CLI routing/help | `Program.cs`, `Commands/CommandCatalog.cs`, command `ShowHelp` methods | +| YAML shape and CLI overrides | `Configuration/MagicQuantYamlConfig.cs`, `MagicQuantYamlLoader.cs`, `config.default.yaml` | +| Baseline identity and roles | `MQ.DB/Models/BaselineQuants.cs`, `BaselineDefinitionResolver`, `HuggingFaceBaselineService` | +| Tensor grouping and profile review | `MQ.DB/tensor_groups.yaml`, `TensorGroupReviewService`, `TensorGroupProfileService`, `TensorGroupRebucketService` | +| Process/tool setup | `InitializeLlamaCpp`, `Helpers/LlamaBuilder`, `Helpers/PythonManager`, `HardwareHelper` | +| Native conversion and quantization | `NativeModelConversionService`, `QuantizationService`, `ExternalBaselineTensorParity`, `CloneManifestTensorMapBuildService` | +| Benchmark execution and GPU planning | `BenchmarkCommands`, `BenchmarkLogParser`, `BenchmarkService`, `BenchmarkGpuPlanning`, `LlamaGpuArgumentBuilder` | +| Isolation sampling and policy | `IsolationPlanningService`, `IsolationOptimizationService`, `Helpers/RuntimeSearchSpace` | +| SQLite measured truth | `MQ.DB/Data/MagicQuantContext.cs`, `MQ.DB/Models/DbModels`, `HybridBenchmarkRepository` | +| DuckDB candidate data | `QuantDatabaseService`, `RemainingCombinationStore`, `CombinationDuckDbSchema` | +| KLD prediction and final selection | `RankSafeKldPredictionService`, `PredictionGuidedHybridSelectionService`, `SmartBaselineTuningFallbackService` | +| Contextual anomaly/synergy evidence | `AnomalyWorkflowService`, `AnomalyRuleRepository`, `AnomalyAdjustedPredictionService` | +| Release artifacts | `HybridArtifactExportService`, `FinalArtifactNamingService`, `FinalReleaseMetadataService`, `ReadmeGenerationService` | +| Native process lifetime | `Runtime/ProcessRunner`, `NativeCommand`, `RunCancellation` | +| Run provenance | `RunProvenanceService` | +| Paths and lifecycle | `ModelArtifactPathService`, `ModelRuntimePathService`, `OutputPathService`, `CombinationDatabasePathService`, `ScratchStorageService` | + +## Invariants worth protecting + +- **Measured truth and prediction are different.** SQLite stores observations and context. DuckDB is a derived candidate/prediction workspace. Do not turn predictions into benchmark truth or silently substitute a standard family for a missing exact custom baseline measurement. +- **Identity is scoped.** Model hash, architecture family, tensor-group profile, baseline identity, and imatrix context determine which evidence may be reused. Similar display names are not sufficient. +- **Effective tensor assignments matter.** A carrier quant and an explicit group quant can describe the same effective assignment. Use existing resolvers and canonical baseline identities instead of inventing equality rules. +- **Runtime search state is mutable.** `RuntimeSearchSpace` controls the current allowed universe. Do not revive legacy static candidate lists as an authority. +- **Paths are contracts.** Writer and reader use `CombinationDatabasePathService` for the same DuckDB file. `OutputPathService` preserves command-specific destinations. Keep existing filenames, schema IDs, and serialized manifests stable unless a migration is part of the change. +- **Scratch is leased; downloads are durable.** Use `ScratchStorageService` leases for heavy temporary GGUF work, and durable external-baseline paths for reusable downloads. Do not add ad hoc cleanup of model roots. + +## Global state and tests + +`Config.Current`, `Cache`, and several baseline/search registries are process-wide mutable state. The CLI runs one command per process. Do not run multiple campaigns concurrently inside one process without redesigning those boundaries. + +Tests currently disable parallel execution because these globals are shared. Tests that change them must save and restore the prior state in `finally`, use unique temporary directories, and clean up only those directories. Prefer testing a pure policy/path helper when possible. Executable-level CLI tests protect the entry point separately from command implementation tests. + +Large benchmark, quantization, and selection services remain candidates for incremental extraction. Extract a cohesive responsibility behind regression tests instead of splitting files by arbitrary line count or changing numerical policy during a readability patch. + +`NativeModelConversionService` owns native artifact completion, while `QuantizationConcurrencyPlan` computes CPU/storage limits without IO. `IProcessRunner` permits failure/cancellation tests at that boundary. `RunCancellation` is an async-scoped bridge for legacy service APIs; new APIs should accept explicit cancellation tokens as well. Numerical policy and persisted evidence remain in their existing services. diff --git a/docs/best-practices.md b/docs/best-practices.md new file mode 100644 index 0000000..a43d666 --- /dev/null +++ b/docs/best-practices.md @@ -0,0 +1,88 @@ +# Campaign best practices + +## Start with a small, identifiable campaign + +Use a complete local source model that the selected llama.cpp converter supports. Keep the exact model revision, architecture/profile identity, imatrix settings, and evaluation data consistent when comparing runs. Begin with a small model and the generated profile before scaling up. Run `magicquant pipeline --config config.yaml --check-config --strict-config` first; this validates input structure and paths, not memory capacity or numerical quality. + +## Give scratch IO its own resources + +Large intermediate GGUFs make storage throughput a potential bottleneck. Prefer fast local SSD/NVMe scratch storage; separate physical disks can allow independent heavy writers. MagicQuant permits one heavy writer per configured scratch root, so two directories on the same disk do not create independent bandwidth and can increase contention. + +```yaml +paths: + scratch_roots: + - /mnt/nvme-a/magicquant-scratch + - /mnt/nvme-b/magicquant-scratch +``` + +Use existing writable parent locations dedicated to this work. Allow space for several large model artifacts, monitor free space and device throughput, and leave room for durable downloads and exports too. A faster disk helps when IO is limiting; GPU/CPU compute, RAM, and evaluation workload can instead dominate. Avoid fixed speedup expectations. See [storage ownership and cleanup](storage.md). + +## Optional Unsloth baselines + +The maintainer recommends Unsloth as a primary place to look for external GGUF tensor assignments. These sources are optional, and their value depends on model compatibility and measured results. Start with a repository for the exact source model; a similar name or matching architecture alone is insufficient. + +In the generated configuration, edit `baselines.custom_repositories`. A fuller, explicitly opt-in template is available in [examples/pipeline-external.yaml](../examples/pipeline-external.yaml), including how to add an external baseline to confirmed-anomaly expansion. The following is a structural example, not a promise that a particular upstream file exists. Replace the model/file placeholders, pin `revision` to the provider commit you inspected, and retain the rest of your campaign configuration: + +```yaml +baselines: + custom_repositories: + - repo_id: unsloth/YOUR-EXACT-MODEL-GGUF + revision: PROVIDER_COMMIT_SHA + enabled: true + short_source_name: UD + source_kind: huggingface_gguf_repository + require_all_includes_to_resolve: true + validate_tensor_names_against_source_model: true + includes: + - file_name: YOUR-EXACT-MODEL-UD-Q4_K_XL.gguf + baseline_family: Q4_K_M + quantize_base_name: Q4_K_M + display_name: UD_Q4_K_XL + allow_as_learning_baseline: true + allow_as_combination_carrier: true + allow_as_explicit_group_candidate: true +``` + +MagicQuant resolves the specified files, validates tensor-name parity, learns assignments, rebuilds using the local source model, and benchmarks the reconstruction. External downloads are durable cache data, distinct from temporary scratch artifacts. Review provider/model licensing and available disk space before enabling sources. Pure external learned baselines are not exported by default; inspect `output.export_external_learned_baselines` if you need them. + +For a provider-free campaign leave `baselines.custom_repositories` empty. See [learning from existing quantizations](../wiki/docs/Learning-From-Existing-Quantizations.md) for the research rationale. + +## Link upstream for the same model; build locally for variants + +For a release of the **same source model** that an external provider such as Unsloth already hosts, the maintainer recommends leaving this pipeline setting off: + +```yaml +output: + export_external_learned_baselines: false +``` + +MagicQuant will link external pure-baseline survivors to the provider instead of exporting local copies. This gives the original creator credit and downloads, avoids unnecessary duplicate hosting, and is the friendly default. MagicQuant's comparisons measure locally reconstructed tensor configurations under its own conditions. They do not, by themselves, establish whether the provider's original artifact is better or worse. Finding a useful hybrid or size/fidelity trade is not a reason to claim superiority over an untested upstream release. + +For a **different model variant**, such as an uncensored model or another fine-tune, the upstream repository may not host those weights. In that case, build the full selected set locally, including both MagicQuant hybrids and external-derived baseline configurations. If running the pipeline on that variant, enable local external-baseline exports: + +```yaml +output: + export_external_learned_baselines: true +``` + +The equivalent pipeline switch is `--export-external-learned-baselines`. Retain provider attribution and the applicable licenses even when rebuilding from different weights. + +**Clone command distinction:** `clone-repository-quants` already rebuilds every artifact entry in its input clone manifest, including external-derived entries; it does not consult this pipeline export flag. You do not need to enable the flag for that command. A source release can leave external export off and still include those configurations in its clone manifest. Clone mode rebuilds the entries present in that manifest, not every quantization ever offered by the provider. + +Cloning is a practical way to reuse a strong set of tensor configurations on a compatible variant without repeating full discovery. In the maintainer's experience, repeating discovery for modest fine-tunes can cost substantial time for little additional improvement. That is a starting assumption, not a guarantee: larger weight changes can shift the useful tradeoffs. Clone mode benchmarks the rebuilt artifacts locally, but does not repeat the full search or prove that inherited choices are optimal. Run discovery again when the model changes substantially, the measurements look poor, or you need stronger evidence for the target model. See the [clone command](commands.md#clone-known-tensor-configurations). + +## Limits of tensor-configuration copying + +MagicQuant learns quantization assignments for tensors and tensor groups, then rebuilds using the local source weights and its supported toolchain. **It does not automatically reproduce every technique used to create an external artifact.** A provider's extra weight transformations, custom quantization procedures, calibration recipes, or other processing are not reproduced merely because their tensor configuration was learned. Such behavior must be explicitly supported to be reproduced. + +Treat external configurations as evidence about useful assignments, not as a byte-for-byte clone of the provider's GGUF or a replication of its entire production process. Keep this distinction clear in release descriptions and benchmark claims. See the [research guide](../wiki/docs/Learning-From-Existing-Quantizations.md). + +## Retain enough evidence to reproduce a result + +Pin the MagicQuant package version and provider/model revisions. Keep the YAML, imatrix/evaluation data identity, llama.cpp revision, hardware context, and local `Runs/*/run.json` records. Provenance captures available versions and settings; it is not a complete frozen environment or numerical reproducibility guarantee. Remove private paths or credentials before sharing logs. + +Use the same measurement conditions for comparisons. Reuse validated caches deliberately; changing runtime roots or tensor profiles can change which evidence is selected. Avoid multiple campaigns in the same process and competing runs in the same model/runtime workspace. + +## Platform expectations + +Linux campaigns have been exercised, including a small-model conversion/quantization smoke test. Windows CI validates builds, ordinary tests, and tool packaging; end-to-end Windows campaigns remain unvalidated. Report failures with package/toolchain versions, sanitized configuration, and relevant logs. diff --git a/docs/commands.md b/docs/commands.md new file mode 100644 index 0000000..22dd852 --- /dev/null +++ b/docs/commands.md @@ -0,0 +1,72 @@ +# Commands and workflows + +Run these examples with the NuGet-installed `magicquant` CLI. Replace paths and identities with your own. For source builds, substitute `dotnet run --project src/MagicQuant -c Release --no-build --` for `magicquant`. `mq` in older help is shorthand, not an installed command. + +## Config creation and version + +`magicquant init-config --output config.yaml` copies the bundled tuning profile without runtime setup and refuses to overwrite a file. `magicquant --version` prints the application version and available source revision. + +## Full discovery pipeline + +```sh +magicquant pipeline \ + --config config.local.yaml \ + --model-dir /data/models/my-model \ + --architecture-family my-model-family \ + --output-dir /data/exports/my-model \ + --output-name-prefix MyModel +``` + +The pipeline converts/loads the native source, reviews tensor groups, resolves identity and baselines, measures isolation samples, predicts useful hybrids, validates them, and exports survivors. The exact runtime choices depend on benchmark truth and YAML policy. `evolution` invokes the same implementation. `build-hybrids` also enters the full pipeline; it is not limited to exporting previously selected files. + +`--use-imatrix` enables the configured acquisition/build flow. Provide an appropriate `imatrix` source in YAML. `--recheck-hardware-probe` refreshes execution-plan probing after hardware changes. `--skip-tensor-group-confirm` suppresses the tensor-group prompt for an already reviewed unattended campaign; it does not bypass all other confirmations. + +## Clone known tensor configurations + +```sh +magicquant clone-repository-quants \ + --config config.local.yaml \ + --model-dir /data/models/compatible-model \ + --architecture-family my-model-family \ + --source-json /data/releases/source/magicquant-manifest/magicquant.clone-configs.json \ + --output-dir /data/exports/cloned-model +``` + +Use `--source-repo owner/repo` instead to read a Hugging Face repository. `--source-json` also accepts an HTTP(S) URL. Clone mode rebuilds the tensor configurations and benchmarks them locally; it does not establish that the new model passed full discovery. + +Clone mode rebuilds all entries present in the manifest, including configurations originally learned from external providers. It does not use `output.export_external_learned_baselines`, which controls the pipeline's choice between local exports and upstream links. This is useful when cloning to a fine-tuned or uncensored variant that the original provider does not host. Review the [upstream-link versus variant-export guidance](best-practices.md#link-upstream-for-the-same-model-build-locally-for-variants) and [limits of tensor-configuration copying](best-practices.md#limits-of-tensor-configuration-copying). + +By default the manifest must match the target tensor inventory. `--allow-missing-manifest-tensors` explicitly allows a strict subset; unmatched target tensors use base quantization. `--missing-manifest-base-quant Q8_0` additionally selects that base quant. Use these only when that compatibility tradeoff is intended. + +## Validate predictions against existing measurements + +```sh +magicquant validate-predictions \ + --config config.local.yaml \ + --model-dir /data/models/my-model \ + --architecture-family my-model-family \ + --output-dir /data/reports/my-model +``` + +This writes `prediction_validation_general.csv` and `prediction_validation_general.md`. It uses existing general-category SQLite benchmark truth rather than launching a fresh full discovery campaign. Startup still runs the common dependency validation and stale-artifact cleanup. + +For imatrix measurements supply `--imatrix-path /data/imatrix.dat` or `--imatrix-identity-hash ` to select the exact bucket. Merely enabling `flags.use_imatrix` does not select a validation bucket. Without either option, validation uses no-imatrix truth. + +## Rerun and reuse + +```sh +magicquant pipeline \ + --config config.local.yaml --reuse-existing-final-artifacts +``` + +The normal pipeline can reuse scoped measurements, but final export normally cleans/rebuilds output. Artifact reuse is an additional opt-in: pipeline exports require exact filename and benchmark byte-size matches; clone mode also validates its benchmark JSON rows. A file merely existing is not sufficient. + +## Help and exit status + +No arguments, `help`, `--help`, or `-h` display top-level help. ` --help` and ` -h` display command help without config loading, cleanup, database access, or dependency installation. + +The host returns `0` on normal completion/help, `2` for an unknown command, `130` for cooperative cancellation, and `1` for an exception caught at the command boundary. Services may handle individual candidate failures and continue a campaign, so also inspect the reported sample failures and final artifacts. + +Use `--check-config` on a normal command to validate local inputs without running it. +Use `--strict-config` to reject unknown/inactive YAML settings rather than warning. +See [testing](testing.md) for the opt-in real-model workflow. diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..d80611e --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,73 @@ +# Configuration and paths + +Create the complete editable tuning profile with `magicquant init-config --output config.yaml`, then pass it explicitly with `--config config.yaml`. See [best practices](best-practices.md) for scratch storage and optional Unsloth baselines. + +## Loading and precedence + +`--config ` selects a YAML file; otherwise the executable loads its adjacent `config.default.yaml`. Debug and Release follow the same rule. `config.dev.yaml` is no longer selected automatically. + +The loader deserializes the selected file into `MagicQuantYamlConfig`, whose property initializers supply omitted fields, applies supported CLI overrides, normalizes values, and updates `Config.Current` and `MQ.DB.Cache`. It does **not** merge a custom file with `config.default.yaml`. The distributed YAML intentionally differs from C# defaults for some research tuning settings. Copy that whole file to reproduce its profile. + +Unknown CLI options, duplicate options, missing values, and values supplied to presence-only flags are rejected. CLI string options generally override nonblank YAML values. Many boolean switches only enable a feature; use YAML to disable it unless a specific negative CLI switch exists. Use `--name value` or `--name=value`; quote paths with spaces using normal shell quoting. + +Unknown or inactive YAML keys produce a warning with their setting path and line number; `--strict-config` rejects them. Compare with the commented default file and `src/MagicQuant/Configuration/MagicQuantYamlConfig.cs`. CI strictly parses the distributed examples so their keys cannot silently drift. + +## Model-neutral startup + +The bundled profile selects no model, architecture, provider repository, imatrix source, GPU limit, or scratch disk. `init-config` produces that same profile. Set model/architecture paths and review output/storage before your first campaign; a blank model is intentionally rejected by preflight. + +Standard llama.cpp families and general research thresholds remain populated so the file describes a usable starting policy. These values are not a Qwen preset or a claim that every model shares an optimum. The Qwen/other-family patterns in `src/MQ.DB/tensor_groups.yaml` are model-compatibility rules, not a selected campaign; do not erase them when configuring a different model. + +Confirmed-anomaly expansion defaults to built-in Q6_K/Q5_K candidates. Add exact external baseline names only after configuring that source. The [external-provider template](../examples/pipeline-external.yaml) shows this opt-in; it contains placeholders and must be edited before use. Existing explicit local campaign configs are not rewritten. + +## Main sections + +| Section | Responsibility | +| --- | --- | +| `paths` | Source model, runtime root, llama.cpp, scratch roots, external cache directory name | +| `identity` | Architecture-family name and explicit alias override | +| `flags` | Imatrix, hardware reprobe, high-precision hybrid policy | +| `learning` | Tensor review, safe profile rebucketing, targeted relearn requests | +| `baselines` | Built-in roles and external repository definitions, including optional revision pins | +| `hardware` | Per-GPU usable VRAM limits | +| `imatrix` | Local/remote matrix or dataset source | +| `isolation_pruning` | Isolation gating and damage/tradeoff thresholds | +| `prediction` | Rank-safe KLD fitting and combination memory limits | +| `candidate_selection` | Final candidate windows, fallback attempts, and spacing/tradeoff rules | +| `anomaly_detection`, `synergy_detection` | Bounded contextual probes and evidence adjustments | +| `output` | Export destination, naming, reuse, and external-baseline export policy | +| `readme` | Generated model card title and frontmatter | + +Built-in `standard_baselines_mode` is `all`, `selected`, or `none`. `selected` uses the explicit role lists; an empty list enables none for that role. In `all`, those lists do not restrict built-ins. Custom repositories are configured independently. Native anchors may still be required by the runtime. Historical `standard_only`/`custom_only` comments did not describe implemented filtering modes. + +## Path contracts + +Paths do not expand shell variables or `~` inside YAML. Prefer absolute paths. Relative `--config`, model, runtime, llama.cpp, dataset, and scratch paths resolve against the process working directory, **not** the YAML file's directory. + +| Setting / command | Blank default | Relative value | +| --- | --- | --- | +| `paths.magic_quant_root` | `/MagicQuant` | Process working directory | +| `paths.model_dir` | Required for model commands | Process working directory | +| Model work directory | `/MagicQuant` | Derived from model path | +| `pipeline` / `build-hybrids` output | `/MagicQuant/Final_Outputs` | Under `/MagicQuant` | +| Clone output | `/MagicQuant/FinalOutput` | Process working directory | +| Prediction validation output via CLI | `/MagicQuant/PredictionValidation` if no YAML output | Process working directory, exact destination | +| Prediction validation with YAML output only | `/PredictionValidation` | YAML output resolves against process working directory | + +These historical output differences are preserved for existing campaigns. `OutputPathService` owns the rules. An absolute `output_dir` avoids ambiguity. + +`external_baseline_cache_dir_name` is intended to be a folder name beneath the model work directory; use a simple name such as `ExternalBaselines`. Scratch roots are separate from durable downloads. They need not be physically distinct disks, but the scheduler's one-heavy-writer-per-root policy assumes you choose them thoughtfully. + +## Model metadata and compatibility + +`readme.frontmatter` accepts arbitrary scalar/list metadata. Set `license`, `base_model`, and other provenance fields for the actual exported model; no model license is inferred for you. + +The old `evolution`, `survival`, sensitivity-group, brain-layer, and collapse-penalty config surfaces had no active consumers and have been removed from the typed configuration. Old YAML containing them is tolerated with warnings unless `--strict-config` is selected; they do not tune the current algorithm. See `candidate_selection` and the research wiki for current selection policy. + +## Preflight and cancellation + +Normal runs validate local inputs before applying global state, creating runtime directories, installing dependencies, or cleaning artifacts. `--check-config` performs only this check. Non-finite numeric values and null required sections are rejected. Clone preflight requires one source manifest/repository, and model discovery requires explicit architecture-family identity. + +Exports cannot contain the source model/runtime root or overlap protected model work directories such as GGUF, Benchmarks, Logs, Runs, and ExternalBaselines. Physical symlink targets are considered. These guards do not make arbitrary existing export contents safe: still choose a dedicated directory. + +Ctrl+C requests cooperative cancellation, stops active native work, and returns status 130. A second Ctrl+C requests immediate OS termination. Process/lease cleanup is cooperative; forced termination or power loss can still require stale-artifact cleanup on the next run. diff --git a/docs/extending.md b/docs/extending.md new file mode 100644 index 0000000..c4edf4d --- /dev/null +++ b/docs/extending.md @@ -0,0 +1,27 @@ +# Worked contributor examples + +## Add a configuration option + +Suppose a future change adds a limit to a selection stage. First find the owning policy (`RuntimeCandidateSelectionConfig` and its service), and decide whether the option actually affects behavior. Do not add another dormant knob. + +1. Add a clearly named typed property with its default and a comment describing the decision it controls. +2. Add the underscored YAML key to `config.default.yaml`; decide explicitly whether the distributed tuning profile uses the same default. +3. If a CLI override is useful, add it to `MagicQuantYamlLoader.ApplyCliOverrides` and the value/flag contract in `CliOptionValidator`. Validate numeric input with invariant culture. Add preflight validation for constraints that should fail before work begins. +4. Add tests for omitted/default values, CLI precedence, and the observable stage behavior. Keep any changed `Config.Current`/`Cache` state scoped and restored. +5. Update command help and configuration documentation. Run the strict distributed-config tests and the complete suite. + +`YamlConfigurationDiagnostics` derives known keys from the typed schema, so it does not need a duplicate property-name list. Free-form `readme.frontmatter` and dictionary keys remain user-defined. + +## Change a path or process invocation + +For a new benchmark option, update `BenchmarkCommands`, then assert the literal argv sequence in `BenchmarkContractTests`. Use `NativeCommand.CreateStartInfo`, not interpolated shell strings. Include a path with spaces and metacharacters in the test. Keep retries in the calling service; `ProcessRunner` returns a nonzero exit code and only throws for launch/IO/cancellation failures. + +For conversion behavior, `NativeModelConversionService` accepts `IProcessRunner`. `NativeConversionTests` injects a small fake that writes a partial output and returns failure or cancellation. That verifies incomplete artifacts never acquire a reusable success marker without invoking Python or a model. Keep production code using the real runner by default. + +For output destinations, update `OutputPathService` and `PathSafety`, preserving existing command semantics or documenting a deliberate migration. Test ordinary paths, parent/child collisions, similarly prefixed sibling directories, and linked directories. Run the optional smoke workflow if native argument or artifact lifecycle behavior changed. + +## Work on numerical policy + +Read the research wiki and the owning service before editing. Add a regression around the actual measured/predicted tradeoff and its context identity. Do not replace exact custom tensor assignments with a built-in family surrogate merely to make a test pass. Model hash, architecture/profile identity, and imatrix scope are part of the input. + +Global configuration and registries still exist; this cleanup does not support multiple concurrent campaigns in one process. New helpers should accept explicit inputs, return results, and be testable without mutating those registries. Extract a responsibility with behavior tests rather than mechanically splitting a large class into partial files. diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..63f04c9 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,14 @@ +# Using and developing MagicQuant + +Start with the [project briefing and quick start](../README.md). These guides describe the application; the [research wiki](../wiki/index.md) explains its measurement and selection methodology. + +- [Setup](setup.md): NuGet installation, toolchain preparation, source builds, troubleshooting. +- [Configuration](configuration.md): YAML profile, CLI overrides and path rules. +- [Commands](commands.md): discovery, clone/export and prediction validation. +- [Best practices](best-practices.md): scratch disks, optional Unsloth baselines and reproducibility. +- [Storage](storage.md): runtime/model directories, caches, scratch and provenance. +- [Architecture](architecture.md): code map, service responsibilities and numerical invariants. +- [Extending the code](extending.md): worked examples for contributors. +- [Testing](testing.md): ordinary PR checks, package tests and opt-in model smoke. +- [Releases](releases.md): automatic versioning and trusted publishing setup. +- [Migration](migration.md): legacy command/config compatibility and repository history. diff --git a/docs/migration.md b/docs/migration.md new file mode 100644 index 0000000..23d9713 --- /dev/null +++ b/docs/migration.md @@ -0,0 +1,43 @@ +# Compatibility notes for existing users + +This cleanup retains numerical selection policy, SQLite schemas/migrations, manifest names, and existing output destinations. There is no data migration. + +## Startup and names + +- Prefer `pipeline`; `evolution` remains a case-insensitive CLI alias. The historical `Evolution` C# class delegates through inheritance to `QuantizationPipeline`. +- Debug builds no longer inject a personal command, model path, or architecture family. With no arguments, both build configurations show help. +- Pass `--config /path/to/your-config.yaml` explicitly for personal campaigns. Debug no longer auto-selects `config.dev.yaml`. Previously tracked developer campaigns were replaced by portable examples; existing local copies still load when selected explicitly. +- Command help runs before initialization. Unknown commands return status 2; caught command/config failures return status 1. +- `initialize-llama-cpp` now honors custom paths from YAML directly, caches the validated paths, and reports invalid custom environments as failures instead of printing an error and returning successfully. + +## Removed inactive surfaces + +The old evolution/survival knobs and unused sensitivity, brain-layer, collapse-penalty, and MoE-indicator config lists had no active runtime consumers. Their typed properties and inactive CLI overrides were removed. Legacy YAML keys warn (or fail with `--strict-config`); they never tuned the current chooser. Use `prediction`, `candidate_selection`, `anomaly_detection`, and `synergy_detection` for current policy. + +Startup no longer enumerates the entire combination universe merely to compare it with a count. That diagnostic helper remains available for explicit development checks. Actual pipeline generation and policy checks remain in place. + +## Paths and metadata + +Relative-path behavior is preserved, including `Final_Outputs` for pipeline and `FinalOutput` for clone. The shared path services make those differences explicit. Use absolute output paths when sharing configs. + +The distributed model-card template no longer asserts `apache-2.0` or a placeholder `base_model`. Set real provenance in your campaign YAML before publishing. Existing explicit frontmatter is still honored. + +The documentation now uses the implemented built-in baseline modes (`all`, `selected`, `none`). Older comments referring to `standard_only` and `custom_only` did not match the loader's behavior. + +To recover a previously tracked campaign before adopting the new layout, save it as +an ignored local config (replace the revision placeholder): + +```sh +git show :MagicQuant/config.dev.yaml > config.local.yaml +``` + +Then continue with `pipeline --config config.local.yaml` and your explicit model/family +arguments. The change to DEBUG startup does not change values inside that saved YAML. + +## Deeper readiness changes + +CLI typos/duplicate options and missing values now fail before work; unknown YAML keys warn, and `--strict-config` makes them errors. `--check-config` performs read-only preflight. Unsafe managed/output overlaps and missing model inputs fail before dependency setup or cleanup. Numeric CLI parsing uses an invariant decimal point. + +Native processes now share cancellation/log cleanup and use literal argv for benchmark and quantization launches. CPU llama-bench uses `-ngl 0` because the tested native version rejects the historical `-backend cpu` argument. Native conversion and low-level export require completion markers for reuse; partial/canceled builds are removed. Existing higher-level benchmark-size reuse checks remain in place. + +.NET package references were updated within the 10.0 patch line to remove the previously reported transitive vulnerabilities. Package lock files are committed. Database schemas and research selection formulas were not changed. diff --git a/docs/nuget-readme.md b/docs/nuget-readme.md new file mode 100644 index 0000000..efdf4a3 --- /dev/null +++ b/docs/nuget-readme.md @@ -0,0 +1,34 @@ +# MagicQuant + +**Benchmark-driven GGUF quantization and mixed-precision hybrid discovery for llama.cpp.** + +MagicQuant measures baselines, learns tensor-group assignments, discovers promising hybrid quantizations, and validates size/fidelity tradeoffs before exporting selected GGUF artifacts. It is a command-line tool, not an evolutionary search algorithm or a new quantization format. + +## Install + +Install the .NET 10 SDK, then: + +```sh +dotnet tool install --global MagicQuant +magicquant init-config --output config.yaml +``` + +Edit model, architecture, output, and scratch paths, then prepare the native toolchain and run: + +```sh +magicquant initialize-llama-cpp +magicquant pipeline --config config.yaml --check-config --strict-config +magicquant pipeline --config config.yaml +``` + +NuGet does not bundle model weights or a ready-to-use GPU toolchain. Linux is the tested campaign platform; Windows has automated build/unit/package checks but full campaigns remain unvalidated. + +Fast, separate physical scratch disks can help substantially when repeated large GGUF writes are the bottleneck. External quantization providers are optional; Unsloth is the maintainer's recommended starting point for compatible tensor-assignment evidence. + +- [Project briefing and research](https://github.com/magiccodingman/MagicQuant) +- [Installation and prerequisites](https://github.com/magiccodingman/MagicQuant/blob/main/docs/setup.md) +- [Configuration](https://github.com/magiccodingman/MagicQuant/blob/main/docs/configuration.md) +- [Scratch disks and optional Unsloth learning](https://github.com/magiccodingman/MagicQuant/blob/main/docs/best-practices.md) +- [Support spare-time development and storage costs](https://sayou.biz/support) + +MagicQuant is licensed under **AGPL-3.0-only**. Model weights and generated GGUFs retain their applicable licenses. See [license and third-party notices](https://github.com/magiccodingman/MagicQuant/blob/main/THIRD-PARTY-NOTICES.md). diff --git a/docs/releases.md b/docs/releases.md new file mode 100644 index 0000000..b94b364 --- /dev/null +++ b/docs/releases.md @@ -0,0 +1,43 @@ +# NuGet releases + +MagicQuant is packaged as a .NET tool, package ID `MagicQuant`, executable `magicquant`. `.github/workflows/publish-nuget.yml` publishes when a commit reaches `release`, normally through a merged PR. Pushes directly to `release` also trigger it; use branch protection to require PRs if desired. Manual dispatch is available to retry publication and only runs on `release`. + +## Trusted publishing setup + +Configure the following on NuGet.org under your account's **Trusted Publishing** settings: + +| Field | Value | +| --- | --- | +| Repository owner | `magiccodingman` | +| Repository | `MagicQuant` | +| Workflow filename | `publish-nuget.yml` | +| Environment | `release` | +| Package scope | `MagicQuant` (allow creation for the first release) | + +The workflow filename has no `.github/workflows/` prefix in NuGet's policy. The publishing job uses **`environment: release`** and requests `id-token: write`. Create that GitHub environment and restrict its deployment branch to `release`. Add the repository secret **`NUGET_USER`** containing your NuGet profile username, not an email address or API key. Choose the intended package owner when creating the policy. + +The workflow uses `NuGet/login@v1` to exchange GitHub OIDC identity for a temporary NuGet key immediately before pushing. No permanent NuGet API key is needed. See [NuGet's official guide](https://learn.microsoft.com/en-us/nuget/nuget-org/trusted-publishing). Private-repository policies may need activation within NuGet's stated time window; make sure the source for a public package is accessible to its recipients when launching. + +## Automatic versions + +The first release is **0.1.0**. Each new release commit increments the greatest reserved stable version's patch number: `0.1.0`, `0.1.1`, `0.1.2`, and so on. You do not edit a version for ordinary patch releases. + +`release-version.txt` is a **minimum next version**, not a counter. To release a new minor or major version, raise it in the release PR (for example, `0.2.0` or `1.0.0`). Leaving it unchanged continues patch increments. Only stable `major.minor.patch` versions are supported by the release workflow. + +After Linux/Windows validation passes, the workflow reserves the chosen version with an annotated `vX.Y.Z` tag pointing at the exact commit. Tag creation on the remote is the atomic claim; concurrent releases retry a version collision without sharing the same version. Jobs are not canceled merely because another release arrives. Publication order may differ if runs finish at different times. + +A retry of the **same commit** reuses its tag/version, even if later releases exist. A failed pack or push can leave a reserved tag; rerun that workflow to finish it. Do not delete or move release tags. A new commit gets a new version. Gaps are acceptable if a failed release is intentionally abandoned. + +The release package is built with that version and source revision, installed into a clean tool directory, and tested before publication. `--skip-duplicate` makes retrying an already published version harmless; the first successfully published package remains authoritative. NuGet versions are immutable. A GitHub release record is created after publication. + +## Validation and maintenance + +PR CI validates Linux and Windows, Debug and Release, with locked restores and warnings as errors. Each job tests a locally packed `0.0.0-ci` package without publishing it. Release validation repeats ordinary and package checks on Linux and Windows before reserving a version. The exact versioned release package is then verified again on Linux before upload. + +Package checks inspect bundled config, Python helper, native libraries, license/readme/icon metadata, and tool startup from a directory outside the source tree. They exercise config creation, refusal to overwrite, and read-only path validation. These are not full Windows campaigns or numerical parity tests. + +The package uses committed dependencies; the installed llama.cpp/Python runtime is separately managed. Record both for research reproducibility. Required merge checks and environment policies are repository settings, not guaranteed merely by committing a workflow. + +## Migration PR merge method + +The launch PR connects the original Wiki and Pipeline histories through an unsquashed subtree import, then reorganizes the tree in later commits. **Merge that PR with a merge commit, not squash or rebase**, to retain both histories in `main`. Do not force-push the existing repository history. Future ordinary PRs can use the project's preferred merge policy. diff --git a/docs/setup.md b/docs/setup.md new file mode 100644 index 0000000..04224ae --- /dev/null +++ b/docs/setup.md @@ -0,0 +1,82 @@ +# Setup and troubleshooting + +## Install from NuGet + +Linux is the tested campaign platform. Windows CI covers builds, ordinary tests and installed-package startup; full Windows campaigns remain unvalidated. Install the [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0) first. + +```sh +dotnet tool install --global MagicQuant +magicquant --version +magicquant init-config --output config.yaml +``` + +NuGet publication begins with the first successful release. If the package is not yet listed, use the source-build route below. The CLI command is `magicquant`; `dotnet add package` is not the installation command for this application. + +If your shell cannot find `magicquant`, ensure the .NET tools directory is on `PATH`: `$HOME/.dotnet/tools` on Linux or `%USERPROFILE%\.dotnet\tools` on Windows, then reopen the shell. Use `dotnet tool update --global MagicQuant` to update, `dotnet tool uninstall --global MagicQuant` to remove the tool, or add `--version X.Y.Z` to install an exact version. Removing/updating the tool does not remove model/runtime data. + +Edit the generated YAML; set `paths.model_dir`, `identity.architecture_family_name`, `output.output_dir`, and dedicated `paths.scratch_roots`. For a first run: + +```sh +magicquant initialize-llama-cpp +magicquant pipeline --config ./config.yaml --check-config --strict-config +magicquant pipeline --config ./config.yaml +``` + +`init-config` copies the full bundled tuning profile and refuses to overwrite existing files. `--check-config` is read-only. The actual pipeline can perform dependency setup and write model/runtime artifacts. Fast scratch disks are especially valuable for the repeated large GGUF writes; read [best practices](best-practices.md) before a large campaign. + +## Build from source + +All solution projects target `net10.0`. The solution includes `src/MagicQuant`, `src/MQ.DB`, `tests/MagicQuant.Tests`, and the offline `tests/MagicQuant.ProcessFixture` helper. + +```sh +git clone https://github.com/magiccodingman/MagicQuant.git +cd MagicQuant +dotnet restore MagicQuant.sln --locked-mode -warnaserror +dotnet build MagicQuant.sln -c Release --no-restore -warnaserror +dotnet test MagicQuant.sln -c Release --no-build +dotnet run --project src/MagicQuant -c Release --no-build -- init-config --output config.yaml +``` + +For source execution, replace `magicquant` in the other examples with `dotnet run --project src/MagicQuant -c Release --no-build --`. Build/test commands do not install llama.cpp or Python packages; ordinary tests do not require CUDA or weights. To exercise the actual package locally, see [testing](testing.md). + +## Runtime setup + +`initialize-llama-cpp` prepares the shared `/MagicQuant` installation. On apt-based Linux it checks build tools, CMake, Ninja, Git, Python/venv/pip, and libcurl development files; NVIDIA detection also adds a CUDA toolkit package. Missing packages may trigger sudo. The Python setup installs model conversion and dataset dependencies, PyTorch, and llama-cpp-python. `--update` requests dependency updates and a native rebuild, so record the resulting llama.cpp revision for reproducible research. + +The installer uses the configured hardware to choose native build options. macOS automatic setup is unsupported. Windows bootstrap code exists, but validate it on the target machine rather than assuming parity with Linux. + +You can bypass automatic native setup by providing an existing environment: + +```yaml +paths: + llama_root: /opt/llama.cpp + llama_bin: /opt/llama.cpp/build/bin + convert_script: /opt/llama.cpp/convert_hf_to_gguf.py +``` + +All three paths are required together. This branch validates their existence and detects hardware; it does not install the Python dependencies. The application expects its Python executable under `/MagicQuant-Env/bin/python` on Linux, or `MagicQuant-Env/python.exe` on Windows. Use the normal initializer first when using the default runtime root. `--validate` and `--verify` are historical setup flags, not read-only dependency checks. + +An explicit `paths.magic_quant_root` changes the SQLite/runtime root but **does not relocate the shared installer**. If you isolate that root, provision its expected Python environment as well. For example, on Linux an isolated root can link `MagicQuant-Env` to an already initialized shared environment. Make that choice explicitly; sharing Python still shares installed dependency versions. See [storage](storage.md) before moving existing campaign data. + +## Model input + +Use a complete source model directory supported by your llama.cpp conversion revision. `pipeline` requires at least one top-level `.safetensors` file. Model config/tokenizer files are also needed by conversion. A directory containing only a downloaded quantized GGUF is not a source model directory. + +Architecture-family identity scopes learned data. Choose the intended family deliberately; `--allow-architecture-family-alias-override` bypasses an identity guard and should not be a routine setup flag. + +## Troubleshooting + +| Symptom | Check | +| --- | --- | +| Missing SDK / unsupported target framework | `dotnet --info`; install a .NET 10 SDK | +| Missing config | Pass `--config` explicitly; the default is next to the executable | +| Missing model directory or safetensors | Set `paths.model_dir` to the complete local source model | +| Partial custom llama.cpp paths | Supply root, binary directory, and converter together | +| Python executable/package failure with isolated root | Check `/MagicQuant-Env` and its installed packages | +| Conversion or unknown tensor failure | Check model support in the actual llama.cpp checkout and review tensor grouping | +| Hardware plan no longer fits | Check GPU visibility and configured VRAM limits; use `--recheck-hardware-probe` after hardware changes | +| Unexpected export location | Check command-specific relative-path rules in [configuration](configuration.md) | +| Prediction reports use the wrong bucket | Use the same model/runtime database and exact imatrix identity as the measured run | +| No cached results reused | Check model hash, architecture family, tensor-group profile, and imatrix scope before considering relearn | + +For a useful bug report include the command, sanitized YAML, commit, .NET/OS/native dependency versions, GPU information, and relevant logs. Do not attach model weights or a whole runtime database by default. diff --git a/docs/storage.md b/docs/storage.md new file mode 100644 index 0000000..bfe81f1 --- /dev/null +++ b/docs/storage.md @@ -0,0 +1,54 @@ +# Storage, caching, and reruns + +The runtime root and model work directory are different things. With default settings: + +```text +/MagicQuant/ + MagicQuant_SQLite.db # measured truth, learned mappings, hardware probe state + llama.cpp/ # shared native checkout/build + MagicQuant-Env/ # Python environment + +/ + *.safetensors # input weights + config.json # source metadata and tokenizer assets alongside it + MagicQuant/ + GGUF/ # durable native/base artifacts + Benchmarks/ # measurements, corpora, reference logits + Logs/Quantization/ # quantization process logs + Runs//run.json # local campaign provenance and terminal status + ExternalBaselines/ # durable downloaded external GGUFs + MagicQuant_Combinations___.duckdb + Final_Outputs/ # pipeline default export directory + FinalOutput/ # clone default export directory + PredictionValidation/ # prediction report default +``` + +Output is configurable and only directories needed by a run are created. Other service-specific files may also appear. Final exports put JSON evidence under `magicquant-manifest/`, including clone configurations, final survivors, replacements, hybrid maps, isolation samples, and bad-trade reports as appropriate to the workflow. `MagicQuantManifestPathService` owns those filenames and links. + +## Durable truth + +SQLite is initialized/migrated automatically when its context is opened. Existing migrations and stored IDs are compatibility boundaries. Back up the database and related model artifacts while the process is stopped before manual moves or experiments. Do not delete the database as routine troubleshooting: it contains measured evidence that can be expensive to reconstruct. + +Changing the runtime root selects another SQLite database. It does not move data or the shared native installer. Changing a tensor-group regex/profile changes the applicable evidence scope; normal rebucketing can copy existing learned mappings into the new profile without erasing the original observations. Targeted relearn settings explicitly delete scoped truth after the program's confirmation step. + +## Derived candidate space + +DuckDB stores the current allowed combinations and prediction materialization. Both writer and reader resolve the same model/imatrix/high-precision filename through `CombinationDatabasePathService`. The pipeline rebuilds candidate data; the database is not interchangeable with SQLite benchmark evidence. + +Do not rename its files or add a scope component on only one side of a writer/reader pair. That can make a populated candidate space appear empty. + +## Scratch and cleanup + +Configured `paths.scratch_roots` hold `.MagicQuant_tmp` directories for leased heavy writes; blank configuration uses the model-local fallback. The service enforces one heavy writer per root and uses artifact leases and stale-state checks. Separate directory names do not establish separate physical disks. + +External baseline downloads are durable and managed separately from transient quantization artifacts. Startup cleanup runs before commands and model-specific cleanup runs after model paths are initialized. Keep personal files outside both managed scratch and dedicated export directories. + +## Reproducibility + +Retain the exact command, selected YAML, program commit, llama.cpp revision, model source revision/hash, external repository revision pins, imatrix identity/source, hardware plan, and emitted manifests/benchmark reports for a release. Custom repository `revision` can pin a branch, tag, or commit; a commit avoids moving references. Reusing output does not replace recording these inputs. + +## Local run provenance + +After preflight, each real command creates a unique `Runs//run.json` under the model work directory (or runtime root for setup). It snapshots argv, normalized input configuration, config SHA-256, program/.NET versions, and timestamps. Available llama.cpp revision, Python version/package inventory, final model/profile/imatrix identities, output path, and completion/failure/cancellation status are recorded as execution progresses. An unfinalized `running` record may indicate abrupt termination. + +Writes replace the manifest atomically. Records are separate from export cleanup and are **not published automatically**: argv/config may include private paths or URLs. Review before sharing. A missing tool-version field means it could not be obtained, not that the tool had a known default version. diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..a062963 --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,71 @@ +# Testing and PR checks + +## Ordinary checks + +```sh +dotnet restore MagicQuant.sln --locked-mode -warnaserror +dotnet build MagicQuant.sln -c Release --no-restore -warnaserror +dotnet test MagicQuant.sln -c Release --no-build +``` + +Repeat with `-c Debug` when changing startup or compilation-dependent behavior. CI runs both configurations on Linux and Windows. It restores the committed NuGet lock files, treats warnings as errors, runs all ordinary tests, and uploads TRX reports. `MagicQuant.ProcessFixture` is a small offline executable used to test native process exit, full stdout/stderr pipes, literal arguments, and cancellation; it is not a user command. + +The suite covers CLI startup/preflight, YAML contracts, managed/output path containment, symlinks on Linux, SQLite/DuckDB identity, native argument construction, log parsing, conversion success markers, concurrency policy, and existing research-policy regressions. It does not establish numerical equivalence for every model or hardware topology. Tests remain serial because the legacy runtime registries are shared. + +To update dependencies intentionally, edit package versions, run an unlocked `dotnet restore`, review `packages.lock.json` changes, and rerun the suite. Audit with: + +```sh +dotnet list MagicQuant.sln package --vulnerable --include-transitive +``` + +## Read-only campaign validation + +```sh +dotnet run --project src/MagicQuant -c Release -- pipeline \ + --config config.local.yaml --check-config --strict-config +``` + +This reads YAML and local input/path metadata but does not create a database, initialize tools, clean artifacts, or quantize. `--strict-config` turns unknown/inactive YAML keys into errors; without it, those keys produce warnings. Normal runs perform the same preflight before runtime mutation. This check verifies local paths and model input structure, not available RAM, remote repository existence, tokenizer compatibility with every converter, or numerical quality. + +## Opt-in small-model smoke test + +The test is explicitly skipped in ordinary CI. On a prepared Linux machine, set these variables to existing resources and a dedicated writable output parent: + +```sh +MQ_RUN_MODEL_SMOKE=1 \ +MQ_SMOKE_MODEL=/data/models/small-model \ +MQ_SMOKE_LLAMA_ROOT=/opt/llama.cpp \ +MQ_SMOKE_RUNTIME_ROOT=/data/MagicQuant \ +MQ_SMOKE_OUTPUT=/data/test-results/magicquant \ +dotnet test tests/MagicQuant.Tests -c Release --filter Category=ModelSmoke +``` + +The runtime root must contain `MagicQuant-Env` with the converter/gguf dependencies already installed. The test does not install dependencies or download a model. It copies source metadata and links weights into a unique test model directory containing spaces, then exercises native conversion/reuse, Q8 scratch leases, export/reuse, GGUF metadata parity, the native CPU benchmark, and manifest-path writing. It has a 20-minute cancellation deadline. It removes generated GGUFs and input weight links and retains logs plus `smoke-result.json` beneath the output parent. The sample tensor-map JSON is a smoke artifact, not a full clone/release manifest. + +This is an IO/toolchain smoke check, not a full discovery campaign or a PPL/KLD parity study. Quantization/selection policy changes still need before/after measurements on representative models. + +A manual GitHub Actions workflow is provided for a trusted self-hosted runner labeled `magicquant-smoke`. Review the selected ref before dispatching it. It never runs automatically for an incoming PR, and no runner has been provisioned by this change. Do not route untrusted PR code to a machine containing private model or campaign data. + +## Requiring checks before merge + +The workflow reports failures; branch protection or a ruleset must require its checks to block merges. Configure `main` to require all four Linux/Windows Debug/Release test jobs after the workflow has run. If merge queues are enabled later, add a `merge_group` workflow trigger as well. + +The unified MagicQuant repository is public. At launch preparation, `main` had no branch protection configured. Require the four `test (OS, Configuration)` checks plus `Secret scan` in repository settings if you want failures to block merging. Configure the `release` branch and deployment environment deliberately before publishing; a workflow alone does not prevent bypassing checks. + +## Installed-package and release checks + +```sh +python3 -m unittest discover -s scripts -p 'test_*.py' +dotnet pack src/MagicQuant -c Release --no-restore -p:Version=0.0.0-ci -o artifacts -warnaserror +python3 scripts/package_smoke.py artifacts/MagicQuant.0.0.0-ci.nupkg +``` + +Use `python` instead of `python3` where appropriate. The package smoke installs only from a temporary local feed, verifies shipped assets and native library presence, exercises CLI help/version and config creation outside the checkout, checks paths containing spaces, and confirms preflight is read-only. It never installs a model or native toolchain. Release-version tests use an isolated local bare Git remote; they never push to GitHub. + +PR CI runs these checks on both operating systems. [Release documentation](releases.md) explains the separately gated trusted-publishing workflow. + +## Secret checks + +The `Secret scan` CI job runs a checksum-pinned Gitleaks binary on full fetched history and the current tree. On Linux x64 run `python3 scripts/scan_secrets.py`. Reports redact candidate credentials. `.gitleaks.toml` retains default detectors and narrowly allows only the exact known tensor-name test fixture; do not suppress whole directories to silence new findings. + +A scanner is one check, not proof that every kind of sensitive information is absent. Review changes for private model names, personal paths, datasets, and credentials too. Git history preserves deleted files and author metadata. If a real credential is found, rotate it before planning any history rewrite. diff --git a/examples/clone.yaml b/examples/clone.yaml new file mode 100644 index 0000000..29482f7 --- /dev/null +++ b/examples/clone.yaml @@ -0,0 +1,12 @@ +# Supply --source-json or --source-repo on the command line. +# Clone rebuilds all manifest entries, including external-derived baselines. +# output.export_external_learned_baselines is a pipeline-only export choice; +# it is not needed to include those entries when cloning to a model variant. +paths: + model_dir: /data/models/my-compatible-model +identity: + architecture_family_name: my-model-family +output: + output_dir: /data/exports/my-cloned-model + output_name_prefix: MyClonedModel + reuse_existing_final_artifacts: true diff --git a/examples/pipeline-external.yaml b/examples/pipeline-external.yaml new file mode 100644 index 0000000..81c3b16 --- /dev/null +++ b/examples/pipeline-external.yaml @@ -0,0 +1,40 @@ +# Optional provider template, NOT a ready-to-run model preset. +# Replace ALL placeholders and confirm the exact model/revision/file match. +# For the bundled research tuning profile, first run magicquant init-config, +# then copy the baselines/anomaly sections below into your generated config. +# Loading this file directly uses typed defaults for omitted sections. +paths: + model_dir: /data/models/YOUR-SOURCE-MODEL +identity: + architecture_family_name: YOUR-MODEL-FAMILY +output: + output_dir: /data/exports/YOUR-MODEL-MagicQuant + output_name_prefix: YOUR-MODEL + # Prefer upstream links when the provider hosts the same model. + export_external_learned_baselines: false +baselines: + standard_baselines_mode: all + custom_repositories: + - repo_id: unsloth/YOUR-EXACT-MODEL-GGUF + revision: PROVIDER_COMMIT_SHA + enabled: true + short_source_name: UD + source_kind: huggingface_gguf_repository + require_all_includes_to_resolve: true + validate_tensor_names_against_source_model: true + delete_partial_or_dirty_downloads: true + resume_or_retry_downloads: true + includes: + - file_name: YOUR-EXACT-MODEL-UD-Q6_K_XL.gguf + baseline_family: Q6_K + quantize_base_name: Q6_K + display_name: UD_Q6_K_XL + allow_as_learning_baseline: true + allow_as_combination_carrier: true + allow_as_explicit_group_candidate: true +# Optional: include the configured external baseline in the small expansion pass +# around confirmed anomalies. This is not required just to learn from a provider. +anomaly_detection: + confirmed_anomaly_expansion: + allowed_reference_quants: [Q8_0] + allowed_candidate_quants: [Q6_K, Q5_K, UD_Q6_K_XL] diff --git a/examples/pipeline.yaml b/examples/pipeline.yaml new file mode 100644 index 0000000..6385850 --- /dev/null +++ b/examples/pipeline.yaml @@ -0,0 +1,12 @@ +# Copy to config.local.yaml and edit. Pass explicitly with --config. +# Omitted settings use C# defaults, not a merge with config.default.yaml. +# For the full distributed tuning profile, run magicquant init-config instead. +paths: + model_dir: /data/models/my-model +identity: + architecture_family_name: my-model-family +output: + output_dir: /data/exports/my-model-MagicQuant + output_name_prefix: MyModel +learning: + confirm_tensor_group_profile: true diff --git a/licenses/Blake3-license.txt b/licenses/Blake3-license.txt new file mode 100644 index 0000000..e091cb9 --- /dev/null +++ b/licenses/Blake3-license.txt @@ -0,0 +1,29 @@ +Copyright (c) 2020, Alexandre Mutel +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification +, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +> The underlying blake3_dotnet native library is using the code from https://github.com/BLAKE3-team/BLAKE3 +> with the following license https://github.com/BLAKE3-team/BLAKE3/blob/master/LICENSE + +This work is released into the public domain with CC0 1.0. Alternatively, it is +licensed under the Apache License 2.0. \ No newline at end of file diff --git a/licenses/DuckDB.NET.Bindings.Full-LICENSE-DuckDB.txt b/licenses/DuckDB.NET.Bindings.Full-LICENSE-DuckDB.txt new file mode 100644 index 0000000..a79d955 --- /dev/null +++ b/licenses/DuckDB.NET.Bindings.Full-LICENSE-DuckDB.txt @@ -0,0 +1,7 @@ +Copyright 2018-2022 Stichting DuckDB Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses/DuckDB.NET.Bindings.Full-LICENSE.md b/licenses/DuckDB.NET.Bindings.Full-LICENSE.md new file mode 100644 index 0000000..7a5aace --- /dev/null +++ b/licenses/DuckDB.NET.Bindings.Full-LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Giorgi Dalakishvili + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/licenses/LibGit2Sharp-LICENSE.md b/licenses/LibGit2Sharp-LICENSE.md new file mode 100644 index 0000000..c705543 --- /dev/null +++ b/licenses/LibGit2Sharp-LICENSE.md @@ -0,0 +1,21 @@ +The MIT License + +Copyright (c) LibGit2Sharp contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/licenses/LibGit2Sharp.NativeBinaries-libgit2.license.txt b/licenses/LibGit2Sharp.NativeBinaries-libgit2.license.txt new file mode 100644 index 0000000..701792e --- /dev/null +++ b/licenses/LibGit2Sharp.NativeBinaries-libgit2.license.txt @@ -0,0 +1,1410 @@ + libgit2 is Copyright (C) the libgit2 contributors, + unless otherwise stated. See the AUTHORS file for details. + + Note that the only valid version of the GPL as far as this project + is concerned is _this_ particular version of the license (ie v2, not + v2.2 or v3.x or whatever), unless explicitly otherwise stated. + +---------------------------------------------------------------------- + + LINKING EXCEPTION + + In addition to the permissions in the GNU General Public License, + the authors give you unlimited permission to link the compiled + version of this library into combinations with other programs, + and to distribute those combinations without any restriction + coming from the use of this file. (The General Public License + restrictions do apply in other respects; for example, they cover + modification of the file, and distribution when not linked into + a combined executable.) + +---------------------------------------------------------------------- + + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. + +---------------------------------------------------------------------- + +The bundled ZLib code is licensed under the ZLib license: + + (C) 1995-2022 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + +---------------------------------------------------------------------- + +The Clar framework is licensed under the ISC license: + +Copyright (c) 2011-2015 Vicent Marti + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +---------------------------------------------------------------------- + +The bundled PCRE implementation (deps/pcre/) is licensed under the BSD +license. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of the University of Cambridge nor the name of Google + Inc. nor the names of their contributors may be used to endorse or + promote products derived from this software without specific prior + written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- + +The bundled winhttp definition files (deps/winhttp/) are licensed under +the GNU LGPL (available at the end of this file). + +Copyright (C) 2007 Francois Gouget + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + +---------------------------------------------------------------------- + + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + +---------------------------------------------------------------------- + +The bundled SHA1 collision detection code is licensed under the MIT license: + +MIT License + +Copyright (c) 2017: + Marc Stevens + Cryptology Group + Centrum Wiskunde & Informatica + P.O. Box 94079, 1090 GB Amsterdam, Netherlands + marc@marc-stevens.nl + + Dan Shumow + Microsoft Research + danshu@microsoft.com + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- + +The bundled wildmatch code is licensed under the BSD license: + +Copyright Rich Salz. +All rights reserved. + +Redistribution and use in any form are permitted provided that the +following restrictions are are met: + +1. Source distributions must retain this entire copyright notice + and comment. +2. Binary distributions must include the acknowledgement ``This + product includes software developed by Rich Salz'' in the + documentation or other materials provided with the + distribution. This must not be represented as an endorsement + or promotion without specific prior written permission. +3. The origin of this software must not be misrepresented, either + by explicit claim or by omission. Credits must appear in the + source and documentation. +4. Altered versions must be plainly marked as such in the source + and documentation and must not be misrepresented as being the + original software. + +THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR IMPLIED +WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. + +---------------------------------------------------------------------- + +Portions of the OpenSSL headers are included under the OpenSSL license: + +Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) +All rights reserved. + +This package is an SSL implementation written +by Eric Young (eay@cryptsoft.com). +The implementation was written so as to conform with Netscapes SSL. + +This library is free for commercial and non-commercial use as long as +the following conditions are aheared to. The following conditions +apply to all code found in this distribution, be it the RC4, RSA, +lhash, DES, etc., code; not just the SSL code. The SSL documentation +included with this distribution is covered by the same copyright terms +except that the holder is Tim Hudson (tjh@cryptsoft.com). + +Copyright remains Eric Young's, and as such any Copyright notices in +the code are not to be removed. +If this package is used in a product, Eric Young should be given attribution +as the author of the parts of the library used. +This can be in the form of a textual message at program startup or +in documentation (online or textual) provided with the package. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. All advertising materials mentioning features or use of this software + must display the following acknowledgement: + "This product includes cryptographic software written by + Eric Young (eay@cryptsoft.com)" + The word 'cryptographic' can be left out if the rouines from the library + being used are not cryptographic related :-). +4. If you include any Windows specific code (or a derivative thereof) from + the apps directory (application code) you must include an acknowledgement: + "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + +THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +The licence and distribution terms for any publically available version or +derivative of this code cannot be changed. i.e. this code cannot simply be +copied and put under another distribution licence +[including the GNU Public Licence.] + +==================================================================== +Copyright (c) 1998-2007 The OpenSSL Project. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +3. All advertising materials mentioning features or use of this + software must display the following acknowledgment: + "This product includes software developed by the OpenSSL Project + for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + +4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + endorse or promote products derived from this software without + prior written permission. For written permission, please contact + openssl-core@openssl.org. + +5. Products derived from this software may not be called "OpenSSL" + nor may "OpenSSL" appear in their names without prior written + permission of the OpenSSL Project. + +6. Redistributions of any form whatsoever must retain the following + acknowledgment: + "This product includes software developed by the OpenSSL Project + for use in the OpenSSL Toolkit (http://www.openssl.org/)" + +THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY +EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR +ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- + +The xoroshiro256** implementation is licensed in the public domain: + +Written in 2018 by David Blackman and Sebastiano Vigna (vigna@acm.org) + +To the extent possible under law, the author has dedicated all copyright +and related and neighboring rights to this software to the public domain +worldwide. This software is distributed without any warranty. + +See . + +---------------------------------------------------------------------- + +The built-in SHA256 support (src/hash/rfc6234) is taken from RFC 6234 +under the following license: + +Copyright (c) 2011 IETF Trust and the persons identified as +authors of the code. All rights reserved. + +Redistribution and use in source and binary forms, with or +without modification, are permitted provided that the following +conditions are met: + +- Redistributions of source code must retain the above + copyright notice, this list of conditions and + the following disclaimer. + +- Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + +- Neither the name of Internet Society, IETF or IETF Trust, nor + the names of specific contributors, may be used to endorse or + promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND +CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- + +The built-in git_fs_path_basename_r() function is based on the +Android implementation, BSD licensed: + +Copyright (C) 2008 The Android Open Source Project +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +* Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT +OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +---------------------------------------------------------------------- + +The bundled ntlmclient code is licensed under the MIT license: + +Copyright (c) Edward Thomson. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +---------------------------------------------------------------------- + +Portions of this software derived from Team Explorer Everywhere: + +Copyright (c) Microsoft Corporation + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +--------------------------------------------------------------------------- + +Portions of this software derived from the LLVM Compiler Infrastructure: + +Copyright (c) 2003-2016 University of Illinois at Urbana-Champaign. +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +--------------------------------------------------------------------------- + +Portions of this software derived from Unicode, Inc: + +Copyright 2001-2004 Unicode, Inc. + +Disclaimer + +This source code is provided as is by Unicode, Inc. No claims are +made as to fitness for any particular purpose. No warranties of any +kind are expressed or implied. The recipient agrees to determine +applicability of information provided. If this file has been +purchased on magnetic or optical media from Unicode, Inc., the +sole remedy for any claim will be exchange of defective media +within 90 days of receipt. + +Limitations on Rights to Redistribute This Code + +Unicode, Inc. hereby grants the right to freely use the information +supplied in this file in the creation of products supporting the +Unicode Standard, and to make copies of this file in any form +for internal or external distribution as long as this notice +remains attached. + +--------------------------------------------------------------------------- + +Portions of this software derived from sheredom/utf8.h: + +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to + +--------------------------------------------------------------------------- + +Portions of this software derived from RFC 1320: + +Copyright (C) 1990-2, RSA Data Security, Inc. All rights reserved. + +License to copy and use this software is granted provided that it +is identified as the "RSA Data Security, Inc. MD4 Message-Digest +Algorithm" in all material mentioning or referencing this software +or this function. + +License is also granted to make and use derivative works provided +that such works are identified as "derived from the RSA Data +Security, Inc. MD4 Message-Digest Algorithm" in all material +mentioning or referencing the derived work. + +RSA Data Security, Inc. makes no representations concerning either +the merchantability of this software or the suitability of this +software for any particular purpose. It is provided "as is" +without express or implied warranty of any kind. + +These notices must be retained in any copies of any part of this +documentation and/or software. + +---------------------------------------------------------------------- + +The bundled llhttp dependency is licensed under the MIT license: + +Copyright Fedor Indutny, 2018. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, subject to the +following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN +NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/licenses/Microsoft.Extensions.Caching.Abstractions-THIRD-PARTY-NOTICES.TXT b/licenses/Microsoft.Extensions.Caching.Abstractions-THIRD-PARTY-NOTICES.TXT new file mode 100644 index 0000000..1e194f3 --- /dev/null +++ b/licenses/Microsoft.Extensions.Caching.Abstractions-THIRD-PARTY-NOTICES.TXT @@ -0,0 +1,1418 @@ +.NET Runtime uses third-party libraries or other resources that may be +distributed under licenses different than the .NET Runtime software. + +In the event that we accidentally failed to list a required notice, please +bring it to our attention. Post an issue or email us: + + dotnet@microsoft.com + +The attached notices are provided for information only. + +License notice for ASP.NET +------------------------------- + +Copyright (c) .NET Foundation. All rights reserved. +Licensed under the Apache License, Version 2.0. + +Available at +https://github.com/dotnet/aspnetcore/blob/main/LICENSE.txt + +License notice for Slicing-by-8 +------------------------------- + +http://sourceforge.net/projects/slicing-by-8/ + +Copyright (c) 2004-2006 Intel Corporation - All Rights Reserved + + +This software program is licensed subject to the BSD License, available at +http://www.opensource.org/licenses/bsd-license.html. + + +License notice for Unicode data +------------------------------- + +https://www.unicode.org/license.html + +Copyright © 1991-2024 Unicode, Inc. All rights reserved. +Distributed under the Terms of Use in https://www.unicode.org/copyright.html. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of data files and any associated documentation (the "Data Files") or +software and any associated documentation (the "Software") to deal in the +Data Files or Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Data Files or Software, and to permit persons to whom the +Data Files or Software are furnished to do so, provided that either (a) +this copyright and permission notice appear with all copies of the Data +Files or Software, or (b) this copyright and permission notice appear in +associated Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. + +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA +FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. + +License notice for zlib-ng +----------------------- + +https://github.com/zlib-ng/zlib-ng/blob/d54e3769be0c522015b784eca2af258b1c026107/LICENSE.md + +(C) 1995-2024 Jean-loup Gailly and Mark Adler + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + +3. This notice may not be removed or altered from any source distribution. + +License notice for opentelemetry-dotnet +--------------------------------------- + +https://github.com/open-telemetry/opentelemetry-dotnet/blob/805dd6b4abfa18ef2706d04c30d0ed28dbc2955e/LICENSE.TXT#L1 + +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +Copyright The OpenTelemetry Authors + + +License notice for LinuxTracepoints +----------------------------------- + +https://github.com/microsoft/LinuxTracepoints/blob/main/LICENSE + +Copyright (c) Microsoft Corporation. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE + +License notice for Mono +------------------------------- + +http://www.mono-project.com/docs/about-mono/ + +Copyright (c) .NET Foundation Contributors + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the Software), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +License notice for International Organization for Standardization +----------------------------------------------------------------- + +Portions (C) International Organization for Standardization 1986: + Permission to copy in any form is granted for use with + conforming SGML systems and applications as defined in + ISO 8879, provided this notice is included in all copies. + +License notice for Intel +------------------------ + +"Copyright (c) 2004-2006 Intel Corporation - All Rights Reserved + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License notice for Xamarin and Novell +------------------------------------- + +Copyright (c) 2015 Xamarin, Inc (http://www.xamarin.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +Copyright (c) 2011 Novell, Inc (http://www.novell.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +Third party notice for W3C +-------------------------- + +"W3C SOFTWARE AND DOCUMENT NOTICE AND LICENSE +Status: This license takes effect 13 May, 2015. +This work is being provided by the copyright holders under the following license. +License +By obtaining and/or copying this work, you (the licensee) agree that you have read, understood, and will comply with the following terms and conditions. +Permission to copy, modify, and distribute this work, with or without modification, for any purpose and without fee or royalty is hereby granted, provided that you include the following on ALL copies of the work or portions thereof, including modifications: +The full text of this NOTICE in a location viewable to users of the redistributed or derivative work. +Any pre-existing intellectual property disclaimers, notices, or terms and conditions. If none exist, the W3C Software and Document Short Notice should be included. +Notice of any changes or modifications, through a copyright statement on the new code or document such as "This software or document includes material copied from or derived from [title and URI of the W3C document]. Copyright © [YEAR] W3C® (MIT, ERCIM, Keio, Beihang)." +Disclaimers +THIS WORK IS PROVIDED "AS IS," AND COPYRIGHT HOLDERS MAKE NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO, WARRANTIES OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE OR DOCUMENT WILL NOT INFRINGE ANY THIRD PARTY PATENTS, COPYRIGHTS, TRADEMARKS OR OTHER RIGHTS. +COPYRIGHT HOLDERS WILL NOT BE LIABLE FOR ANY DIRECT, INDIRECT, SPECIAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF ANY USE OF THE SOFTWARE OR DOCUMENT. +The name and trademarks of copyright holders may NOT be used in advertising or publicity pertaining to the work without specific, written prior permission. Title to copyright in this work will at all times remain with copyright holders." + +License notice for Bit Twiddling Hacks +-------------------------------------- + +Bit Twiddling Hacks + +By Sean Eron Anderson +seander@cs.stanford.edu + +Individually, the code snippets here are in the public domain (unless otherwise +noted) — feel free to use them however you please. The aggregate collection and +descriptions are © 1997-2005 Sean Eron Anderson. The code and descriptions are +distributed in the hope that they will be useful, but WITHOUT ANY WARRANTY and +without even the implied warranty of merchantability or fitness for a particular +purpose. + +License notice for Brotli +-------------------------------------- + +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +compress_fragment.c: +Copyright (c) 2011, Google Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +""AS IS"" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +decode_fuzzer.c: +Copyright (c) 2015 The Chromium Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +""AS IS"" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE." + +License notice for Json.NET +------------------------------- + +https://github.com/JamesNK/Newtonsoft.Json/blob/master/LICENSE.md + +The MIT License (MIT) + +Copyright (c) 2007 James Newton-King + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +License notice for vectorized base64 encoding / decoding +-------------------------------------------------------- + +Copyright (c) 2005-2007, Nick Galbreath +Copyright (c) 2013-2017, Alfred Klomp +Copyright (c) 2015-2017, Wojciech Mula +Copyright (c) 2016-2017, Matthieu Darbois +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +- Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License notice for vectorized hex parsing +-------------------------------------------------------- + +Copyright (c) 2022, Geoff Langdale +Copyright (c) 2022, Wojciech Mula +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +- Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License notice for RFC 3492 +--------------------------- + +The punycode implementation is based on the sample code in RFC 3492 + +Copyright (C) The Internet Society (2003). All Rights Reserved. + +This document and translations of it may be copied and furnished to +others, and derivative works that comment on or otherwise explain it +or assist in its implementation may be prepared, copied, published +and distributed, in whole or in part, without restriction of any +kind, provided that the above copyright notice and this paragraph are +included on all such copies and derivative works. However, this +document itself may not be modified in any way, such as by removing +the copyright notice or references to the Internet Society or other +Internet organizations, except as needed for the purpose of +developing Internet standards in which case the procedures for +copyrights defined in the Internet Standards process must be +followed, or as required to translate it into languages other than +English. + +The limited permissions granted above are perpetual and will not be +revoked by the Internet Society or its successors or assigns. + +This document and the information contained herein is provided on an +"AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING +TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING +BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION +HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF +MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +Copyright(C) The Internet Society 1997. All Rights Reserved. + +This document and translations of it may be copied and furnished to others, +and derivative works that comment on or otherwise explain it or assist in +its implementation may be prepared, copied, published and distributed, in +whole or in part, without restriction of any kind, provided that the above +copyright notice and this paragraph are included on all such copies and +derivative works.However, this document itself may not be modified in any +way, such as by removing the copyright notice or references to the Internet +Society or other Internet organizations, except as needed for the purpose of +developing Internet standards in which case the procedures for copyrights +defined in the Internet Standards process must be followed, or as required +to translate it into languages other than English. + +The limited permissions granted above are perpetual and will not be revoked +by the Internet Society or its successors or assigns. + +This document and the information contained herein is provided on an "AS IS" +basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING TASK FORCE +DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +ANY WARRANTY THAT THE USE OF THE INFORMATION HEREIN WILL NOT INFRINGE ANY +RIGHTS OR ANY IMPLIED WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A +PARTICULAR PURPOSE. + +License notice for Algorithm from RFC 4122 - +A Universally Unique IDentifier (UUID) URN Namespace +---------------------------------------------------- + +Copyright (c) 1990- 1993, 1996 Open Software Foundation, Inc. +Copyright (c) 1989 by Hewlett-Packard Company, Palo Alto, Ca. & +Digital Equipment Corporation, Maynard, Mass. +Copyright (c) 1998 Microsoft. +To anyone who acknowledges that this file is provided "AS IS" +without any express or implied warranty: permission to use, copy, +modify, and distribute this file for any purpose is hereby +granted without fee, provided that the above copyright notices and +this notice appears in all source code copies, and that none of +the names of Open Software Foundation, Inc., Hewlett-Packard +Company, Microsoft, or Digital Equipment Corporation be used in +advertising or publicity pertaining to distribution of the software +without specific, written prior permission. Neither Open Software +Foundation, Inc., Hewlett-Packard Company, Microsoft, nor Digital +Equipment Corporation makes any representations about the +suitability of this software for any purpose." + +License notice for The LLVM Compiler Infrastructure +--------------------------------------------------- + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +License notice for Bob Jenkins +------------------------------ + +By Bob Jenkins, 1996. bob_jenkins@burtleburtle.net. You may use this +code any way you wish, private, educational, or commercial. It's free. + +License notice for Greg Parker +------------------------------ + +Greg Parker gparker@cs.stanford.edu December 2000 +This code is in the public domain and may be copied or modified without +permission. + +License notice for libunwind based code +---------------------------------------- + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +License notice for Printing Floating-Point Numbers (Dragon4) +------------------------------------------------------------ + +/****************************************************************************** + Copyright (c) 2014 Ryan Juckett + http://www.ryanjuckett.com/ + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + + 3. This notice may not be removed or altered from any source + distribution. +******************************************************************************/ + +License notice for Printing Floating-point Numbers (Grisu3) +----------------------------------------------------------- + +Copyright 2012 the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License notice for xxHash +------------------------- + +xxHash - Extremely Fast Hash algorithm +Header File +Copyright (C) 2012-2021 Yann Collet + +BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +You can contact the author at: + - xxHash homepage: https://www.xxhash.com + - xxHash source repository: https://github.com/Cyan4973/xxHash + +License notice for Berkeley SoftFloat Release 3e +------------------------------------------------ + +https://github.com/ucb-bar/berkeley-softfloat-3 +https://github.com/ucb-bar/berkeley-softfloat-3/blob/master/COPYING.txt + +License for Berkeley SoftFloat Release 3e + +John R. Hauser +2018 January 20 + +The following applies to the whole of SoftFloat Release 3e as well as to +each source file individually. + +Copyright 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018 The Regents of the +University of California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions, and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions, and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + 3. Neither the name of the University nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS", AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ARE +DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License notice for xoshiro RNGs +-------------------------------- + +Written in 2018 by David Blackman and Sebastiano Vigna (vigna@acm.org) + +To the extent possible under law, the author has dedicated all copyright +and related and neighboring rights to this software to the public domain +worldwide. This software is distributed without any warranty. + +See . + +License for fastmod (https://github.com/lemire/fastmod), ibm-fpgen (https://github.com/nigeltao/parse-number-fxx-test-data) and fastrange (https://github.com/lemire/fastrange) +-------------------------------------- + + Copyright 2018 Daniel Lemire + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +License for sse4-strstr (https://github.com/WojciechMula/sse4-strstr) +-------------------------------------- + + Copyright (c) 2008-2016, Wojciech Mula + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License notice for The C++ REST SDK +----------------------------------- + +C++ REST SDK + +The MIT License (MIT) + +Copyright (c) Microsoft Corporation + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +License notice for MessagePack-CSharp +------------------------------------- + +MessagePack for C# + +MIT License + +Copyright (c) 2017 Yoshifumi Kawai + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +License notice for lz4net +------------------------------------- + +lz4net + +Copyright (c) 2013-2017, Milosz Krajewski + +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License notice for Nerdbank.Streams +----------------------------------- + +The MIT License (MIT) + +Copyright (c) Andrew Arnott + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +License notice for RapidJSON +---------------------------- + +Tencent is pleased to support the open source community by making RapidJSON available. + +Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. + +Licensed under the MIT License (the "License"); you may not use this file except +in compliance with the License. You may obtain a copy of the License at + +http://opensource.org/licenses/MIT + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +License notice for DirectX Math Library +--------------------------------------- + +https://github.com/microsoft/DirectXMath/blob/master/LICENSE + + The MIT License (MIT) + +Copyright (c) 2011-2020 Microsoft Corp + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, +merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be included in all copies +or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE +OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +License notice for ldap4net +--------------------------- + +The MIT License (MIT) + +Copyright (c) 2018 Alexander Chermyanin + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +License notice for vectorized sorting code +------------------------------------------ + +MIT License + +Copyright (c) 2020 Dan Shechter + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +License notice for musl +----------------------- + +musl as a whole is licensed under the following standard MIT license: + +Copyright © 2005-2020 Rich Felker, et al. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +License notice for "Faster Unsigned Division by Constants" +------------------------------ + +Reference implementations of computing and using the "magic number" approach to dividing +by constants, including codegen instructions. The unsigned division incorporates the +"round down" optimization per ridiculous_fish. + +This is free and unencumbered software. Any copyright is dedicated to the Public Domain. + + +License notice for mimalloc +----------------------------------- + +MIT License + +Copyright (c) 2019 Microsoft Corporation, Daan Leijen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +License for remote stack unwind (https://github.com/llvm/llvm-project/blob/main/lldb/source/Symbol/CompactUnwindInfo.cpp) +-------------------------------------- + +Copyright 2019 LLVM Project + +Licensed under the Apache License, Version 2.0 (the "License") with LLVM Exceptions; +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +https://llvm.org/LICENSE.txt + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +License notice for Apple header files +------------------------------------- + +Copyright (c) 1980, 1986, 1993 + The Regents of the University of California. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. +3. All advertising materials mentioning features or use of this software + must display the following acknowledgement: + This product includes software developed by the University of + California, Berkeley and its contributors. +4. Neither the name of the University nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +License notice for JavaScript queues +------------------------------------- + +CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED HEREUNDER. + +Statement of Purpose +The laws of most jurisdictions throughout the world automatically confer exclusive Copyright and Related Rights (defined below) upon the creator and subsequent owner(s) (each and all, an "owner") of an original work of authorship and/or a database (each, a "Work"). +Certain owners wish to permanently relinquish those rights to a Work for the purpose of contributing to a commons of creative, cultural and scientific works ("Commons") that the public can reliably and without fear of later claims of infringement build upon, modify, incorporate in other works, reuse and redistribute as freely as possible in any form whatsoever and for any purposes, including without limitation commercial purposes. These owners may contribute to the Commons to promote the ideal of a free culture and the further production of creative, cultural and scientific works, or to gain reputation or greater distribution for their Work in part through the use and efforts of others. +For these and/or other purposes and motivations, and without any expectation of additional consideration or compensation, the person associating CC0 with a Work (the "Affirmer"), to the extent that he or she is an owner of Copyright and Related Rights in the Work, voluntarily elects to apply CC0 to the Work and publicly distribute the Work under its terms, with knowledge of his or her Copyright and Related Rights in the Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be protected by copyright and related or neighboring rights ("Copyright and Related Rights"). Copyright and Related Rights include, but are not limited to, the following: +the right to reproduce, adapt, distribute, perform, display, communicate, and translate a Work; +moral rights retained by the original author(s) and/or performer(s); +publicity and privacy rights pertaining to a person's image or likeness depicted in a Work; +rights protecting against unfair competition in regards to a Work, subject to the limitations in paragraph 4(a), below; +rights protecting the extraction, dissemination, use and reuse of data in a Work; +database rights (such as those arising under Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, and under any national implementation thereof, including any amended or successor version of such directive); and +other similar, equivalent or corresponding rights throughout the world based on applicable law or treaty, and any national implementations thereof. +2. Waiver. To the greatest extent permitted by, but not in contravention of, applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and unconditionally waives, abandons, and surrenders all of Affirmer's Copyright and Related Rights and associated claims and causes of action, whether now known or unknown (including existing as well as future claims and causes of action), in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each member of the public at large and to the detriment of Affirmer's heirs and successors, fully intending that such Waiver shall not be subject to revocation, rescission, cancellation, termination, or any other legal or equitable action to disrupt the quiet enjoyment of the Work by the public as contemplated by Affirmer's express Statement of Purpose. +3. Public License Fallback. Should any part of the Waiver for any reason be judged legally invalid or ineffective under applicable law, then the Waiver shall be preserved to the maximum extent permitted taking into account Affirmer's express Statement of Purpose. In addition, to the extent the Waiver is so judged Affirmer hereby grants to each affected person a royalty-free, non transferable, non sublicensable, non exclusive, irrevocable and unconditional license to exercise Affirmer's Copyright and Related Rights in the Work (i) in all territories worldwide, (ii) for the maximum duration provided by applicable law or treaty (including future time extensions), (iii) in any current or future medium and for any number of copies, and (iv) for any purpose whatsoever, including without limitation commercial, advertising or promotional purposes (the "License"). The License shall be deemed effective as of the date CC0 was applied by Affirmer to the Work. Should any part of the License for any reason be judged legally invalid or ineffective under applicable law, such partial invalidity or ineffectiveness shall not invalidate the remainder of the License, and in such case Affirmer hereby affirms that he or she will not (i) exercise any of his or her remaining Copyright and Related Rights in the Work or (ii) assert any associated claims and causes of action with respect to the Work, in either case contrary to Affirmer's express Statement of Purpose. +4. Limitations and Disclaimers. +a. No trademark or patent rights held by Affirmer are waived, abandoned, surrendered, licensed or otherwise affected by this document. +b. Affirmer offers the Work as-is and makes no representations or warranties of any kind concerning the Work, express, implied, statutory or otherwise, including without limitation warranties of title, merchantability, fitness for a particular purpose, non infringement, or the absence of latent or other defects, accuracy, or the present or absence of errors, whether or not discoverable, all to the greatest extent permissible under applicable law. +c. Affirmer disclaims responsibility for clearing rights of other persons that may apply to the Work or any use thereof, including without limitation any person's Copyright and Related Rights in the Work. Further, Affirmer disclaims responsibility for obtaining any necessary consents, permissions or other rights required for any use of the Work. +d. Affirmer understands and acknowledges that Creative Commons is not a party to this document and has no duty or obligation with respect to this CC0 or use of the Work. + + +License notice for FastFloat algorithm +------------------------------------- +MIT License +Copyright (c) 2021 csFastFloat authors +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +License notice for MsQuic +-------------------------------------- + +Copyright (c) Microsoft Corporation. +Licensed under the MIT License. + +Available at +https://github.com/microsoft/msquic/blob/main/LICENSE + +License notice for m-ou-se/floatconv +------------------------------- + +Copyright (c) 2020 Mara Bos +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License notice for code from The Practice of Programming +------------------------------- + +Copyright (C) 1999 Lucent Technologies + +Excerpted from 'The Practice of Programming +by Brian W. Kernighan and Rob Pike + +You may use this code for any purpose, as long as you leave the copyright notice and book citation attached. + +Notice for Euclidean Affine Functions and Applications to Calendar +Algorithms +------------------------------- + +Aspects of Date/Time processing based on algorithm described in "Euclidean Affine Functions and Applications to Calendar +Algorithms", Cassio Neri and Lorenz Schneider. https://arxiv.org/pdf/2102.06959.pdf + +License notice for amd/aocl-libm-ose +------------------------------- + +Copyright (C) 2008-2020 Advanced Micro Devices, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +License notice for fmtlib/fmt +------------------------------- + +Formatting library for C++ + +Copyright (c) 2012 - present, Victor Zverovich + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +License for Jb Evain +--------------------- + +Copyright (c) 2006 Jb Evain (jbevain@gmail.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +--- Optional exception to the license --- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into a machine-executable object form of such +source code, you may redistribute such embedded portions in such object form +without including the above copyright and permission notices. + + +License for MurmurHash3 +-------------------------------------- + +https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp + +MurmurHash3 was written by Austin Appleby, and is placed in the public +domain. The author hereby disclaims copyright to this source + +License for Fast CRC Computation +-------------------------------------- + +https://github.com/intel/isa-l/blob/33a2d9484595c2d6516c920ce39a694c144ddf69/crc/crc32_ieee_by4.asm +https://github.com/intel/isa-l/blob/33a2d9484595c2d6516c920ce39a694c144ddf69/crc/crc64_ecma_norm_by8.asm + +Copyright(c) 2011-2015 Intel Corporation All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +License for C# Implementation of Fast CRC Computation +----------------------------------------------------- + +https://github.com/SixLabors/ImageSharp/blob/f4f689ce67ecbcc35cebddba5aacb603e6d1068a/src/ImageSharp/Formats/Png/Zlib/Crc32.cs + +Copyright (c) Six Labors. +Licensed under the Apache License, Version 2.0. + +Available at +https://github.com/SixLabors/ImageSharp/blob/f4f689ce67ecbcc35cebddba5aacb603e6d1068a/LICENSE + +License for the Teddy multi-substring searching implementation +-------------------------------------- + +https://github.com/BurntSushi/aho-corasick + +The MIT License (MIT) + +Copyright (c) 2015 Andrew Gallant + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +License notice for Avx512Vbmi base64 encoding / decoding +-------------------------------------------------------- + +Copyright (c) 2015-2018, Wojciech Muła +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +-------------------------------------------------------- + +Aspects of base64 encoding / decoding are based on algorithm described in "Base64 encoding and decoding at almost the speed of a memory +copy", Wojciech Muła and Daniel Lemire. https://arxiv.org/pdf/1910.05109.pdf + +License for FormatJS Intl.Segmenter grapheme segmentation algorithm +-------------------------------------------------------------------------- +Available at https://github.com/formatjs/formatjs/blob/58d6a7b398d776ca3d2726d72ae1573b65cc3bef/packages/intl-segmenter/LICENSE.md + +MIT License + +Copyright (c) 2022 FormatJS + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +License for SharpFuzz and related samples +-------------------------------------- + +https://github.com/Metalnem/sharpfuzz +https://github.com/Metalnem/dotnet-fuzzers +https://github.com/Metalnem/libfuzzer-dotnet + +MIT License + +Copyright (c) 2018 Nemanja Mijailovic + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +License for National Institute of Standards and Technology ACVP Data +-------------------------------------------------------------------- +Available at https://github.com/usnistgov/ACVP-Server/blob/85f8742965b2691862079172982683757d8d91db/README.md#License + +NIST-developed software is provided by NIST as a public service. You may use, copy, and distribute copies of the software in any medium, provided that you keep intact this entire notice. You may improve, modify, and create derivative works of the software or any portion of the software, and you may copy and distribute such modifications or works. Modified works should carry a notice stating that you changed the software and should note the date and nature of any such change. Please explicitly acknowledge the National Institute of Standards and Technology as the source of the software. + +NIST-developed software is expressly provided "AS IS." NIST MAKES NO WARRANTY OF ANY KIND, EXPRESS, IMPLIED, IN FACT, OR ARISING BY OPERATION OF LAW, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND DATA ACCURACY. NIST NEITHER REPRESENTS NOR WARRANTS THAT THE OPERATION OF THE SOFTWARE WILL BE UNINTERRUPTED OR ERROR-FREE, OR THAT ANY DEFECTS WILL BE CORRECTED. NIST DOES NOT WARRANT OR MAKE ANY REPRESENTATIONS REGARDING THE USE OF THE SOFTWARE OR THE RESULTS THEREOF, INCLUDING BUT NOT LIMITED TO THE CORRECTNESS, ACCURACY, RELIABILITY, OR USEFULNESS OF THE SOFTWARE. + +You are solely responsible for determining the appropriateness of using and distributing the software and you assume all risks associated with its use, including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and the unavailability or interruption of operation. This software is not intended to be used in any situation where a failure could cause risk of injury or damage to property. The software developed by NIST employees is not subject to copyright protection within the United States. + diff --git a/licenses/SQLitePCLRaw-LICENSE.TXT b/licenses/SQLitePCLRaw-LICENSE.TXT new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/licenses/SQLitePCLRaw-LICENSE.TXT @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/licenses/Spectre.Console-LICENSE.md b/licenses/Spectre.Console-LICENSE.md new file mode 100644 index 0000000..a8373c4 --- /dev/null +++ b/licenses/Spectre.Console-LICENSE.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Patrik Svensson, Phil Scott, Nils Andresen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/licenses/YamlDotNet-LICENSE.txt b/licenses/YamlDotNet-LICENSE.txt new file mode 100644 index 0000000..d4f2924 --- /dev/null +++ b/licenses/YamlDotNet-LICENSE.txt @@ -0,0 +1,19 @@ +Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014 Antoine Aubry and contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/licenses/dotnet-LICENSE.TXT b/licenses/dotnet-LICENSE.TXT new file mode 100644 index 0000000..a616ed1 --- /dev/null +++ b/licenses/dotnet-LICENSE.TXT @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) .NET Foundation and Contributors + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/release-version.txt b/release-version.txt new file mode 100644 index 0000000..6e8bf73 --- /dev/null +++ b/release-version.txt @@ -0,0 +1 @@ +0.1.0 diff --git a/scripts/package_smoke.py b/scripts/package_smoke.py new file mode 100644 index 0000000..fcca021 --- /dev/null +++ b/scripts/package_smoke.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Install the actual nupkg into a clean local tool directory and test portable startup.""" +import argparse +import os +from pathlib import Path +import subprocess +import tempfile +import xml.etree.ElementTree as ET +import zipfile + + +def run(args, cwd, success=True, env=None): + result = subprocess.run(list(map(str, args)), cwd=cwd, capture_output=True, text=True, timeout=120, env=env) + if success and result.returncode != 0: + raise RuntimeError(result.stdout + result.stderr) + if not success and result.returncode == 0: + raise AssertionError("Command unexpectedly succeeded: " + str(args)) + return result.stdout + result.stderr + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("package", type=Path) + args = parser.parse_args() + package = args.package.resolve() + with zipfile.ZipFile(package) as archive: + names = archive.namelist() + for expected in ["README.md", "LICENSE", "icon.png", "THIRD-PARTY-NOTICES.md", + "tools/net10.0/any/config.default.yaml", "tools/net10.0/any/Helpers/pip_runner.py", + "tools/net10.0/any/MQ.DB.dll"]: + assert expected in names, f"Missing packaged asset: {expected}" + for native in ["libe_sqlite3.so", "e_sqlite3.dll", "libduckdb.so", "duckdb.dll"]: + assert any(n.endswith('/'+native) for n in names), f"Missing native asset: {native}" + assert any(n.startswith("licenses/") for n in names), "Missing third-party license texts" + root = ET.fromstring(archive.read("MagicQuant.nuspec")) + ns = {"n": root.tag.split("}")[0][1:]} + meta = root.find("n:metadata", ns) + version = meta.find("n:version", ns).text + assert meta.find("n:license", ns).text == "AGPL-3.0-only" + with tempfile.TemporaryDirectory(prefix="magicquant package smoke ") as temp: + root = Path(temp) + feed = root / "feed" + feed.mkdir() + import shutil + shutil.copy2(package, feed / package.name) + config = root / "nuget.config" + config.write_text('') + tool = root / "tool" + run(["dotnet", "tool", "install", "MagicQuant", "--tool-path", tool, "--version", version, + "--configfile", config], root, env={**os.environ, "NUGET_PACKAGES": str(root / "packages")}) + command = tool / ("magicquant.exe" if os.name == "nt" else "magicquant") + work = root / "unrelated working directory" + work.mkdir() + assert run([command, "--version"], work).strip().split("+")[0] == version + for sub in [[], ["pipeline"], ["init-config"], ["initialize-llama-cpp"], ["clone-repository-quants"]]: + run([command, *sub, "--help"], work) + assert list(work.iterdir()) == [], "Help created runtime files" + run([command, "init-config", "--output", "campaign with spaces.yaml"], work) + campaign = work / "campaign with spaces.yaml" + original = campaign.read_bytes() + assert b"scratch_roots:" in original and b"custom_repositories:" in original + run([command, "init-config", "--output", campaign], work, success=False) + assert campaign.read_bytes() == original, "init-config overwrote user settings" + # Fake input metadata is enough for read-only path checks, not conversion. + model = work / "model with spaces" + model.mkdir() + (model / "config.json").write_text('{}') + (model / "model.safetensors").touch() + import json + preflight = work / "preflight.yaml" + preflight.write_text('paths:\n model_dir: ' + json.dumps(str(model)) + '\n magic_quant_root: ' + + json.dumps(str(work / 'runtime')) + '\nidentity:\n architecture_family_name: package-test\n') + run([command, "pipeline", "--config", preflight, "--check-config", "--strict-config"], work) + assert not (work / "runtime").exists() and not (model / "MagicQuant").exists() + assert "not found" in run([command, "pipeline", "--config", "missing.yaml"], work, success=False) + print(f"Installed package {version}: assets, native libraries, help, config generation, and path preflight passed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/release_version.py b/scripts/release_version.py new file mode 100644 index 0000000..de11825 --- /dev/null +++ b/scripts/release_version.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Reserve a release version with an immutable remote Git tag. + +Tag creation is the atomic claim: concurrent releases retry collisions, while +reruns of the same commit reuse its reservation even if publication failed. +""" +import argparse +import os +from pathlib import Path +import re +import subprocess + +PATTERN = re.compile(r"v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)\Z") + + +def parse_version(value): + match = PATTERN.fullmatch("v" + value) + if not match: + raise ValueError("Release version must be a stable major.minor.patch value.") + return tuple(map(int, match.groups())) + + +def select_version(tags, current_tags, minimum): + existing = [t for t in current_tags if PATTERN.fullmatch(t)] + if len(existing) > 1: + raise ValueError("Commit has multiple release tags; resolve the ambiguity manually.") + if existing: + return existing[0][1:] + versions = [parse_version(t[1:]) for t in tags if PATTERN.fullmatch(t)] + version = parse_version(minimum) + if versions: + major, minor, patch = max(versions) + version = max(version, (major, minor, patch + 1)) + return ".".join(map(str, version)) + + +def git(*args, check=True): + return subprocess.run(["git", *args], check=check, capture_output=True, text=True) + + +def reserve(minimum): + head = git("rev-parse", "HEAD").stdout.strip() + for _ in range(20): + git("fetch", "origin", "--tags") + tags = git("tag", "--list").stdout.splitlines() + current = git("tag", "--points-at", head).stdout.splitlines() + version = select_version(tags, current, minimum) + tag = "v" + version + if tag in current: + # A local tag alone is not a successful remote reservation. + remote = git("ls-remote", "origin", "refs/tags/" + tag).stdout.strip() + if remote: + return version + else: + git("tag", "-a", tag, "-m", f"MagicQuant {version}; reserved for {head}", head) + pushed = git("push", "origin", "refs/tags/" + tag, check=False) + if pushed.returncode == 0: + return version + git("tag", "-d", tag) + # Only a genuine tag race is retryable; auth/network failures must fail. + if not git("ls-remote", "origin", "refs/tags/" + tag).stdout.strip(): + raise RuntimeError(pushed.stderr) + raise RuntimeError("Could not reserve a release version after 20 concurrent tag collisions.") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--reserve", action="store_true", help="Create and push an immutable version tag") + args = parser.parse_args() + minimum = Path("release-version.txt").read_text().strip() + parse_version(minimum) + if args.reserve: + version = reserve(minimum) + else: + version = select_version(git("tag", "--list").stdout.splitlines(), + git("tag", "--points-at", "HEAD").stdout.splitlines(), minimum) + print(version) + if os.environ.get("GITHUB_OUTPUT"): + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"version={version}\n") + + +if __name__ == "__main__": + main() diff --git a/scripts/scan_secrets.py b/scripts/scan_secrets.py new file mode 100644 index 0000000..cf12023 --- /dev/null +++ b/scripts/scan_secrets.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Run a checksum-pinned Gitleaks build on Git history and the working tree (Linux x64).""" +import hashlib +from pathlib import Path +import subprocess +import tarfile +import tempfile +import urllib.request + +VERSION = '8.30.1' +SHA256 = '551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb' + + +def main(): + with tempfile.TemporaryDirectory(prefix='mq-secret-scan-') as temp: + root = Path(temp) + archive = root / 'gitleaks.tar.gz' + url = f'https://github.com/gitleaks/gitleaks/releases/download/v{VERSION}/gitleaks_{VERSION}_linux_x64.tar.gz' + urllib.request.urlretrieve(url, archive) + if hashlib.sha256(archive.read_bytes()).hexdigest() != SHA256: + raise RuntimeError('Gitleaks checksum mismatch') + binary = root / 'gitleaks' + with tarfile.open(archive) as tar: + binary.write_bytes(tar.extractfile('gitleaks').read()) + binary.chmod(0o700) + results = [] + for mode, extra in [('git', ['--log-opts=--all']), ('dir', [])]: + results.append(subprocess.run([str(binary), mode, '.', *extra, '--redact', '--config', '.gitleaks.toml']).returncode) + if any(results): + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/scripts/test_doc_links.py b/scripts/test_doc_links.py new file mode 100644 index 0000000..be886b5 --- /dev/null +++ b/scripts/test_doc_links.py @@ -0,0 +1,32 @@ +"""Protect active documentation links; archival documents retain their historical context.""" +from pathlib import Path +import re +import unittest +from urllib.parse import unquote + +ROOT = Path(__file__).resolve().parent.parent + + +class DocumentationTests(unittest.TestCase): + def test_local_documentation_targets_exist(self): + files = [ROOT / name for name in ['README.md', 'CONTRIBUTING.md', 'THIRD-PARTY-NOTICES.md']] + files += list((ROOT / 'docs').rglob('*.md')) + list((ROOT / 'wiki').rglob('*.md')) + errors = [] + for path in files: + for match in re.finditer(r'\]\(([^)]+)\)', path.read_text(encoding='utf-8')): + url = match.group(1).split(' "')[0].strip('<>') + if url.startswith(('https:', 'http:', 'mailto:', '#')): + continue + target = unquote(url.split('#')[0]) + if target and not (path.parent / target).exists(): + errors.append(f'{path.relative_to(ROOT)}: {url}') + self.assertEqual([], errors) + + def test_current_docs_use_canonical_repository_url(self): + files = [ROOT / 'README.md', *list((ROOT / 'docs').rglob('*.md')), *list((ROOT / 'wiki').rglob('*.md'))] + for path in files: + self.assertNotIn('github.com/magiccodingman/magicquant-wiki', path.read_text(encoding='utf-8').lower(), str(path)) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/test_release_version.py b/scripts/test_release_version.py new file mode 100644 index 0000000..2c122cc --- /dev/null +++ b/scripts/test_release_version.py @@ -0,0 +1,60 @@ +import importlib.util +from pathlib import Path +import subprocess +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location("release_version", Path(__file__).with_name("release_version.py")) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class ReleaseVersionTests(unittest.TestCase): + def test_first_release(self): + self.assertEqual("0.1.0", module.select_version([], [], "0.1.0")) + + def test_patch_uses_numeric_order(self): + self.assertEqual("0.1.11", module.select_version(["v0.1.9", "v0.1.10", "research-v2"], [], "0.1.0")) + + def test_retry_reuses_original_version(self): + self.assertEqual("0.1.2", module.select_version(["v0.1.2", "v0.1.3"], ["v0.1.2"], "1.0.0")) + + def test_minor_and_major_floor(self): + for floor in ["0.2.0", "1.0.0"]: + self.assertEqual(floor, module.select_version(["v0.1.9"], [], floor)) + + def test_invalid_versions_and_ambiguous_tags_fail(self): + for value in ["1.0", "1.0.0-rc.1", "01.2.3", "1.0.0\nmalicious"]: + with self.assertRaises(ValueError): + module.parse_version(value) + with self.assertRaises(ValueError): + module.select_version([], ["v0.1.0", "v0.1.1"], "0.1.0") + + def test_real_remote_reservation_and_retry(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + def git(*args): + subprocess.run(["git", *args], cwd=root, check=True, capture_output=True) + git("init", "--bare", "remote.git") + git("clone", "remote.git", "checkout") + checkout = root / "checkout" + def local(*args): + subprocess.run(["git", *args], cwd=checkout, check=True, capture_output=True) + local("config", "user.email", "test@example.invalid") + local("config", "user.name", "Test") + (checkout / "release-version.txt").write_text("0.1.0\n") + local("add", ".") + local("commit", "-m", "first") + script = str(Path(__file__).with_name("release_version.py").resolve()) + def reserve(): + import sys + return subprocess.check_output([sys.executable, script, "--reserve"], cwd=checkout, text=True).strip() + self.assertEqual("0.1.0", reserve()) + self.assertEqual("0.1.0", reserve()) + local("commit", "--allow-empty", "-m", "second") + self.assertEqual("0.1.1", reserve()) + self.assertEqual("0.1.1", reserve()) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/MQ.DB/Cache.cs b/src/MQ.DB/Cache.cs new file mode 100644 index 0000000..a1d403a --- /dev/null +++ b/src/MQ.DB/Cache.cs @@ -0,0 +1,142 @@ +using MQ.DB.Models; + +namespace MQ.DB; + +public class Cache +{ + /// + /// Full path to llama.cpp repo + /// + public static string? LlamaRoot; + + /// + /// Full path to /llama.cpp/build/bin/ + /// + public static string? LlamaBin; + + /// + /// full path to convert_hf_to_gguf.py + /// + public static string? ConvertScript; + + /// + /// System information about the PC that's detected + /// during the initial llama cpp validation phase. + /// + public static SystemInfo? SysInfo; + + /// + /// Root MagicQuant working directory. This is also where the Python + /// environment, default config files, and shared caches live. + /// + public static string? MagicQuantDirectory; + + /// + /// Full path to the desired model directory where the safetensors are. + /// + public static string? ModelDirectory; + + /// + /// Per-model MagicQuant working directory. + /// + public static string? ModelMagicQuantDirectory; + + /// + /// Absolute path to the active YAML config that was loaded for this run. + /// + public static string? ActiveConfigPath { get; set; } + + /// + /// Root directory where external/custom baseline GGUF files are staged. + /// + public static string? ExternalBaselineCacheDirectory { get; set; } + + + /// + /// Normalized configured scratch roots for transient heavy GGUF writes. + /// + public static List ScratchRoots { get; set; } = new(); + + + /// + /// Aka BF16, F16, or F32 + /// + public static MainTorchType? TorchType; + + public enum MainTorchType + { + BF16 = 1, + F16 = 2, + F32 = 3 + } + + /* + * Groups not present in the current model graph. These are forced to NULL/ignored + * by runtime search-space planning. + */ + public static List UnusedTensorGroups = new(); + + public static string CurrentModelId { get; set; } = string.Empty; + + public static string CurrentArchitectureFamilyName { get; set; } = string.Empty; + + public static string CurrentArchitectureFamilyNormalizedName => + string.IsNullOrWhiteSpace(CurrentArchitectureFamilyName) + ? string.Empty + : CurrentArchitectureFamilyName.Trim().ToLowerInvariant(); + + public static bool AllowArchitectureFamilyAliasOverride { get; set; } + + public static int? CurrentArchitectureFamilyId { get; set; } + + public static int? CurrentTensorGroupProfileId { get; set; } + + public static string? CurrentTensorGroupProfileFingerprintHash { get; set; } + + /// + /// When true, MagicQuant prints the BF16/native tensor grouping summary and asks + /// for confirmation before any tensor-group-scoped learning/search work continues. + /// + public static bool ConfirmTensorGroupProfile { get; set; } = true; + + /// + /// Transient repair mode for regex/profile mistakes. When true, MagicQuant tries + /// to rebuild learned tensor mappings for the active TensorGroupProfile from + /// existing family/profile truth instead of redownloading/requantizing pure + /// learning baselines just to rediscover per-tensor truth. + /// + public static bool RebucketLearnedTensorGroupsFromExistingTruth { get; set; } = true; + + public static bool ForceRefreshHardwareProbe { get; set; } + + public static bool UseImatrix { get; set; } + + public static bool ForceImatrixRebuild { get; set; } + + public static Dictionary GpuMemoryLimitsGb { get; set; } = new(); + + public static bool IsImatrixAvailable { get; set; } + + public static string? ActiveImatrixPath { get; set; } + + public static string? ActiveImatrixIdentityHash { get; set; } + + + /// + /// When false, long-running llama.cpp child processes write their full stdout/stderr + /// to log files only. This keeps the CLI readable during large export/clone runs. + /// + public static bool VerboseProcessOutput { get; set; } + + /// + /// Clone/export-only flows may benchmark for release metadata without polluting the + /// learning/discovery SQLite truth tables. + /// + public static bool SuppressBenchmarkPersistence { get; set; } + + + /// + /// Final export/output directory for selected survivor artifacts. + /// + public static string? OutputDirectory { get; set; } +} diff --git a/src/MQ.DB/Data/MagicQuantContext.cs b/src/MQ.DB/Data/MagicQuantContext.cs new file mode 100644 index 0000000..22145bc --- /dev/null +++ b/src/MQ.DB/Data/MagicQuantContext.cs @@ -0,0 +1,389 @@ +using Microsoft.EntityFrameworkCore; +using MQ.DB.Interfaces; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; + +namespace MQ.DB.Data; + +public class MagicQuantContext : DbContext +{ + // -------------------------------------------------------- + // Self-Initialization Logic + // -------------------------------------------------------- + private static readonly HashSet InitializedDatabaseDirectories = new(StringComparer.Ordinal); + private static readonly object _initLock = new(); + + public MagicQuantContext() + { + EnsureInitialized(); + } + + public MagicQuantContext(DbContextOptions options) + : base(options) + { + EnsureInitialized(); + } + + private void EnsureInitialized() + { + // 🚫 Never run during EF tooling (migrations, etc.) + if (IsDesignTime()) + return; + + string initializationKey = ResolveDatabaseDirectory(); + lock (_initLock) + { + if (InitializedDatabaseDirectories.Contains(initializationKey)) + return; + + InitializeDatabase(); + InitializedDatabaseDirectories.Add(initializationKey); + } + } + + private static string ResolveDatabaseDirectory() + { + string directory = string.IsNullOrWhiteSpace(Cache.MagicQuantDirectory) + ? Directory.GetCurrentDirectory() + : Cache.MagicQuantDirectory; + return Path.GetFullPath(directory); + } + + private void InitializeDatabase() + { + var directory = Cache.MagicQuantDirectory; + + // Safety fallback + if (string.IsNullOrEmpty(directory)) + directory = Directory.GetCurrentDirectory(); + + if (!Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + + // 🔥 Apply migrations automatically + Database.Migrate(); + EnsureBaselineQuantDefinitions(); + } + + +private void EnsureBaselineQuantDefinitions() +{ + var now = DateTime.UtcNow; + var expected = BaselineQuants.GetBuiltInStandardBaselines() + .Concat(BaselineQuants.GetExactHighPrecisionAliases(allowHighPrecisionHybrids: true)) + .Append(BaselineQuants.GetNativeQuant()) + .Select(x => new BaselineQuantDefinition + { + ArchitectureFamilyId = null, + RuntimeBaselineId = x.UniqueId, + CanonicalKey = x.CanonicalKey, + NormalizedCanonicalKey = NormalizeKey(x.CanonicalKey), + BaselineName = x.Names[0], + DisplayName = x.Names[0], + QuantizeBaseArgumentName = x.QuantizeBaseArgumentName, + DefaultTensorSchemeId = x.DefaultTensorScheme!.UniqueId, + DefaultTensorSchemeName = x.DefaultTensorScheme.Names[0], + SourceKind = x.SourceKind, + SourceOwner = x.SourceOwner, + SourceRepository = x.SourceRepository, + NormalizedSourceRepository = NormalizeNullable(x.SourceRepository), + SourceFileName = x.SourceFileName, + NormalizedSourceFileName = NormalizeFileNullable(x.SourceFileName), + ShortSourceName = x.ShortSourceName, + BaselineFamily = x.Names[0], + IsCustomBaseline = x.IsCustomBaseline, + IsLearningBaseline = x.IsLearningBaseline, + IsCombinationCarrierCandidate = x.IsCombinationCarrierCandidate, + IsExplicitGroupCombinationCandidate = x.IsExplicitGroupCombinationCandidate, + RequiresImatrix = x.RequiresImatrix, + BitRange = x.BitRange, + ExplicitCandidateSortOrder = x.ExplicitCandidateSortOrder, + IsActiveInCurrentConfig = true, + FirstSeenUtc = now, + LastSeenUtc = now, + LastUpdatedUtc = now + }) + .OrderBy(x => x.RuntimeBaselineId) + .ToList(); + + var current = BaselineQuantDefinitions + .Where(x => x.ArchitectureFamilyId == null) + .ToList(); + + var currentByCanonicalKey = current + .Where(x => !string.IsNullOrWhiteSpace(x.NormalizedCanonicalKey)) + .GroupBy(x => x.NormalizedCanonicalKey, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.OrderBy(x => x.RuntimeBaselineId).First(), StringComparer.Ordinal); + + var currentByRuntimeId = current.ToDictionary(x => x.RuntimeBaselineId); + var changed = false; + + foreach (var expectedRow in expected) + { + BaselineQuantDefinition? target = null; + + if (!string.IsNullOrWhiteSpace(expectedRow.NormalizedCanonicalKey) && + currentByCanonicalKey.TryGetValue(expectedRow.NormalizedCanonicalKey, out var byCanonicalKey)) + { + target = byCanonicalKey; + } + else if (currentByRuntimeId.TryGetValue(expectedRow.RuntimeBaselineId, out var byId)) + { + target = byId; + } + + if (target == null) + { + BaselineQuantDefinitions.Add(expectedRow); + changed = true; + continue; + } + + if (!BaselineDefinitionEquals(target, expectedRow)) + { + ApplyBaselineDefinitionUpdate(target, expectedRow, preserveFirstSeen: true); + target.LastUpdatedUtc = now; + changed = true; + } + + target.IsActiveInCurrentConfig = true; + target.LastSeenUtc = now; + } + + if (changed) + SaveChanges(); +} + +private static bool BaselineDefinitionEquals(BaselineQuantDefinition a, BaselineQuantDefinition b) +{ + return a.ArchitectureFamilyId == b.ArchitectureFamilyId && + a.RuntimeBaselineId == b.RuntimeBaselineId && + a.DefaultTensorSchemeId == b.DefaultTensorSchemeId && + a.IsCustomBaseline == b.IsCustomBaseline && + a.IsLearningBaseline == b.IsLearningBaseline && + a.IsCombinationCarrierCandidate == b.IsCombinationCarrierCandidate && + a.IsExplicitGroupCombinationCandidate == b.IsExplicitGroupCombinationCandidate && + a.RequiresImatrix == b.RequiresImatrix && + a.BitRange == b.BitRange && + a.ExplicitCandidateSortOrder == b.ExplicitCandidateSortOrder && + a.IsActiveInCurrentConfig == b.IsActiveInCurrentConfig && + string.Equals(a.CanonicalKey, b.CanonicalKey, StringComparison.Ordinal) && + string.Equals(a.NormalizedCanonicalKey, b.NormalizedCanonicalKey, StringComparison.Ordinal) && + string.Equals(a.BaselineName, b.BaselineName, StringComparison.Ordinal) && + string.Equals(a.DisplayName, b.DisplayName, StringComparison.Ordinal) && + string.Equals(a.QuantizeBaseArgumentName, b.QuantizeBaseArgumentName, StringComparison.Ordinal) && + string.Equals(a.DefaultTensorSchemeName, b.DefaultTensorSchemeName, StringComparison.Ordinal) && + string.Equals(a.SourceKind, b.SourceKind, StringComparison.Ordinal) && + string.Equals(a.SourceOwner, b.SourceOwner, StringComparison.Ordinal) && + string.Equals(a.SourceRepository, b.SourceRepository, StringComparison.Ordinal) && + string.Equals(a.NormalizedSourceRepository, b.NormalizedSourceRepository, StringComparison.Ordinal) && + string.Equals(a.SourceFileName, b.SourceFileName, StringComparison.Ordinal) && + string.Equals(a.NormalizedSourceFileName, b.NormalizedSourceFileName, StringComparison.Ordinal) && + string.Equals(a.ShortSourceName, b.ShortSourceName, StringComparison.Ordinal) && + string.Equals(a.BaselineFamily, b.BaselineFamily, StringComparison.Ordinal); +} + +public static void ApplyBaselineDefinitionUpdate(BaselineQuantDefinition target, BaselineQuantDefinition source, bool preserveFirstSeen = true) +{ + var firstSeen = target.FirstSeenUtc; + target.ArchitectureFamilyId = source.ArchitectureFamilyId; + target.RuntimeBaselineId = source.RuntimeBaselineId; + target.CanonicalKey = source.CanonicalKey; + target.NormalizedCanonicalKey = source.NormalizedCanonicalKey; + target.BaselineName = source.BaselineName; + target.DisplayName = source.DisplayName; + target.QuantizeBaseArgumentName = source.QuantizeBaseArgumentName; + target.DefaultTensorSchemeId = source.DefaultTensorSchemeId; + target.DefaultTensorSchemeName = source.DefaultTensorSchemeName; + target.SourceKind = source.SourceKind; + target.SourceOwner = source.SourceOwner; + target.SourceRepository = source.SourceRepository; + target.NormalizedSourceRepository = source.NormalizedSourceRepository; + target.SourceFileName = source.SourceFileName; + target.NormalizedSourceFileName = source.NormalizedSourceFileName; + target.ShortSourceName = source.ShortSourceName; + target.BaselineFamily = source.BaselineFamily; + target.IsCustomBaseline = source.IsCustomBaseline; + target.IsLearningBaseline = source.IsLearningBaseline; + target.IsCombinationCarrierCandidate = source.IsCombinationCarrierCandidate; + target.IsExplicitGroupCombinationCandidate = source.IsExplicitGroupCombinationCandidate; + target.RequiresImatrix = source.RequiresImatrix; + target.BitRange = source.BitRange; + target.ExplicitCandidateSortOrder = source.ExplicitCandidateSortOrder; + target.IsActiveInCurrentConfig = source.IsActiveInCurrentConfig; + target.LastSeenUtc = source.LastSeenUtc; + target.LastUpdatedUtc = source.LastUpdatedUtc; + if (!preserveFirstSeen) + target.FirstSeenUtc = source.FirstSeenUtc; + else if (firstSeen != default) + target.FirstSeenUtc = firstSeen; +} + +private static string NormalizeKey(string value) => (value ?? string.Empty).Trim().ToLowerInvariant(); +private static string? NormalizeNullable(string? value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim().ToLowerInvariant(); +private static string? NormalizeFileNullable(string? value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim().Replace('\\', '/').ToLowerInvariant(); + + private static bool IsDesignTime() + { + return AppDomain.CurrentDomain.GetAssemblies() + .Any(a => a.FullName != null && + a.FullName.Contains("EntityFrameworkCore.Design", StringComparison.OrdinalIgnoreCase)); + } + + // -------------------------------------------------------- + // DbSets + // -------------------------------------------------------- + + public DbSet AiBenchmarks { get; set; } + public DbSet AiModelHashes { get; set; } + public DbSet TensorCombos { get; set; } + public DbSet QuantizationRuns { get; set; } + public DbSet BenchmarkRuns { get; set; } + public DbSet LearnedBaselineTensorQuants { get; set; } + public DbSet BaselineQuantDefinitions { get; set; } + public DbSet TensorGroupProfiles { get; set; } + public DbSet AiBenchmarkLearnedSources { get; set; } + public DbSet ExecutionPlanProbeCaches { get; set; } + public DbSet ImatrixDefinitions { get; set; } + public DbSet ArchitectureFamilies { get; set; } + public DbSet ArchitectureFamilyModelHashes { get; set; } + public DbSet AnomalyProbeSessions { get; set; } + public DbSet AnomalyProbeObservations { get; set; } + public DbSet AnomalyInteractionRules { get; set; } + public DbSet AnomalyInteractionRuleGroupStates { get; set; } + + + // -------------------------------------------------------- + // Imatrix Ownership Guard + // -------------------------------------------------------- + + public override int SaveChanges() + { + ValidateImatrixOwnershipBeforeSaveAsync(CancellationToken.None).GetAwaiter().GetResult(); + return base.SaveChanges(); + } + + public override int SaveChanges(bool acceptAllChangesOnSuccess) + { + ValidateImatrixOwnershipBeforeSaveAsync(CancellationToken.None).GetAwaiter().GetResult(); + return base.SaveChanges(acceptAllChangesOnSuccess); + } + + public override async Task SaveChangesAsync(CancellationToken cancellationToken = default) + { + await ValidateImatrixOwnershipBeforeSaveAsync(cancellationToken); + return await base.SaveChangesAsync(cancellationToken); + } + + public override async Task SaveChangesAsync(bool acceptAllChangesOnSuccess, CancellationToken cancellationToken = default) + { + await ValidateImatrixOwnershipBeforeSaveAsync(cancellationToken); + return await base.SaveChangesAsync(acceptAllChangesOnSuccess, cancellationToken); + } + + private async Task ValidateImatrixOwnershipBeforeSaveAsync(CancellationToken ct) + { + var pairs = ChangeTracker.Entries() + .Where(e => e.State is EntityState.Added or EntityState.Modified) + .Select(e => e.Entity) + .Select(entity => entity switch + { + AiBenchmark x => (EntityName: nameof(AiBenchmark), x.AiModelHashId, x.ImatrixDefinitionId), + BenchmarkRun x => (EntityName: nameof(BenchmarkRun), x.AiModelHashId, x.ImatrixDefinitionId), + QuantizationRun x => (EntityName: nameof(QuantizationRun), x.AiModelHashId, x.ImatrixDefinitionId), + ExecutionPlanProbeCache x => (EntityName: nameof(ExecutionPlanProbeCache), x.AiModelHashId, x.ImatrixDefinitionId), + AnomalyProbeSession x => (EntityName: nameof(AnomalyProbeSession), x.AiModelHashId, x.ImatrixDefinitionId), + AnomalyProbeObservation x => (EntityName: nameof(AnomalyProbeObservation), x.AiModelHashId, x.ImatrixDefinitionId), + AnomalyInteractionRule x => (EntityName: nameof(AnomalyInteractionRule), x.AiModelHashId, x.ImatrixDefinitionId), + _ => default + }) + .Where(x => !string.IsNullOrWhiteSpace(x.EntityName) && x.ImatrixDefinitionId.HasValue) + .Distinct() + .ToList(); + + if (pairs.Count == 0) + return; + + var ids = pairs + .Select(x => x.ImatrixDefinitionId!.Value) + .Distinct() + .ToList(); + + var owners = await ImatrixDefinitions + .AsNoTracking() + .Where(x => ids.Contains(x.Id)) + .Select(x => new { x.Id, x.AiModelHashId }) + .ToDictionaryAsync(x => x.Id, x => x.AiModelHashId, ct); + + foreach (var pair in pairs) + { + if (!owners.TryGetValue(pair.ImatrixDefinitionId!.Value, out var ownerHashId) || + ownerHashId != pair.AiModelHashId) + { + throw new InvalidOperationException( + $"{pair.EntityName} attempted to save AiModelHashId={pair.AiModelHashId} with " + + $"ImatrixDefinitionId={pair.ImatrixDefinitionId.Value}, but that imatrix belongs to " + + $"AiModelHashId={(owners.TryGetValue(pair.ImatrixDefinitionId.Value, out var found) ? found.ToString() : "missing")}. " + + "ImatrixDefinition ownership is exact-model-hash scoped."); + } + } + } + + // -------------------------------------------------------- + // Configuration + // -------------------------------------------------------- + + protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) + { + if (!optionsBuilder.IsConfigured) + { + var directory = Cache.MagicQuantDirectory; + + if (string.IsNullOrEmpty(directory)) + directory = Directory.GetCurrentDirectory(); + + var dbPath = Path.Combine(directory, "MagicQuant_SQLite.db"); + + optionsBuilder.UseSqlite($"Data Source={dbPath};Foreign Keys=True;"); + } + } + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.ApplyConfigurationsFromAssembly(typeof(MagicQuantContext).Assembly); + ValidateDbSetsImplementInterface(); + base.OnModelCreating(modelBuilder); + } + + // -------------------------------------------------------- + // Strict Validation + // -------------------------------------------------------- + + private void ValidateDbSetsImplementInterface() + { + var dbSetGenericTypes = this.GetType() + .GetProperties() + .Where(p => p.PropertyType.IsGenericType && + p.PropertyType.GetGenericTypeDefinition() == typeof(DbSet<>)) + .Select(p => p.PropertyType.GetGenericArguments()[0]) + .ToHashSet(); + + var configuredTypes = typeof(MagicQuantContext).Assembly + .GetTypes() + .Where(t => t.GetInterfaces().Any(i => + i.IsGenericType && i.GetGenericTypeDefinition() == typeof(ISQLiteEntity<>))) + .ToHashSet(); + + var missingConfigs = dbSetGenericTypes.Except(configuredTypes).ToList(); + + if (missingConfigs.Any()) + { + var names = string.Join(", ", missingConfigs.Select(t => t.Name)); + throw new InvalidOperationException( + $"STRICT MODE ERROR: The following DbSets do not implement ISQLiteEntity: [{names}]. " + ); + } + } +} diff --git a/src/MQ.DB/Interfaces/ISQLiteEntity.cs b/src/MQ.DB/Interfaces/ISQLiteEntity.cs new file mode 100644 index 0000000..3e6cba9 --- /dev/null +++ b/src/MQ.DB/Interfaces/ISQLiteEntity.cs @@ -0,0 +1,10 @@ +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace MQ.DB.Interfaces; +using Microsoft.EntityFrameworkCore; + +internal interface ISQLiteEntity : IEntityTypeConfiguration + where T : class +{ + +} \ No newline at end of file diff --git a/src/MQ.DB/MQ.DB.csproj b/src/MQ.DB/MQ.DB.csproj new file mode 100644 index 0000000..dbbc1aa --- /dev/null +++ b/src/MQ.DB/MQ.DB.csproj @@ -0,0 +1,31 @@ + + + + net10.0 + enable + enable + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + Always + + + + diff --git a/src/MQ.DB/Migrations/20260501195554_InitialCreate.Designer.cs b/src/MQ.DB/Migrations/20260501195554_InitialCreate.Designer.cs new file mode 100644 index 0000000..4e8e213 --- /dev/null +++ b/src/MQ.DB/Migrations/20260501195554_InitialCreate.Designer.cs @@ -0,0 +1,1151 @@ +// +using System; +using MQ.DB.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace MQ.DB.Migrations +{ + [DbContext(typeof(MagicQuantContext))] + [Migration("20260501195554_InitialCreate")] + partial class InitialCreate + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "10.0.1"); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmark", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("Ngl") + .HasColumnType("INTEGER"); + + b.Property("SizeBytes") + .HasColumnType("INTEGER"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("TokensPerSecond") + .HasColumnType("REAL"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "TensorComboId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "TensorComboId") + .IsUnique(); + + b.ToTable("AiBenchmarks"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmarkLearnedSource", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BaselineCanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("BaselineQuantDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("SourceLearningBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupId") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("BaselineQuantDefinitionId"); + + b.HasIndex("SourceLearningBenchmarkId"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("AiBenchmarkId", "TensorGroupId") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "BaselineQuantDefinitionId"); + + b.ToTable("AiBenchmarkLearnedSources"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiModelHash", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("UniqueHash") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("UniqueHash"); + + b.ToTable("AiModelHashes"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ArchitectureFamily", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("NormalizedName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("TensorCount") + .HasColumnType("INTEGER"); + + b.Property("TensorSignatureHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique(); + + b.HasIndex("TensorSignatureHash", "TensorCount"); + + b.ToTable("ArchitectureFamilies"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ArchitectureFamilyModelHash", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("IsCanonical") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "AiModelHashId") + .IsUnique(); + + b.ToTable("ArchitectureFamilyModelHashes"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BaselineQuantDefinition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BaselineFamily") + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("BaselineName") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("BitRange") + .HasColumnType("INTEGER"); + + b.Property("CanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("DefaultTensorSchemeId") + .HasColumnType("INTEGER"); + + b.Property("DefaultTensorSchemeName") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ExplicitCandidateSortOrder") + .HasColumnType("INTEGER"); + + b.Property("FirstSeenUtc") + .HasColumnType("TEXT"); + + b.Property("IsActiveInCurrentConfig") + .HasColumnType("INTEGER"); + + b.Property("IsCombinationCarrierCandidate") + .HasColumnType("INTEGER"); + + b.Property("IsCustomBaseline") + .HasColumnType("INTEGER"); + + b.Property("IsExplicitGroupCombinationCandidate") + .HasColumnType("INTEGER"); + + b.Property("IsLearningBaseline") + .HasColumnType("INTEGER"); + + b.Property("LastSeenUtc") + .HasColumnType("TEXT"); + + b.Property("LastUpdatedUtc") + .HasColumnType("TEXT"); + + b.Property("NormalizedCanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("NormalizedSourceFileName") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("NormalizedSourceRepository") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("QuantizeBaseArgumentName") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("RequiresImatrix") + .HasColumnType("INTEGER"); + + b.Property("RuntimeBaselineId") + .HasColumnType("INTEGER"); + + b.Property("ShortSourceName") + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("SourceFileName") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("SourceKind") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("SourceOwner") + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("SourceRepository") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("IsActiveInCurrentConfig"); + + b.HasIndex("ArchitectureFamilyId", "NormalizedCanonicalKey") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "RuntimeBaselineId") + .IsUnique(); + + b.HasIndex("RuntimeBaselineId", "ArchitectureFamilyId"); + + b.HasIndex("ArchitectureFamilyId", "NormalizedSourceRepository", "NormalizedSourceFileName") + .IsUnique(); + + b.ToTable("BaselineQuantDefinitions"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BenchmarkRun", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("Category") + .HasColumnType("INTEGER"); + + b.Property("CategoryBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("CompletedUtc") + .HasColumnType("TEXT"); + + b.Property("DurationMs") + .HasColumnType("INTEGER"); + + b.Property("Error") + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("StartedUtc") + .HasColumnType("TEXT"); + + b.Property("Succeeded") + .HasColumnType("INTEGER"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ArchitectureFamilyId"); + + b.HasIndex("CategoryBenchmarkId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("StartedUtc"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("AiBenchmarkId", "Category"); + + b.ToTable("BenchmarkRuns"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.CategoryBenchmark", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("Category") + .HasColumnType("INTEGER"); + + b.Property("Kld") + .HasColumnType("REAL"); + + b.Property("Ppl") + .HasColumnType("REAL"); + + b.Property("PplError") + .HasColumnType("REAL"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.ToTable("CategoryBenchmark"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ExecutionPlanProbeCache", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("DiscoveryTokenTarget") + .HasColumnType("INTEGER"); + + b.Property("GpuMemoryLimitsJson") + .IsRequired() + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("GroupSize") + .HasColumnType("INTEGER"); + + b.Property("HardwareFingerprint") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("MaxCandidateNgl") + .HasColumnType("INTEGER"); + + b.Property("NativeModelSizeBytes") + .HasColumnType("INTEGER"); + + b.Property("NativeQuantizationKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("NativeStableNgl") + .HasColumnType("INTEGER"); + + b.Property("ProbeSchemaVersion") + .HasColumnType("INTEGER"); + + b.Property("Q8ModelSizeBytes") + .HasColumnType("INTEGER"); + + b.Property("Q8StableNgl") + .HasColumnType("INTEGER"); + + b.Property("QuantizationKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("QuantizedModelFingerprint") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("TEXT"); + + b.Property("SlotsJson") + .IsRequired() + .HasMaxLength(8000) + .HasColumnType("TEXT"); + + b.Property("StaticNgl") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("TensorSplitJson") + .IsRequired() + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("UpdatedUtc") + .HasColumnType("TEXT"); + + b.Property("UsesGpu") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ArchitectureFamilyId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "HardwareFingerprint", "QuantizedModelFingerprint", "QuantizationKey", "DiscoveryTokenTarget") + .IsUnique(); + + b.ToTable("ExecutionPlanProbeCaches"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ImatrixDefinition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("BuildFingerprint") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("CanonicalPath") + .HasMaxLength(2048) + .HasColumnType("TEXT"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("IdentityHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("MetadataJson") + .HasMaxLength(8000) + .HasColumnType("TEXT"); + + b.Property("SourceKind") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("TokenCount") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId", "IdentityHash") + .IsUnique(); + + b.ToTable("ImatrixDefinitions"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.LearnedBaselineTensorQuant", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BaselineCanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("BaselineQuantDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("BaselineQuantId") + .HasColumnType("INTEGER"); + + b.Property("BaselineSourceFileName") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("BaselineSourceKind") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("BaselineSourceRepository") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("FinalQuantType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("TEXT"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupId") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("TensorName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("TensorWeightSchemeId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("BaselineQuantDefinitionId"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "BaselineQuantDefinitionId", "TensorGroupId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "BaselineQuantDefinitionId", "TensorWeightSchemeId", "TensorName") + .IsUnique(); + + b.ToTable("LearnedBaselineTensorQuants"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.QuantizationRun", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CompletedUtc") + .HasColumnType("TEXT"); + + b.Property("DurationMs") + .HasColumnType("INTEGER"); + + b.Property("Error") + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("OutputModelPath") + .HasMaxLength(2048) + .HasColumnType("TEXT"); + + b.Property("StartedUtc") + .HasColumnType("TEXT"); + + b.Property("Succeeded") + .HasColumnType("INTEGER"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ArchitectureFamilyId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("StartedUtc"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.ToTable("QuantizationRuns"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.TensorCombo", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AttnKV") + .HasColumnType("INTEGER"); + + b.Property("AttnOutput") + .HasColumnType("INTEGER"); + + b.Property("AttnQ") + .HasColumnType("INTEGER"); + + b.Property("BaseQuant") + .HasColumnType("INTEGER"); + + b.Property("Embeddings") + .HasColumnType("INTEGER"); + + b.Property("FfnDown") + .HasColumnType("INTEGER"); + + b.Property("FfnUpGate") + .HasColumnType("INTEGER"); + + b.Property("LmHead") + .HasColumnType("INTEGER"); + + b.Property("MoeExperts") + .HasColumnType("INTEGER"); + + b.Property("MoeRouter") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("BaseQuant", "Embeddings", "LmHead", "AttnQ", "AttnKV", "AttnOutput", "FfnUpGate", "FfnDown", "MoeExperts", "MoeRouter") + .IsUnique(); + + b.ToTable("TensorCombos"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.TensorGroupProfile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("FingerprintHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("IsActive") + .HasColumnType("INTEGER"); + + b.Property("SnapshotJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("ArchitectureFamilyId", "FingerprintHash") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "IsActive"); + + b.ToTable("TensorGroupProfiles"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmark", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmarkLearnedSource", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany("LearnedSources") + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.BaselineQuantDefinition", "BaselineQuantDefinition") + .WithMany() + .HasForeignKey("BaselineQuantDefinitionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "SourceLearningBenchmark") + .WithMany() + .HasForeignKey("SourceLearningBenchmarkId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("BaselineQuantDefinition"); + + b.Navigation("SourceLearningBenchmark"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ArchitectureFamilyModelHash", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BaselineQuantDefinition", b => + { + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("ArchitectureFamily"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BenchmarkRun", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany() + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.CategoryBenchmark", "CategoryBenchmark") + .WithMany() + .HasForeignKey("CategoryBenchmarkId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("CategoryBenchmark"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.CategoryBenchmark", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany("CategorBenchmarks") + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiBenchmark"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ExecutionPlanProbeCache", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ImatrixDefinition", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiModelHash"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.LearnedBaselineTensorQuant", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany() + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.BaselineQuantDefinition", "BaselineQuantDefinition") + .WithMany() + .HasForeignKey("BaselineQuantDefinitionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("BaselineQuantDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.QuantizationRun", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany() + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.TensorGroupProfile", b => + { + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ArchitectureFamily"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmark", b => + { + b.Navigation("CategorBenchmarks"); + + b.Navigation("LearnedSources"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/MQ.DB/Migrations/20260501195554_InitialCreate.cs b/src/MQ.DB/Migrations/20260501195554_InitialCreate.cs new file mode 100644 index 0000000..d03c1b5 --- /dev/null +++ b/src/MQ.DB/Migrations/20260501195554_InitialCreate.cs @@ -0,0 +1,895 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace MQ.DB.Migrations +{ + /// + public partial class InitialCreate : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "AiModelHashes", + columns: table => new + { + Id = table.Column(type: "INTEGER", nullable: false) + .Annotation("Sqlite:Autoincrement", true), + UniqueHash = table.Column(type: "TEXT", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_AiModelHashes", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "ArchitectureFamilies", + columns: table => new + { + Id = table.Column(type: "INTEGER", nullable: false) + .Annotation("Sqlite:Autoincrement", true), + NormalizedName = table.Column(type: "TEXT", maxLength: 256, nullable: false), + DisplayName = table.Column(type: "TEXT", maxLength: 256, nullable: false), + TensorSignatureHash = table.Column(type: "TEXT", maxLength: 128, nullable: false), + TensorCount = table.Column(type: "INTEGER", nullable: false), + CreatedUtc = table.Column(type: "TEXT", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_ArchitectureFamilies", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "TensorCombos", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + AttnKV = table.Column(type: "INTEGER", nullable: false), + AttnOutput = table.Column(type: "INTEGER", nullable: false), + AttnQ = table.Column(type: "INTEGER", nullable: false), + BaseQuant = table.Column(type: "INTEGER", nullable: false), + Embeddings = table.Column(type: "INTEGER", nullable: false), + FfnDown = table.Column(type: "INTEGER", nullable: false), + FfnUpGate = table.Column(type: "INTEGER", nullable: false), + LmHead = table.Column(type: "INTEGER", nullable: false), + MoeExperts = table.Column(type: "INTEGER", nullable: false), + MoeRouter = table.Column(type: "INTEGER", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_TensorCombos", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "ImatrixDefinitions", + columns: table => new + { + Id = table.Column(type: "INTEGER", nullable: false) + .Annotation("Sqlite:Autoincrement", true), + AiModelHashId = table.Column(type: "INTEGER", nullable: false), + IdentityHash = table.Column(type: "TEXT", maxLength: 128, nullable: false), + CanonicalPath = table.Column(type: "TEXT", maxLength: 2048, nullable: true), + SourceKind = table.Column(type: "TEXT", maxLength: 64, nullable: false), + CreatedUtc = table.Column(type: "TEXT", nullable: false), + MetadataJson = table.Column(type: "TEXT", maxLength: 8000, nullable: true), + TokenCount = table.Column(type: "INTEGER", nullable: true), + BuildFingerprint = table.Column(type: "TEXT", maxLength: 512, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_ImatrixDefinitions", x => x.Id); + table.ForeignKey( + name: "FK_ImatrixDefinitions_AiModelHashes_AiModelHashId", + column: x => x.AiModelHashId, + principalTable: "AiModelHashes", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "ArchitectureFamilyModelHashes", + columns: table => new + { + Id = table.Column(type: "INTEGER", nullable: false) + .Annotation("Sqlite:Autoincrement", true), + ArchitectureFamilyId = table.Column(type: "INTEGER", nullable: false), + AiModelHashId = table.Column(type: "INTEGER", nullable: false), + IsCanonical = table.Column(type: "INTEGER", nullable: false), + CreatedUtc = table.Column(type: "TEXT", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_ArchitectureFamilyModelHashes", x => x.Id); + table.ForeignKey( + name: "FK_ArchitectureFamilyModelHashes_AiModelHashes_AiModelHashId", + column: x => x.AiModelHashId, + principalTable: "AiModelHashes", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_ArchitectureFamilyModelHashes_ArchitectureFamilies_ArchitectureFamilyId", + column: x => x.ArchitectureFamilyId, + principalTable: "ArchitectureFamilies", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "BaselineQuantDefinitions", + columns: table => new + { + Id = table.Column(type: "INTEGER", nullable: false) + .Annotation("Sqlite:Autoincrement", true), + ArchitectureFamilyId = table.Column(type: "INTEGER", nullable: true), + RuntimeBaselineId = table.Column(type: "INTEGER", nullable: false), + CanonicalKey = table.Column(type: "TEXT", maxLength: 512, nullable: false), + NormalizedCanonicalKey = table.Column(type: "TEXT", maxLength: 512, nullable: false), + BaselineName = table.Column(type: "TEXT", maxLength: 128, nullable: false), + DisplayName = table.Column(type: "TEXT", maxLength: 128, nullable: false), + QuantizeBaseArgumentName = table.Column(type: "TEXT", maxLength: 64, nullable: false), + DefaultTensorSchemeId = table.Column(type: "INTEGER", nullable: false), + DefaultTensorSchemeName = table.Column(type: "TEXT", maxLength: 64, nullable: false), + SourceKind = table.Column(type: "TEXT", maxLength: 64, nullable: false), + SourceOwner = table.Column(type: "TEXT", maxLength: 128, nullable: true), + SourceRepository = table.Column(type: "TEXT", maxLength: 256, nullable: true), + NormalizedSourceRepository = table.Column(type: "TEXT", maxLength: 256, nullable: true), + SourceFileName = table.Column(type: "TEXT", maxLength: 512, nullable: true), + NormalizedSourceFileName = table.Column(type: "TEXT", maxLength: 512, nullable: true), + ShortSourceName = table.Column(type: "TEXT", maxLength: 64, nullable: true), + BaselineFamily = table.Column(type: "TEXT", maxLength: 128, nullable: true), + IsCustomBaseline = table.Column(type: "INTEGER", nullable: false), + IsLearningBaseline = table.Column(type: "INTEGER", nullable: false), + IsCombinationCarrierCandidate = table.Column(type: "INTEGER", nullable: false), + IsExplicitGroupCombinationCandidate = table.Column(type: "INTEGER", nullable: false), + RequiresImatrix = table.Column(type: "INTEGER", nullable: false), + BitRange = table.Column(type: "INTEGER", nullable: false), + ExplicitCandidateSortOrder = table.Column(type: "INTEGER", nullable: false), + IsActiveInCurrentConfig = table.Column(type: "INTEGER", nullable: false), + FirstSeenUtc = table.Column(type: "TEXT", nullable: false), + LastSeenUtc = table.Column(type: "TEXT", nullable: false), + LastUpdatedUtc = table.Column(type: "TEXT", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_BaselineQuantDefinitions", x => x.Id); + table.ForeignKey( + name: "FK_BaselineQuantDefinitions_ArchitectureFamilies_ArchitectureFamilyId", + column: x => x.ArchitectureFamilyId, + principalTable: "ArchitectureFamilies", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "TensorGroupProfiles", + columns: table => new + { + Id = table.Column(type: "INTEGER", nullable: false) + .Annotation("Sqlite:Autoincrement", true), + ArchitectureFamilyId = table.Column(type: "INTEGER", nullable: false), + FingerprintHash = table.Column(type: "TEXT", maxLength: 128, nullable: false), + SnapshotJson = table.Column(type: "TEXT", nullable: false), + CreatedUtc = table.Column(type: "TEXT", nullable: false), + IsActive = table.Column(type: "INTEGER", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_TensorGroupProfiles", x => x.Id); + table.ForeignKey( + name: "FK_TensorGroupProfiles_ArchitectureFamilies_ArchitectureFamilyId", + column: x => x.ArchitectureFamilyId, + principalTable: "ArchitectureFamilies", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "AiBenchmarks", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + Ngl = table.Column(type: "INTEGER", nullable: false), + SizeBytes = table.Column(type: "INTEGER", nullable: false), + TokensPerSecond = table.Column(type: "REAL", nullable: false), + ArchitectureFamilyId = table.Column(type: "INTEGER", nullable: false), + TensorGroupProfileId = table.Column(type: "INTEGER", nullable: false), + TensorComboId = table.Column(type: "TEXT", nullable: false), + AiModelHashId = table.Column(type: "INTEGER", nullable: false), + ImatrixDefinitionId = table.Column(type: "INTEGER", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_AiBenchmarks", x => x.Id); + table.ForeignKey( + name: "FK_AiBenchmarks_AiModelHashes_AiModelHashId", + column: x => x.AiModelHashId, + principalTable: "AiModelHashes", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_AiBenchmarks_ArchitectureFamilies_ArchitectureFamilyId", + column: x => x.ArchitectureFamilyId, + principalTable: "ArchitectureFamilies", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_AiBenchmarks_ImatrixDefinitions_ImatrixDefinitionId", + column: x => x.ImatrixDefinitionId, + principalTable: "ImatrixDefinitions", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_AiBenchmarks_TensorCombos_TensorComboId", + column: x => x.TensorComboId, + principalTable: "TensorCombos", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_AiBenchmarks_TensorGroupProfiles_TensorGroupProfileId", + column: x => x.TensorGroupProfileId, + principalTable: "TensorGroupProfiles", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + }); + + migrationBuilder.CreateTable( + name: "ExecutionPlanProbeCaches", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + ArchitectureFamilyId = table.Column(type: "INTEGER", nullable: false), + TensorGroupProfileId = table.Column(type: "INTEGER", nullable: false), + AiModelHashId = table.Column(type: "INTEGER", nullable: false), + ImatrixDefinitionId = table.Column(type: "INTEGER", nullable: true), + HardwareFingerprint = table.Column(type: "TEXT", maxLength: 1024, nullable: false), + QuantizedModelFingerprint = table.Column(type: "TEXT", maxLength: 2048, nullable: false), + QuantizationKey = table.Column(type: "TEXT", maxLength: 128, nullable: false), + DiscoveryTokenTarget = table.Column(type: "INTEGER", nullable: false), + StaticNgl = table.Column(type: "INTEGER", nullable: false), + UsesGpu = table.Column(type: "INTEGER", nullable: false), + GroupSize = table.Column(type: "INTEGER", nullable: false), + SlotsJson = table.Column(type: "TEXT", maxLength: 8000, nullable: false), + ProbeSchemaVersion = table.Column(type: "INTEGER", nullable: false), + Q8ModelSizeBytes = table.Column(type: "INTEGER", nullable: false), + Q8StableNgl = table.Column(type: "INTEGER", nullable: false), + NativeModelSizeBytes = table.Column(type: "INTEGER", nullable: false), + NativeStableNgl = table.Column(type: "INTEGER", nullable: false), + NativeQuantizationKey = table.Column(type: "TEXT", maxLength: 128, nullable: false), + MaxCandidateNgl = table.Column(type: "INTEGER", nullable: false), + GpuMemoryLimitsJson = table.Column(type: "TEXT", maxLength: 4000, nullable: false), + TensorSplitJson = table.Column(type: "TEXT", maxLength: 4000, nullable: false), + CreatedUtc = table.Column(type: "TEXT", nullable: false), + UpdatedUtc = table.Column(type: "TEXT", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_ExecutionPlanProbeCaches", x => x.Id); + table.ForeignKey( + name: "FK_ExecutionPlanProbeCaches_AiModelHashes_AiModelHashId", + column: x => x.AiModelHashId, + principalTable: "AiModelHashes", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_ExecutionPlanProbeCaches_ArchitectureFamilies_ArchitectureFamilyId", + column: x => x.ArchitectureFamilyId, + principalTable: "ArchitectureFamilies", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_ExecutionPlanProbeCaches_ImatrixDefinitions_ImatrixDefinitionId", + column: x => x.ImatrixDefinitionId, + principalTable: "ImatrixDefinitions", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_ExecutionPlanProbeCaches_TensorGroupProfiles_TensorGroupProfileId", + column: x => x.TensorGroupProfileId, + principalTable: "TensorGroupProfiles", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + }); + + migrationBuilder.CreateTable( + name: "AiBenchmarkLearnedSources", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + AiBenchmarkId = table.Column(type: "TEXT", nullable: false), + ArchitectureFamilyId = table.Column(type: "INTEGER", nullable: false), + TensorGroupProfileId = table.Column(type: "INTEGER", nullable: false), + TensorComboId = table.Column(type: "TEXT", nullable: false), + TensorGroupId = table.Column(type: "INTEGER", nullable: false), + BaselineQuantDefinitionId = table.Column(type: "INTEGER", nullable: false), + SourceLearningBenchmarkId = table.Column(type: "TEXT", nullable: true), + BaselineCanonicalKey = table.Column(type: "TEXT", maxLength: 512, nullable: false), + CreatedUtc = table.Column(type: "TEXT", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_AiBenchmarkLearnedSources", x => x.Id); + table.ForeignKey( + name: "FK_AiBenchmarkLearnedSources_AiBenchmarks_AiBenchmarkId", + column: x => x.AiBenchmarkId, + principalTable: "AiBenchmarks", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_AiBenchmarkLearnedSources_AiBenchmarks_SourceLearningBenchmarkId", + column: x => x.SourceLearningBenchmarkId, + principalTable: "AiBenchmarks", + principalColumn: "Id", + onDelete: ReferentialAction.SetNull); + table.ForeignKey( + name: "FK_AiBenchmarkLearnedSources_ArchitectureFamilies_ArchitectureFamilyId", + column: x => x.ArchitectureFamilyId, + principalTable: "ArchitectureFamilies", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_AiBenchmarkLearnedSources_BaselineQuantDefinitions_BaselineQuantDefinitionId", + column: x => x.BaselineQuantDefinitionId, + principalTable: "BaselineQuantDefinitions", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_AiBenchmarkLearnedSources_TensorCombos_TensorComboId", + column: x => x.TensorComboId, + principalTable: "TensorCombos", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_AiBenchmarkLearnedSources_TensorGroupProfiles_TensorGroupProfileId", + column: x => x.TensorGroupProfileId, + principalTable: "TensorGroupProfiles", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "CategoryBenchmark", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + AiBenchmarkId = table.Column(type: "TEXT", nullable: false), + Category = table.Column(type: "INTEGER", nullable: false), + Kld = table.Column(type: "REAL", nullable: false), + Ppl = table.Column(type: "REAL", nullable: false), + PplError = table.Column(type: "REAL", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_CategoryBenchmark", x => x.Id); + table.ForeignKey( + name: "FK_CategoryBenchmark_AiBenchmarks_AiBenchmarkId", + column: x => x.AiBenchmarkId, + principalTable: "AiBenchmarks", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "LearnedBaselineTensorQuants", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + ArchitectureFamilyId = table.Column(type: "INTEGER", nullable: false), + TensorGroupProfileId = table.Column(type: "INTEGER", nullable: false), + BaselineQuantDefinitionId = table.Column(type: "INTEGER", nullable: false), + TensorComboId = table.Column(type: "TEXT", nullable: false), + AiBenchmarkId = table.Column(type: "TEXT", nullable: false), + AiModelHashId = table.Column(type: "INTEGER", nullable: false), + BaselineQuantId = table.Column(type: "INTEGER", nullable: false), + BaselineCanonicalKey = table.Column(type: "TEXT", maxLength: 512, nullable: false), + BaselineSourceKind = table.Column(type: "TEXT", maxLength: 64, nullable: false), + BaselineSourceRepository = table.Column(type: "TEXT", maxLength: 256, nullable: true), + BaselineSourceFileName = table.Column(type: "TEXT", maxLength: 512, nullable: true), + TensorWeightSchemeId = table.Column(type: "INTEGER", nullable: false), + TensorGroupId = table.Column(type: "INTEGER", nullable: false), + TensorName = table.Column(type: "TEXT", maxLength: 512, nullable: false), + FinalQuantType = table.Column(type: "TEXT", maxLength: 32, nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_LearnedBaselineTensorQuants", x => x.Id); + table.ForeignKey( + name: "FK_LearnedBaselineTensorQuants_AiBenchmarks_AiBenchmarkId", + column: x => x.AiBenchmarkId, + principalTable: "AiBenchmarks", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_LearnedBaselineTensorQuants_AiModelHashes_AiModelHashId", + column: x => x.AiModelHashId, + principalTable: "AiModelHashes", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_LearnedBaselineTensorQuants_ArchitectureFamilies_ArchitectureFamilyId", + column: x => x.ArchitectureFamilyId, + principalTable: "ArchitectureFamilies", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_LearnedBaselineTensorQuants_BaselineQuantDefinitions_BaselineQuantDefinitionId", + column: x => x.BaselineQuantDefinitionId, + principalTable: "BaselineQuantDefinitions", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_LearnedBaselineTensorQuants_TensorCombos_TensorComboId", + column: x => x.TensorComboId, + principalTable: "TensorCombos", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_LearnedBaselineTensorQuants_TensorGroupProfiles_TensorGroupProfileId", + column: x => x.TensorGroupProfileId, + principalTable: "TensorGroupProfiles", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + }); + + migrationBuilder.CreateTable( + name: "QuantizationRuns", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + ArchitectureFamilyId = table.Column(type: "INTEGER", nullable: false), + TensorGroupProfileId = table.Column(type: "INTEGER", nullable: false), + AiModelHashId = table.Column(type: "INTEGER", nullable: false), + ImatrixDefinitionId = table.Column(type: "INTEGER", nullable: true), + TensorComboId = table.Column(type: "TEXT", nullable: false), + AiBenchmarkId = table.Column(type: "TEXT", nullable: true), + StartedUtc = table.Column(type: "TEXT", nullable: false), + CompletedUtc = table.Column(type: "TEXT", nullable: false), + DurationMs = table.Column(type: "INTEGER", nullable: false), + Succeeded = table.Column(type: "INTEGER", nullable: false), + Error = table.Column(type: "TEXT", maxLength: 4000, nullable: true), + OutputModelPath = table.Column(type: "TEXT", maxLength: 2048, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_QuantizationRuns", x => x.Id); + table.ForeignKey( + name: "FK_QuantizationRuns_AiBenchmarks_AiBenchmarkId", + column: x => x.AiBenchmarkId, + principalTable: "AiBenchmarks", + principalColumn: "Id", + onDelete: ReferentialAction.SetNull); + table.ForeignKey( + name: "FK_QuantizationRuns_AiModelHashes_AiModelHashId", + column: x => x.AiModelHashId, + principalTable: "AiModelHashes", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_QuantizationRuns_ArchitectureFamilies_ArchitectureFamilyId", + column: x => x.ArchitectureFamilyId, + principalTable: "ArchitectureFamilies", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_QuantizationRuns_ImatrixDefinitions_ImatrixDefinitionId", + column: x => x.ImatrixDefinitionId, + principalTable: "ImatrixDefinitions", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_QuantizationRuns_TensorCombos_TensorComboId", + column: x => x.TensorComboId, + principalTable: "TensorCombos", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_QuantizationRuns_TensorGroupProfiles_TensorGroupProfileId", + column: x => x.TensorGroupProfileId, + principalTable: "TensorGroupProfiles", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + }); + + migrationBuilder.CreateTable( + name: "BenchmarkRuns", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + ArchitectureFamilyId = table.Column(type: "INTEGER", nullable: false), + TensorGroupProfileId = table.Column(type: "INTEGER", nullable: false), + AiModelHashId = table.Column(type: "INTEGER", nullable: false), + ImatrixDefinitionId = table.Column(type: "INTEGER", nullable: true), + TensorComboId = table.Column(type: "TEXT", nullable: false), + AiBenchmarkId = table.Column(type: "TEXT", nullable: false), + CategoryBenchmarkId = table.Column(type: "TEXT", nullable: true), + Category = table.Column(type: "INTEGER", nullable: false), + StartedUtc = table.Column(type: "TEXT", nullable: false), + CompletedUtc = table.Column(type: "TEXT", nullable: false), + DurationMs = table.Column(type: "INTEGER", nullable: false), + Succeeded = table.Column(type: "INTEGER", nullable: false), + Error = table.Column(type: "TEXT", maxLength: 4000, nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_BenchmarkRuns", x => x.Id); + table.ForeignKey( + name: "FK_BenchmarkRuns_AiBenchmarks_AiBenchmarkId", + column: x => x.AiBenchmarkId, + principalTable: "AiBenchmarks", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_BenchmarkRuns_AiModelHashes_AiModelHashId", + column: x => x.AiModelHashId, + principalTable: "AiModelHashes", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_BenchmarkRuns_ArchitectureFamilies_ArchitectureFamilyId", + column: x => x.ArchitectureFamilyId, + principalTable: "ArchitectureFamilies", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_BenchmarkRuns_CategoryBenchmark_CategoryBenchmarkId", + column: x => x.CategoryBenchmarkId, + principalTable: "CategoryBenchmark", + principalColumn: "Id", + onDelete: ReferentialAction.SetNull); + table.ForeignKey( + name: "FK_BenchmarkRuns_ImatrixDefinitions_ImatrixDefinitionId", + column: x => x.ImatrixDefinitionId, + principalTable: "ImatrixDefinitions", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_BenchmarkRuns_TensorCombos_TensorComboId", + column: x => x.TensorComboId, + principalTable: "TensorCombos", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_BenchmarkRuns_TensorGroupProfiles_TensorGroupProfileId", + column: x => x.TensorGroupProfileId, + principalTable: "TensorGroupProfiles", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + }); + + migrationBuilder.CreateIndex( + name: "IX_AiBenchmarkLearnedSources_AiBenchmarkId_TensorGroupId", + table: "AiBenchmarkLearnedSources", + columns: new[] { "AiBenchmarkId", "TensorGroupId" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_AiBenchmarkLearnedSources_ArchitectureFamilyId_TensorGroupProfileId_BaselineQuantDefinitionId", + table: "AiBenchmarkLearnedSources", + columns: new[] { "ArchitectureFamilyId", "TensorGroupProfileId", "BaselineQuantDefinitionId" }); + + migrationBuilder.CreateIndex( + name: "IX_AiBenchmarkLearnedSources_BaselineQuantDefinitionId", + table: "AiBenchmarkLearnedSources", + column: "BaselineQuantDefinitionId"); + + migrationBuilder.CreateIndex( + name: "IX_AiBenchmarkLearnedSources_SourceLearningBenchmarkId", + table: "AiBenchmarkLearnedSources", + column: "SourceLearningBenchmarkId"); + + migrationBuilder.CreateIndex( + name: "IX_AiBenchmarkLearnedSources_TensorComboId", + table: "AiBenchmarkLearnedSources", + column: "TensorComboId"); + + migrationBuilder.CreateIndex( + name: "IX_AiBenchmarkLearnedSources_TensorGroupProfileId", + table: "AiBenchmarkLearnedSources", + column: "TensorGroupProfileId"); + + migrationBuilder.CreateIndex( + name: "IX_AiBenchmarks_AiModelHashId", + table: "AiBenchmarks", + column: "AiModelHashId"); + + migrationBuilder.CreateIndex( + name: "IX_AiBenchmarks_ArchitectureFamilyId_TensorGroupProfileId_AiModelHashId_ImatrixDefinitionId_TensorComboId", + table: "AiBenchmarks", + columns: new[] { "ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "TensorComboId" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_AiBenchmarks_ArchitectureFamilyId_TensorGroupProfileId_TensorComboId", + table: "AiBenchmarks", + columns: new[] { "ArchitectureFamilyId", "TensorGroupProfileId", "TensorComboId" }); + + migrationBuilder.CreateIndex( + name: "IX_AiBenchmarks_ImatrixDefinitionId", + table: "AiBenchmarks", + column: "ImatrixDefinitionId"); + + migrationBuilder.CreateIndex( + name: "IX_AiBenchmarks_TensorComboId", + table: "AiBenchmarks", + column: "TensorComboId"); + + migrationBuilder.CreateIndex( + name: "IX_AiBenchmarks_TensorGroupProfileId", + table: "AiBenchmarks", + column: "TensorGroupProfileId"); + + migrationBuilder.CreateIndex( + name: "IX_AiModelHashes_UniqueHash", + table: "AiModelHashes", + column: "UniqueHash"); + + migrationBuilder.CreateIndex( + name: "IX_ArchitectureFamilies_NormalizedName", + table: "ArchitectureFamilies", + column: "NormalizedName", + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_ArchitectureFamilies_TensorSignatureHash_TensorCount", + table: "ArchitectureFamilies", + columns: new[] { "TensorSignatureHash", "TensorCount" }); + + migrationBuilder.CreateIndex( + name: "IX_ArchitectureFamilyModelHashes_AiModelHashId", + table: "ArchitectureFamilyModelHashes", + column: "AiModelHashId", + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_ArchitectureFamilyModelHashes_ArchitectureFamilyId_AiModelHashId", + table: "ArchitectureFamilyModelHashes", + columns: new[] { "ArchitectureFamilyId", "AiModelHashId" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_BaselineQuantDefinitions_ArchitectureFamilyId_NormalizedCanonicalKey", + table: "BaselineQuantDefinitions", + columns: new[] { "ArchitectureFamilyId", "NormalizedCanonicalKey" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_BaselineQuantDefinitions_ArchitectureFamilyId_NormalizedSourceRepository_NormalizedSourceFileName", + table: "BaselineQuantDefinitions", + columns: new[] { "ArchitectureFamilyId", "NormalizedSourceRepository", "NormalizedSourceFileName" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_BaselineQuantDefinitions_ArchitectureFamilyId_RuntimeBaselineId", + table: "BaselineQuantDefinitions", + columns: new[] { "ArchitectureFamilyId", "RuntimeBaselineId" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_BaselineQuantDefinitions_IsActiveInCurrentConfig", + table: "BaselineQuantDefinitions", + column: "IsActiveInCurrentConfig"); + + migrationBuilder.CreateIndex( + name: "IX_BaselineQuantDefinitions_RuntimeBaselineId_ArchitectureFamilyId", + table: "BaselineQuantDefinitions", + columns: new[] { "RuntimeBaselineId", "ArchitectureFamilyId" }); + + migrationBuilder.CreateIndex( + name: "IX_BenchmarkRuns_AiBenchmarkId", + table: "BenchmarkRuns", + column: "AiBenchmarkId"); + + migrationBuilder.CreateIndex( + name: "IX_BenchmarkRuns_AiBenchmarkId_Category", + table: "BenchmarkRuns", + columns: new[] { "AiBenchmarkId", "Category" }); + + migrationBuilder.CreateIndex( + name: "IX_BenchmarkRuns_AiModelHashId", + table: "BenchmarkRuns", + column: "AiModelHashId"); + + migrationBuilder.CreateIndex( + name: "IX_BenchmarkRuns_ArchitectureFamilyId", + table: "BenchmarkRuns", + column: "ArchitectureFamilyId"); + + migrationBuilder.CreateIndex( + name: "IX_BenchmarkRuns_CategoryBenchmarkId", + table: "BenchmarkRuns", + column: "CategoryBenchmarkId"); + + migrationBuilder.CreateIndex( + name: "IX_BenchmarkRuns_ImatrixDefinitionId", + table: "BenchmarkRuns", + column: "ImatrixDefinitionId"); + + migrationBuilder.CreateIndex( + name: "IX_BenchmarkRuns_StartedUtc", + table: "BenchmarkRuns", + column: "StartedUtc"); + + migrationBuilder.CreateIndex( + name: "IX_BenchmarkRuns_TensorComboId", + table: "BenchmarkRuns", + column: "TensorComboId"); + + migrationBuilder.CreateIndex( + name: "IX_BenchmarkRuns_TensorGroupProfileId", + table: "BenchmarkRuns", + column: "TensorGroupProfileId"); + + migrationBuilder.CreateIndex( + name: "IX_CategoryBenchmark_AiBenchmarkId", + table: "CategoryBenchmark", + column: "AiBenchmarkId"); + + migrationBuilder.CreateIndex( + name: "IX_ExecutionPlanProbeCaches_AiModelHashId", + table: "ExecutionPlanProbeCaches", + column: "AiModelHashId"); + + migrationBuilder.CreateIndex( + name: "IX_ExecutionPlanProbeCaches_ArchitectureFamilyId", + table: "ExecutionPlanProbeCaches", + column: "ArchitectureFamilyId"); + + migrationBuilder.CreateIndex( + name: "IX_ExecutionPlanProbeCaches_ArchitectureFamilyId_TensorGroupProfileId_AiModelHashId_ImatrixDefinitionId_HardwareFingerprint_QuantizedModelFingerprint_QuantizationKey_DiscoveryTokenTarget", + table: "ExecutionPlanProbeCaches", + columns: new[] { "ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "HardwareFingerprint", "QuantizedModelFingerprint", "QuantizationKey", "DiscoveryTokenTarget" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_ExecutionPlanProbeCaches_ImatrixDefinitionId", + table: "ExecutionPlanProbeCaches", + column: "ImatrixDefinitionId"); + + migrationBuilder.CreateIndex( + name: "IX_ExecutionPlanProbeCaches_TensorGroupProfileId", + table: "ExecutionPlanProbeCaches", + column: "TensorGroupProfileId"); + + migrationBuilder.CreateIndex( + name: "IX_ImatrixDefinitions_AiModelHashId_IdentityHash", + table: "ImatrixDefinitions", + columns: new[] { "AiModelHashId", "IdentityHash" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_LearnedBaselineTensorQuants_AiBenchmarkId", + table: "LearnedBaselineTensorQuants", + column: "AiBenchmarkId"); + + migrationBuilder.CreateIndex( + name: "IX_LearnedBaselineTensorQuants_AiModelHashId", + table: "LearnedBaselineTensorQuants", + column: "AiModelHashId"); + + migrationBuilder.CreateIndex( + name: "IX_LearnedBaselineTensorQuants_ArchitectureFamilyId_TensorGroupProfileId_BaselineQuantDefinitionId_TensorGroupId", + table: "LearnedBaselineTensorQuants", + columns: new[] { "ArchitectureFamilyId", "TensorGroupProfileId", "BaselineQuantDefinitionId", "TensorGroupId" }); + + migrationBuilder.CreateIndex( + name: "IX_LearnedBaselineTensorQuants_ArchitectureFamilyId_TensorGroupProfileId_BaselineQuantDefinitionId_TensorWeightSchemeId_TensorName", + table: "LearnedBaselineTensorQuants", + columns: new[] { "ArchitectureFamilyId", "TensorGroupProfileId", "BaselineQuantDefinitionId", "TensorWeightSchemeId", "TensorName" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_LearnedBaselineTensorQuants_BaselineQuantDefinitionId", + table: "LearnedBaselineTensorQuants", + column: "BaselineQuantDefinitionId"); + + migrationBuilder.CreateIndex( + name: "IX_LearnedBaselineTensorQuants_TensorComboId", + table: "LearnedBaselineTensorQuants", + column: "TensorComboId"); + + migrationBuilder.CreateIndex( + name: "IX_LearnedBaselineTensorQuants_TensorGroupProfileId", + table: "LearnedBaselineTensorQuants", + column: "TensorGroupProfileId"); + + migrationBuilder.CreateIndex( + name: "IX_QuantizationRuns_AiBenchmarkId", + table: "QuantizationRuns", + column: "AiBenchmarkId"); + + migrationBuilder.CreateIndex( + name: "IX_QuantizationRuns_AiModelHashId", + table: "QuantizationRuns", + column: "AiModelHashId"); + + migrationBuilder.CreateIndex( + name: "IX_QuantizationRuns_ArchitectureFamilyId", + table: "QuantizationRuns", + column: "ArchitectureFamilyId"); + + migrationBuilder.CreateIndex( + name: "IX_QuantizationRuns_ImatrixDefinitionId", + table: "QuantizationRuns", + column: "ImatrixDefinitionId"); + + migrationBuilder.CreateIndex( + name: "IX_QuantizationRuns_StartedUtc", + table: "QuantizationRuns", + column: "StartedUtc"); + + migrationBuilder.CreateIndex( + name: "IX_QuantizationRuns_TensorComboId", + table: "QuantizationRuns", + column: "TensorComboId"); + + migrationBuilder.CreateIndex( + name: "IX_QuantizationRuns_TensorGroupProfileId", + table: "QuantizationRuns", + column: "TensorGroupProfileId"); + + migrationBuilder.CreateIndex( + name: "IX_TensorCombos_BaseQuant_Embeddings_LmHead_AttnQ_AttnKV_AttnOutput_FfnUpGate_FfnDown_MoeExperts_MoeRouter", + table: "TensorCombos", + columns: new[] { "BaseQuant", "Embeddings", "LmHead", "AttnQ", "AttnKV", "AttnOutput", "FfnUpGate", "FfnDown", "MoeExperts", "MoeRouter" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_TensorGroupProfiles_ArchitectureFamilyId_FingerprintHash", + table: "TensorGroupProfiles", + columns: new[] { "ArchitectureFamilyId", "FingerprintHash" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_TensorGroupProfiles_ArchitectureFamilyId_IsActive", + table: "TensorGroupProfiles", + columns: new[] { "ArchitectureFamilyId", "IsActive" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "AiBenchmarkLearnedSources"); + + migrationBuilder.DropTable( + name: "ArchitectureFamilyModelHashes"); + + migrationBuilder.DropTable( + name: "BenchmarkRuns"); + + migrationBuilder.DropTable( + name: "ExecutionPlanProbeCaches"); + + migrationBuilder.DropTable( + name: "LearnedBaselineTensorQuants"); + + migrationBuilder.DropTable( + name: "QuantizationRuns"); + + migrationBuilder.DropTable( + name: "CategoryBenchmark"); + + migrationBuilder.DropTable( + name: "BaselineQuantDefinitions"); + + migrationBuilder.DropTable( + name: "AiBenchmarks"); + + migrationBuilder.DropTable( + name: "ImatrixDefinitions"); + + migrationBuilder.DropTable( + name: "TensorCombos"); + + migrationBuilder.DropTable( + name: "TensorGroupProfiles"); + + migrationBuilder.DropTable( + name: "AiModelHashes"); + + migrationBuilder.DropTable( + name: "ArchitectureFamilies"); + } + } +} diff --git a/src/MQ.DB/Migrations/20260503222121_PredictionEngineAnomalyDetect.Designer.cs b/src/MQ.DB/Migrations/20260503222121_PredictionEngineAnomalyDetect.Designer.cs new file mode 100644 index 0000000..8dfb79f --- /dev/null +++ b/src/MQ.DB/Migrations/20260503222121_PredictionEngineAnomalyDetect.Designer.cs @@ -0,0 +1,1685 @@ +// +using System; +using MQ.DB.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace MQ.DB.Migrations +{ + [DbContext(typeof(MagicQuantContext))] + [Migration("20260503222121_PredictionEngineAnomalyDetect")] + partial class PredictionEngineAnomalyDetect + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "10.0.1"); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmark", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("Ngl") + .HasColumnType("INTEGER"); + + b.Property("SizeBytes") + .HasColumnType("INTEGER"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("TokensPerSecond") + .HasColumnType("REAL"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "TensorComboId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "TensorComboId") + .IsUnique(); + + b.ToTable("AiBenchmarks"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmarkLearnedSource", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BaselineCanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("BaselineQuantDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("SourceLearningBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupId") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("BaselineQuantDefinitionId"); + + b.HasIndex("SourceLearningBenchmarkId"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("AiBenchmarkId", "TensorGroupId") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "BaselineQuantDefinitionId"); + + b.ToTable("AiBenchmarkLearnedSources"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiModelHash", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("UniqueHash") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("UniqueHash"); + + b.ToTable("AiModelHashes"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyInteractionRule", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("AppliedPredictionSpaceAdjustmentKld") + .HasColumnType("REAL"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BenchmarkCategory") + .HasColumnType("INTEGER"); + + b.Property("BestActualGainVsTwin") + .HasColumnType("REAL"); + + b.Property("BestPredictionSpaceGap") + .HasColumnType("REAL"); + + b.Property("CandidateEffectiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Confidence") + .HasColumnType("REAL"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("EvidenceCount") + .HasColumnType("INTEGER"); + + b.Property("FullTensorConfigKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("GroupCount") + .HasColumnType("INTEGER"); + + b.Property("GroupSetHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("InactiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("MeanActualGainVsTwin") + .HasColumnType("REAL"); + + b.Property("MeanPredictionSpaceGap") + .HasColumnType("REAL"); + + b.Property("MetadataJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("MovementClassification") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("ReferenceContextKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ReferenceEffectiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ReferenceQuantId") + .HasColumnType("INTEGER"); + + b.Property("RuleDirection") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("RuleStatus") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("RuleType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("ShrinkFactor") + .HasColumnType("REAL"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("UpdatedUtc") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("FullTensorConfigKey"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "RuleDirection", "RuleStatus"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "ReferenceQuantId", "GroupSetHash", "RuleDirection") + .IsUnique(); + + b.ToTable("AnomalyInteractionRules"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyInteractionRuleGroupState", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("CandidateQuantId") + .HasColumnType("INTEGER"); + + b.Property("Movement") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("ReferenceQuantId") + .HasColumnType("INTEGER"); + + b.Property("RuleId") + .HasColumnType("TEXT"); + + b.Property("SortOrder") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("RuleId", "TensorGroupId") + .IsUnique(); + + b.ToTable("AnomalyInteractionRuleGroupStates"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyProbeObservation", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("Accepted") + .HasColumnType("INTEGER"); + + b.Property("ActualGainVsTwin") + .HasColumnType("REAL"); + + b.Property("ActualKld") + .HasColumnType("REAL"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("AllActiveGroupsExplicit") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BenchmarkCategory") + .HasColumnType("INTEGER"); + + b.Property("CandidateEffectiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CandidateQuantsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ChangedGroupSetHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ChangedGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Classification") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("DowngradeCount") + .HasColumnType("INTEGER"); + + b.Property("FailureCode") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("HypothesisLabel") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("InactiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("IsContextualAnomalyProbe") + .HasColumnType("INTEGER"); + + b.Property("Message") + .IsRequired() + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("MovementClassification") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("NetBitDelta") + .HasColumnType("INTEGER"); + + b.Property("OldBf16Isolation") + .HasColumnType("INTEGER"); + + b.Property("PredictedKld") + .HasColumnType("REAL"); + + b.Property("PredictionSpaceGapVsTwin") + .HasColumnType("REAL"); + + b.Property("ProbeTensorComboId") + .HasColumnType("TEXT"); + + b.Property("ProbeTensorConfigKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ProbeType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("ReferenceActualKld") + .HasColumnType("REAL"); + + b.Property("ReferenceEffectiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ReferencePredictedKld") + .HasColumnType("REAL"); + + b.Property("ReferenceQuantId") + .HasColumnType("INTEGER"); + + b.Property("ReferenceTensorComboId") + .HasColumnType("TEXT"); + + b.Property("ReferenceTensorConfigKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("RuleDirection") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("SameCount") + .HasColumnType("INTEGER"); + + b.Property("SessionId") + .HasColumnType("TEXT"); + + b.Property("SizeSavingsBytes") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("UnknownCount") + .HasColumnType("INTEGER"); + + b.Property("UpgradeCount") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("ProbeTensorComboId"); + + b.HasIndex("ProbeTensorConfigKey"); + + b.HasIndex("ReferenceTensorComboId"); + + b.HasIndex("ReferenceTensorConfigKey"); + + b.HasIndex("SessionId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "ChangedGroupSetHash"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "ReferenceTensorComboId", "ProbeTensorComboId", "ProbeType"); + + b.ToTable("AnomalyProbeObservations"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyProbeSession", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BenchmarkCategory") + .HasColumnType("INTEGER"); + + b.Property("CompletedUtc") + .HasColumnType("TEXT"); + + b.Property("ConfigJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("SourceRunLabel") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("StartedUtc") + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "StartedUtc"); + + b.ToTable("AnomalyProbeSessions"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ArchitectureFamily", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("NormalizedName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("TensorCount") + .HasColumnType("INTEGER"); + + b.Property("TensorSignatureHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique(); + + b.HasIndex("TensorSignatureHash", "TensorCount"); + + b.ToTable("ArchitectureFamilies"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ArchitectureFamilyModelHash", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("IsCanonical") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "AiModelHashId") + .IsUnique(); + + b.ToTable("ArchitectureFamilyModelHashes"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BaselineQuantDefinition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BaselineFamily") + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("BaselineName") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("BitRange") + .HasColumnType("INTEGER"); + + b.Property("CanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("DefaultTensorSchemeId") + .HasColumnType("INTEGER"); + + b.Property("DefaultTensorSchemeName") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ExplicitCandidateSortOrder") + .HasColumnType("INTEGER"); + + b.Property("FirstSeenUtc") + .HasColumnType("TEXT"); + + b.Property("IsActiveInCurrentConfig") + .HasColumnType("INTEGER"); + + b.Property("IsCombinationCarrierCandidate") + .HasColumnType("INTEGER"); + + b.Property("IsCustomBaseline") + .HasColumnType("INTEGER"); + + b.Property("IsExplicitGroupCombinationCandidate") + .HasColumnType("INTEGER"); + + b.Property("IsLearningBaseline") + .HasColumnType("INTEGER"); + + b.Property("LastSeenUtc") + .HasColumnType("TEXT"); + + b.Property("LastUpdatedUtc") + .HasColumnType("TEXT"); + + b.Property("NormalizedCanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("NormalizedSourceFileName") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("NormalizedSourceRepository") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("QuantizeBaseArgumentName") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("RequiresImatrix") + .HasColumnType("INTEGER"); + + b.Property("RuntimeBaselineId") + .HasColumnType("INTEGER"); + + b.Property("ShortSourceName") + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("SourceFileName") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("SourceKind") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("SourceOwner") + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("SourceRepository") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("IsActiveInCurrentConfig"); + + b.HasIndex("ArchitectureFamilyId", "NormalizedCanonicalKey") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "RuntimeBaselineId") + .IsUnique(); + + b.HasIndex("RuntimeBaselineId", "ArchitectureFamilyId"); + + b.HasIndex("ArchitectureFamilyId", "NormalizedSourceRepository", "NormalizedSourceFileName") + .IsUnique(); + + b.ToTable("BaselineQuantDefinitions"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BenchmarkRun", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("Category") + .HasColumnType("INTEGER"); + + b.Property("CategoryBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("CompletedUtc") + .HasColumnType("TEXT"); + + b.Property("DurationMs") + .HasColumnType("INTEGER"); + + b.Property("Error") + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("StartedUtc") + .HasColumnType("TEXT"); + + b.Property("Succeeded") + .HasColumnType("INTEGER"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ArchitectureFamilyId"); + + b.HasIndex("CategoryBenchmarkId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("StartedUtc"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("AiBenchmarkId", "Category"); + + b.ToTable("BenchmarkRuns"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.CategoryBenchmark", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("Category") + .HasColumnType("INTEGER"); + + b.Property("Kld") + .HasColumnType("REAL"); + + b.Property("Ppl") + .HasColumnType("REAL"); + + b.Property("PplError") + .HasColumnType("REAL"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.ToTable("CategoryBenchmark"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ExecutionPlanProbeCache", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("DiscoveryTokenTarget") + .HasColumnType("INTEGER"); + + b.Property("GpuMemoryLimitsJson") + .IsRequired() + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("GroupSize") + .HasColumnType("INTEGER"); + + b.Property("HardwareFingerprint") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("MaxCandidateNgl") + .HasColumnType("INTEGER"); + + b.Property("NativeModelSizeBytes") + .HasColumnType("INTEGER"); + + b.Property("NativeQuantizationKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("NativeStableNgl") + .HasColumnType("INTEGER"); + + b.Property("ProbeSchemaVersion") + .HasColumnType("INTEGER"); + + b.Property("Q8ModelSizeBytes") + .HasColumnType("INTEGER"); + + b.Property("Q8StableNgl") + .HasColumnType("INTEGER"); + + b.Property("QuantizationKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("QuantizedModelFingerprint") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("TEXT"); + + b.Property("SlotsJson") + .IsRequired() + .HasMaxLength(8000) + .HasColumnType("TEXT"); + + b.Property("StaticNgl") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("TensorSplitJson") + .IsRequired() + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("UpdatedUtc") + .HasColumnType("TEXT"); + + b.Property("UsesGpu") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ArchitectureFamilyId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "HardwareFingerprint", "QuantizedModelFingerprint", "QuantizationKey", "DiscoveryTokenTarget") + .IsUnique(); + + b.ToTable("ExecutionPlanProbeCaches"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ImatrixDefinition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("BuildFingerprint") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("CanonicalPath") + .HasMaxLength(2048) + .HasColumnType("TEXT"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("IdentityHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("MetadataJson") + .HasMaxLength(8000) + .HasColumnType("TEXT"); + + b.Property("SourceKind") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("TokenCount") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId", "IdentityHash") + .IsUnique(); + + b.ToTable("ImatrixDefinitions"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.LearnedBaselineTensorQuant", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BaselineCanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("BaselineQuantDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("BaselineQuantId") + .HasColumnType("INTEGER"); + + b.Property("BaselineSourceFileName") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("BaselineSourceKind") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("BaselineSourceRepository") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("FinalQuantType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("TEXT"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupId") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("TensorName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("TensorWeightSchemeId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("BaselineQuantDefinitionId"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "BaselineQuantDefinitionId", "TensorGroupId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "BaselineQuantDefinitionId", "TensorWeightSchemeId", "TensorName") + .IsUnique(); + + b.ToTable("LearnedBaselineTensorQuants"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.QuantizationRun", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CompletedUtc") + .HasColumnType("TEXT"); + + b.Property("DurationMs") + .HasColumnType("INTEGER"); + + b.Property("Error") + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("OutputModelPath") + .HasMaxLength(2048) + .HasColumnType("TEXT"); + + b.Property("StartedUtc") + .HasColumnType("TEXT"); + + b.Property("Succeeded") + .HasColumnType("INTEGER"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ArchitectureFamilyId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("StartedUtc"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.ToTable("QuantizationRuns"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.TensorCombo", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AttnKV") + .HasColumnType("INTEGER"); + + b.Property("AttnOutput") + .HasColumnType("INTEGER"); + + b.Property("AttnQ") + .HasColumnType("INTEGER"); + + b.Property("BaseQuant") + .HasColumnType("INTEGER"); + + b.Property("Embeddings") + .HasColumnType("INTEGER"); + + b.Property("FfnDown") + .HasColumnType("INTEGER"); + + b.Property("FfnUpGate") + .HasColumnType("INTEGER"); + + b.Property("LmHead") + .HasColumnType("INTEGER"); + + b.Property("MoeExperts") + .HasColumnType("INTEGER"); + + b.Property("MoeRouter") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("BaseQuant", "Embeddings", "LmHead", "AttnQ", "AttnKV", "AttnOutput", "FfnUpGate", "FfnDown", "MoeExperts", "MoeRouter") + .IsUnique(); + + b.ToTable("TensorCombos"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.TensorGroupProfile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("FingerprintHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("IsActive") + .HasColumnType("INTEGER"); + + b.Property("SnapshotJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("ArchitectureFamilyId", "FingerprintHash") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "IsActive"); + + b.ToTable("TensorGroupProfiles"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmark", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmarkLearnedSource", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany("LearnedSources") + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.BaselineQuantDefinition", "BaselineQuantDefinition") + .WithMany() + .HasForeignKey("BaselineQuantDefinitionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "SourceLearningBenchmark") + .WithMany() + .HasForeignKey("SourceLearningBenchmarkId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("BaselineQuantDefinition"); + + b.Navigation("SourceLearningBenchmark"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyInteractionRule", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyInteractionRuleGroupState", b => + { + b.HasOne("MQ.DB.Models.DbModels.AnomalyInteractionRule", "Rule") + .WithMany("GroupStates") + .HasForeignKey("RuleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Rule"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyProbeObservation", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "ProbeTensorCombo") + .WithMany() + .HasForeignKey("ProbeTensorComboId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "ReferenceTensorCombo") + .WithMany() + .HasForeignKey("ReferenceTensorComboId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.AnomalyProbeSession", "Session") + .WithMany("Observations") + .HasForeignKey("SessionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("ProbeTensorCombo"); + + b.Navigation("ReferenceTensorCombo"); + + b.Navigation("Session"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyProbeSession", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ArchitectureFamilyModelHash", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BaselineQuantDefinition", b => + { + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("ArchitectureFamily"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BenchmarkRun", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany() + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.CategoryBenchmark", "CategoryBenchmark") + .WithMany() + .HasForeignKey("CategoryBenchmarkId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("CategoryBenchmark"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.CategoryBenchmark", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany("CategorBenchmarks") + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiBenchmark"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ExecutionPlanProbeCache", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ImatrixDefinition", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiModelHash"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.LearnedBaselineTensorQuant", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany() + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.BaselineQuantDefinition", "BaselineQuantDefinition") + .WithMany() + .HasForeignKey("BaselineQuantDefinitionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("BaselineQuantDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.QuantizationRun", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany() + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.TensorGroupProfile", b => + { + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ArchitectureFamily"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmark", b => + { + b.Navigation("CategorBenchmarks"); + + b.Navigation("LearnedSources"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyInteractionRule", b => + { + b.Navigation("GroupStates"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyProbeSession", b => + { + b.Navigation("Observations"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/MQ.DB/Migrations/20260503222121_PredictionEngineAnomalyDetect.cs b/src/MQ.DB/Migrations/20260503222121_PredictionEngineAnomalyDetect.cs new file mode 100644 index 0000000..14e0bad --- /dev/null +++ b/src/MQ.DB/Migrations/20260503222121_PredictionEngineAnomalyDetect.cs @@ -0,0 +1,363 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace MQ.DB.Migrations +{ + /// + public partial class PredictionEngineAnomalyDetect : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "AnomalyInteractionRules", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + ArchitectureFamilyId = table.Column(type: "INTEGER", nullable: false), + TensorGroupProfileId = table.Column(type: "INTEGER", nullable: false), + AiModelHashId = table.Column(type: "INTEGER", nullable: false), + ImatrixDefinitionId = table.Column(type: "INTEGER", nullable: true), + BenchmarkCategory = table.Column(type: "INTEGER", nullable: false), + ReferenceQuantId = table.Column(type: "INTEGER", nullable: false), + ReferenceContextKey = table.Column(type: "TEXT", maxLength: 512, nullable: false), + ReferenceEffectiveGroupsJson = table.Column(type: "TEXT", nullable: false), + CandidateEffectiveGroupsJson = table.Column(type: "TEXT", nullable: false), + InactiveGroupsJson = table.Column(type: "TEXT", nullable: false), + FullTensorConfigKey = table.Column(type: "TEXT", maxLength: 128, nullable: false), + RuleType = table.Column(type: "TEXT", maxLength: 64, nullable: false), + RuleDirection = table.Column(type: "TEXT", maxLength: 64, nullable: false), + RuleStatus = table.Column(type: "TEXT", maxLength: 64, nullable: false), + MovementClassification = table.Column(type: "TEXT", maxLength: 64, nullable: false), + GroupSetHash = table.Column(type: "TEXT", maxLength: 128, nullable: false), + GroupCount = table.Column(type: "INTEGER", nullable: false), + MeanActualGainVsTwin = table.Column(type: "REAL", nullable: false), + BestActualGainVsTwin = table.Column(type: "REAL", nullable: false), + MeanPredictionSpaceGap = table.Column(type: "REAL", nullable: false), + BestPredictionSpaceGap = table.Column(type: "REAL", nullable: false), + AppliedPredictionSpaceAdjustmentKld = table.Column(type: "REAL", nullable: false), + EvidenceCount = table.Column(type: "INTEGER", nullable: false), + Confidence = table.Column(type: "REAL", nullable: false), + ShrinkFactor = table.Column(type: "REAL", nullable: false), + Status = table.Column(type: "TEXT", maxLength: 64, nullable: false), + CreatedUtc = table.Column(type: "TEXT", nullable: false), + UpdatedUtc = table.Column(type: "TEXT", nullable: false), + MetadataJson = table.Column(type: "TEXT", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_AnomalyInteractionRules", x => x.Id); + table.ForeignKey( + name: "FK_AnomalyInteractionRules_AiModelHashes_AiModelHashId", + column: x => x.AiModelHashId, + principalTable: "AiModelHashes", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_AnomalyInteractionRules_ArchitectureFamilies_ArchitectureFamilyId", + column: x => x.ArchitectureFamilyId, + principalTable: "ArchitectureFamilies", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_AnomalyInteractionRules_ImatrixDefinitions_ImatrixDefinitionId", + column: x => x.ImatrixDefinitionId, + principalTable: "ImatrixDefinitions", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_AnomalyInteractionRules_TensorGroupProfiles_TensorGroupProfileId", + column: x => x.TensorGroupProfileId, + principalTable: "TensorGroupProfiles", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + }); + + migrationBuilder.CreateTable( + name: "AnomalyProbeSessions", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + ArchitectureFamilyId = table.Column(type: "INTEGER", nullable: false), + TensorGroupProfileId = table.Column(type: "INTEGER", nullable: false), + AiModelHashId = table.Column(type: "INTEGER", nullable: false), + ImatrixDefinitionId = table.Column(type: "INTEGER", nullable: true), + BenchmarkCategory = table.Column(type: "INTEGER", nullable: false), + StartedUtc = table.Column(type: "TEXT", nullable: false), + CompletedUtc = table.Column(type: "TEXT", nullable: true), + SourceRunLabel = table.Column(type: "TEXT", maxLength: 256, nullable: false), + ConfigJson = table.Column(type: "TEXT", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_AnomalyProbeSessions", x => x.Id); + table.ForeignKey( + name: "FK_AnomalyProbeSessions_AiModelHashes_AiModelHashId", + column: x => x.AiModelHashId, + principalTable: "AiModelHashes", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_AnomalyProbeSessions_ArchitectureFamilies_ArchitectureFamilyId", + column: x => x.ArchitectureFamilyId, + principalTable: "ArchitectureFamilies", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_AnomalyProbeSessions_ImatrixDefinitions_ImatrixDefinitionId", + column: x => x.ImatrixDefinitionId, + principalTable: "ImatrixDefinitions", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_AnomalyProbeSessions_TensorGroupProfiles_TensorGroupProfileId", + column: x => x.TensorGroupProfileId, + principalTable: "TensorGroupProfiles", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + }); + + migrationBuilder.CreateTable( + name: "AnomalyInteractionRuleGroupStates", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + RuleId = table.Column(type: "TEXT", nullable: false), + TensorGroupId = table.Column(type: "INTEGER", nullable: false), + CandidateQuantId = table.Column(type: "INTEGER", nullable: false), + ReferenceQuantId = table.Column(type: "INTEGER", nullable: false), + Movement = table.Column(type: "TEXT", maxLength: 64, nullable: false), + SortOrder = table.Column(type: "INTEGER", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_AnomalyInteractionRuleGroupStates", x => x.Id); + table.ForeignKey( + name: "FK_AnomalyInteractionRuleGroupStates_AnomalyInteractionRules_RuleId", + column: x => x.RuleId, + principalTable: "AnomalyInteractionRules", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "AnomalyProbeObservations", + columns: table => new + { + Id = table.Column(type: "TEXT", nullable: false), + SessionId = table.Column(type: "TEXT", nullable: false), + ArchitectureFamilyId = table.Column(type: "INTEGER", nullable: false), + TensorGroupProfileId = table.Column(type: "INTEGER", nullable: false), + AiModelHashId = table.Column(type: "INTEGER", nullable: false), + ImatrixDefinitionId = table.Column(type: "INTEGER", nullable: true), + BenchmarkCategory = table.Column(type: "INTEGER", nullable: false), + ReferenceTensorComboId = table.Column(type: "TEXT", nullable: true), + ProbeTensorComboId = table.Column(type: "TEXT", nullable: true), + ProbeType = table.Column(type: "TEXT", maxLength: 64, nullable: false), + Classification = table.Column(type: "TEXT", maxLength: 64, nullable: false), + HypothesisLabel = table.Column(type: "TEXT", maxLength: 128, nullable: false), + MovementClassification = table.Column(type: "TEXT", maxLength: 64, nullable: false), + ChangedGroupSetHash = table.Column(type: "TEXT", maxLength: 128, nullable: false), + ChangedGroupsJson = table.Column(type: "TEXT", nullable: false), + CandidateQuantsJson = table.Column(type: "TEXT", nullable: false), + ReferenceEffectiveGroupsJson = table.Column(type: "TEXT", nullable: false), + CandidateEffectiveGroupsJson = table.Column(type: "TEXT", nullable: false), + InactiveGroupsJson = table.Column(type: "TEXT", nullable: false), + ReferenceTensorConfigKey = table.Column(type: "TEXT", maxLength: 128, nullable: false), + ProbeTensorConfigKey = table.Column(type: "TEXT", maxLength: 128, nullable: false), + IsContextualAnomalyProbe = table.Column(type: "INTEGER", nullable: false), + OldBf16Isolation = table.Column(type: "INTEGER", nullable: false), + AllActiveGroupsExplicit = table.Column(type: "INTEGER", nullable: false), + ReferenceQuantId = table.Column(type: "INTEGER", nullable: false), + ActualKld = table.Column(type: "REAL", nullable: false), + PredictedKld = table.Column(type: "REAL", nullable: false), + ReferenceActualKld = table.Column(type: "REAL", nullable: false), + ReferencePredictedKld = table.Column(type: "REAL", nullable: false), + ActualGainVsTwin = table.Column(type: "REAL", nullable: false), + PredictionSpaceGapVsTwin = table.Column(type: "REAL", nullable: false), + SizeSavingsBytes = table.Column(type: "INTEGER", nullable: false), + UpgradeCount = table.Column(type: "INTEGER", nullable: false), + DowngradeCount = table.Column(type: "INTEGER", nullable: false), + SameCount = table.Column(type: "INTEGER", nullable: false), + UnknownCount = table.Column(type: "INTEGER", nullable: false), + NetBitDelta = table.Column(type: "INTEGER", nullable: false), + RuleDirection = table.Column(type: "TEXT", maxLength: 64, nullable: false), + Accepted = table.Column(type: "INTEGER", nullable: false), + FailureCode = table.Column(type: "TEXT", maxLength: 128, nullable: false), + Message = table.Column(type: "TEXT", maxLength: 4000, nullable: false), + CreatedUtc = table.Column(type: "TEXT", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_AnomalyProbeObservations", x => x.Id); + table.ForeignKey( + name: "FK_AnomalyProbeObservations_AiModelHashes_AiModelHashId", + column: x => x.AiModelHashId, + principalTable: "AiModelHashes", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_AnomalyProbeObservations_AnomalyProbeSessions_SessionId", + column: x => x.SessionId, + principalTable: "AnomalyProbeSessions", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_AnomalyProbeObservations_ArchitectureFamilies_ArchitectureFamilyId", + column: x => x.ArchitectureFamilyId, + principalTable: "ArchitectureFamilies", + principalColumn: "Id", + onDelete: ReferentialAction.Cascade); + table.ForeignKey( + name: "FK_AnomalyProbeObservations_ImatrixDefinitions_ImatrixDefinitionId", + column: x => x.ImatrixDefinitionId, + principalTable: "ImatrixDefinitions", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_AnomalyProbeObservations_TensorCombos_ProbeTensorComboId", + column: x => x.ProbeTensorComboId, + principalTable: "TensorCombos", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_AnomalyProbeObservations_TensorCombos_ReferenceTensorComboId", + column: x => x.ReferenceTensorComboId, + principalTable: "TensorCombos", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + table.ForeignKey( + name: "FK_AnomalyProbeObservations_TensorGroupProfiles_TensorGroupProfileId", + column: x => x.TensorGroupProfileId, + principalTable: "TensorGroupProfiles", + principalColumn: "Id", + onDelete: ReferentialAction.Restrict); + }); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyInteractionRuleGroupStates_RuleId_TensorGroupId", + table: "AnomalyInteractionRuleGroupStates", + columns: new[] { "RuleId", "TensorGroupId" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyInteractionRules_AiModelHashId", + table: "AnomalyInteractionRules", + column: "AiModelHashId"); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyInteractionRules_ArchitectureFamilyId_TensorGroupProfileId_AiModelHashId_ImatrixDefinitionId_BenchmarkCategory_ReferenceQuantId_GroupSetHash_RuleDirection", + table: "AnomalyInteractionRules", + columns: new[] { "ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "ReferenceQuantId", "GroupSetHash", "RuleDirection" }, + unique: true); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyInteractionRules_ArchitectureFamilyId_TensorGroupProfileId_AiModelHashId_ImatrixDefinitionId_BenchmarkCategory_RuleDirection_RuleStatus", + table: "AnomalyInteractionRules", + columns: new[] { "ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "RuleDirection", "RuleStatus" }); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyInteractionRules_FullTensorConfigKey", + table: "AnomalyInteractionRules", + column: "FullTensorConfigKey"); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyInteractionRules_ImatrixDefinitionId", + table: "AnomalyInteractionRules", + column: "ImatrixDefinitionId"); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyInteractionRules_TensorGroupProfileId", + table: "AnomalyInteractionRules", + column: "TensorGroupProfileId"); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyProbeObservations_AiModelHashId", + table: "AnomalyProbeObservations", + column: "AiModelHashId"); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyProbeObservations_ArchitectureFamilyId_TensorGroupProfileId_AiModelHashId_ImatrixDefinitionId_BenchmarkCategory_ChangedGroupSetHash", + table: "AnomalyProbeObservations", + columns: new[] { "ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "ChangedGroupSetHash" }); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyProbeObservations_ArchitectureFamilyId_TensorGroupProfileId_AiModelHashId_ImatrixDefinitionId_BenchmarkCategory_ReferenceTensorComboId_ProbeTensorComboId_ProbeType", + table: "AnomalyProbeObservations", + columns: new[] { "ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "ReferenceTensorComboId", "ProbeTensorComboId", "ProbeType" }); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyProbeObservations_ImatrixDefinitionId", + table: "AnomalyProbeObservations", + column: "ImatrixDefinitionId"); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyProbeObservations_ProbeTensorComboId", + table: "AnomalyProbeObservations", + column: "ProbeTensorComboId"); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyProbeObservations_ProbeTensorConfigKey", + table: "AnomalyProbeObservations", + column: "ProbeTensorConfigKey"); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyProbeObservations_ReferenceTensorComboId", + table: "AnomalyProbeObservations", + column: "ReferenceTensorComboId"); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyProbeObservations_ReferenceTensorConfigKey", + table: "AnomalyProbeObservations", + column: "ReferenceTensorConfigKey"); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyProbeObservations_SessionId", + table: "AnomalyProbeObservations", + column: "SessionId"); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyProbeObservations_TensorGroupProfileId", + table: "AnomalyProbeObservations", + column: "TensorGroupProfileId"); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyProbeSessions_AiModelHashId", + table: "AnomalyProbeSessions", + column: "AiModelHashId"); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyProbeSessions_ArchitectureFamilyId_TensorGroupProfileId_AiModelHashId_ImatrixDefinitionId_BenchmarkCategory_StartedUtc", + table: "AnomalyProbeSessions", + columns: new[] { "ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "StartedUtc" }); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyProbeSessions_ImatrixDefinitionId", + table: "AnomalyProbeSessions", + column: "ImatrixDefinitionId"); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyProbeSessions_TensorGroupProfileId", + table: "AnomalyProbeSessions", + column: "TensorGroupProfileId"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "AnomalyInteractionRuleGroupStates"); + + migrationBuilder.DropTable( + name: "AnomalyProbeObservations"); + + migrationBuilder.DropTable( + name: "AnomalyInteractionRules"); + + migrationBuilder.DropTable( + name: "AnomalyProbeSessions"); + } + } +} diff --git a/src/MQ.DB/Migrations/20260503224957_PredictionEngineAnomalyDetect2.Designer.cs b/src/MQ.DB/Migrations/20260503224957_PredictionEngineAnomalyDetect2.Designer.cs new file mode 100644 index 0000000..bc48555 --- /dev/null +++ b/src/MQ.DB/Migrations/20260503224957_PredictionEngineAnomalyDetect2.Designer.cs @@ -0,0 +1,1738 @@ +// +using System; +using MQ.DB.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace MQ.DB.Migrations +{ + [DbContext(typeof(MagicQuantContext))] + [Migration("20260503224957_PredictionEngineAnomalyDetect2")] + partial class PredictionEngineAnomalyDetect2 + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "10.0.1"); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmark", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("Ngl") + .HasColumnType("INTEGER"); + + b.Property("SizeBytes") + .HasColumnType("INTEGER"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("TokensPerSecond") + .HasColumnType("REAL"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "TensorComboId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "TensorComboId") + .IsUnique(); + + b.ToTable("AiBenchmarks"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmarkLearnedSource", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BaselineCanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("BaselineQuantDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("SourceLearningBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupId") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("BaselineQuantDefinitionId"); + + b.HasIndex("SourceLearningBenchmarkId"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("AiBenchmarkId", "TensorGroupId") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "BaselineQuantDefinitionId"); + + b.ToTable("AiBenchmarkLearnedSources"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiModelHash", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("UniqueHash") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("UniqueHash"); + + b.ToTable("AiModelHashes"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyInteractionRule", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("AppliedPredictionSpaceAdjustmentKld") + .HasColumnType("REAL"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BenchmarkCategory") + .HasColumnType("INTEGER"); + + b.Property("BestActualGainVsTwin") + .HasColumnType("REAL"); + + b.Property("BestPredictionSpaceGap") + .HasColumnType("REAL"); + + b.Property("CandidateDisplayName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("CandidateEffectiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CandidateInternalName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("Confidence") + .HasColumnType("REAL"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("EvidenceCount") + .HasColumnType("INTEGER"); + + b.Property("FullTensorConfigKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("GroupCount") + .HasColumnType("INTEGER"); + + b.Property("GroupSetHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("InactiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("MeanActualGainVsTwin") + .HasColumnType("REAL"); + + b.Property("MeanPredictionSpaceGap") + .HasColumnType("REAL"); + + b.Property("MetadataJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("MovementClassification") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("ReferenceContextKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ReferenceDisplayName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ReferenceEffectiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ReferenceInternalName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ReferenceQuantId") + .HasColumnType("INTEGER"); + + b.Property("RuleDirection") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("RuleStatus") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("RuleType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("ShrinkFactor") + .HasColumnType("REAL"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("UpdatedUtc") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("FullTensorConfigKey"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "RuleDirection", "RuleStatus"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "ReferenceQuantId", "ReferenceContextKey", "GroupSetHash", "RuleDirection") + .IsUnique(); + + b.ToTable("AnomalyInteractionRules"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyInteractionRuleGroupState", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("CandidateQuantId") + .HasColumnType("INTEGER"); + + b.Property("Movement") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("ReferenceQuantId") + .HasColumnType("INTEGER"); + + b.Property("RuleId") + .HasColumnType("TEXT"); + + b.Property("SortOrder") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("RuleId", "TensorGroupId") + .IsUnique(); + + b.ToTable("AnomalyInteractionRuleGroupStates"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyProbeObservation", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("Accepted") + .HasColumnType("INTEGER"); + + b.Property("ActualGainVsTwin") + .HasColumnType("REAL"); + + b.Property("ActualKld") + .HasColumnType("REAL"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("AllActiveGroupsExplicit") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BenchmarkCategory") + .HasColumnType("INTEGER"); + + b.Property("CandidateEffectiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CandidateQuantsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ChangedGroupSetHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ChangedGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Classification") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("DowngradeCount") + .HasColumnType("INTEGER"); + + b.Property("FailureCode") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("HypothesisLabel") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("InactiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("IsContextualAnomalyProbe") + .HasColumnType("INTEGER"); + + b.Property("Message") + .IsRequired() + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("MovementClassification") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("NetBitDelta") + .HasColumnType("INTEGER"); + + b.Property("OldBf16Isolation") + .HasColumnType("INTEGER"); + + b.Property("PredictedKld") + .HasColumnType("REAL"); + + b.Property("PredictionSpaceGapVsTwin") + .HasColumnType("REAL"); + + b.Property("ProbeDisplayName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ProbeInternalName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ProbePlanClass") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("ProbeTensorComboId") + .HasColumnType("TEXT"); + + b.Property("ProbeTensorConfigKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ProbeType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("ReferenceActualKld") + .HasColumnType("REAL"); + + b.Property("ReferenceDisplayName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ReferenceEffectiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ReferenceInternalName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ReferencePredictedKld") + .HasColumnType("REAL"); + + b.Property("ReferenceQuantId") + .HasColumnType("INTEGER"); + + b.Property("ReferenceTensorComboId") + .HasColumnType("TEXT"); + + b.Property("ReferenceTensorConfigKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("RuleDirection") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("SameCount") + .HasColumnType("INTEGER"); + + b.Property("SeedClass") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("SeedPriority") + .HasColumnType("INTEGER"); + + b.Property("SessionId") + .HasColumnType("TEXT"); + + b.Property("SizeSavingsBytes") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("UnknownCount") + .HasColumnType("INTEGER"); + + b.Property("UpgradeCount") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("ProbeTensorComboId"); + + b.HasIndex("ProbeTensorConfigKey"); + + b.HasIndex("ReferenceTensorComboId"); + + b.HasIndex("ReferenceTensorConfigKey"); + + b.HasIndex("SessionId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "ChangedGroupSetHash"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "ReferenceTensorComboId", "ProbeTensorComboId", "ProbeType"); + + b.ToTable("AnomalyProbeObservations"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyProbeSession", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BenchmarkCategory") + .HasColumnType("INTEGER"); + + b.Property("CompletedUtc") + .HasColumnType("TEXT"); + + b.Property("ConfigJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("SourceRunLabel") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("StartedUtc") + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "StartedUtc"); + + b.ToTable("AnomalyProbeSessions"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ArchitectureFamily", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("NormalizedName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("TensorCount") + .HasColumnType("INTEGER"); + + b.Property("TensorSignatureHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique(); + + b.HasIndex("TensorSignatureHash", "TensorCount"); + + b.ToTable("ArchitectureFamilies"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ArchitectureFamilyModelHash", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("IsCanonical") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "AiModelHashId") + .IsUnique(); + + b.ToTable("ArchitectureFamilyModelHashes"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BaselineQuantDefinition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BaselineFamily") + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("BaselineName") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("BitRange") + .HasColumnType("INTEGER"); + + b.Property("CanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("DefaultTensorSchemeId") + .HasColumnType("INTEGER"); + + b.Property("DefaultTensorSchemeName") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ExplicitCandidateSortOrder") + .HasColumnType("INTEGER"); + + b.Property("FirstSeenUtc") + .HasColumnType("TEXT"); + + b.Property("IsActiveInCurrentConfig") + .HasColumnType("INTEGER"); + + b.Property("IsCombinationCarrierCandidate") + .HasColumnType("INTEGER"); + + b.Property("IsCustomBaseline") + .HasColumnType("INTEGER"); + + b.Property("IsExplicitGroupCombinationCandidate") + .HasColumnType("INTEGER"); + + b.Property("IsLearningBaseline") + .HasColumnType("INTEGER"); + + b.Property("LastSeenUtc") + .HasColumnType("TEXT"); + + b.Property("LastUpdatedUtc") + .HasColumnType("TEXT"); + + b.Property("NormalizedCanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("NormalizedSourceFileName") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("NormalizedSourceRepository") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("QuantizeBaseArgumentName") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("RequiresImatrix") + .HasColumnType("INTEGER"); + + b.Property("RuntimeBaselineId") + .HasColumnType("INTEGER"); + + b.Property("ShortSourceName") + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("SourceFileName") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("SourceKind") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("SourceOwner") + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("SourceRepository") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("IsActiveInCurrentConfig"); + + b.HasIndex("ArchitectureFamilyId", "NormalizedCanonicalKey") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "RuntimeBaselineId") + .IsUnique(); + + b.HasIndex("RuntimeBaselineId", "ArchitectureFamilyId"); + + b.HasIndex("ArchitectureFamilyId", "NormalizedSourceRepository", "NormalizedSourceFileName") + .IsUnique(); + + b.ToTable("BaselineQuantDefinitions"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BenchmarkRun", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("Category") + .HasColumnType("INTEGER"); + + b.Property("CategoryBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("CompletedUtc") + .HasColumnType("TEXT"); + + b.Property("DurationMs") + .HasColumnType("INTEGER"); + + b.Property("Error") + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("StartedUtc") + .HasColumnType("TEXT"); + + b.Property("Succeeded") + .HasColumnType("INTEGER"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ArchitectureFamilyId"); + + b.HasIndex("CategoryBenchmarkId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("StartedUtc"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("AiBenchmarkId", "Category"); + + b.ToTable("BenchmarkRuns"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.CategoryBenchmark", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("Category") + .HasColumnType("INTEGER"); + + b.Property("Kld") + .HasColumnType("REAL"); + + b.Property("Ppl") + .HasColumnType("REAL"); + + b.Property("PplError") + .HasColumnType("REAL"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.ToTable("CategoryBenchmark"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ExecutionPlanProbeCache", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("DiscoveryTokenTarget") + .HasColumnType("INTEGER"); + + b.Property("GpuMemoryLimitsJson") + .IsRequired() + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("GroupSize") + .HasColumnType("INTEGER"); + + b.Property("HardwareFingerprint") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("MaxCandidateNgl") + .HasColumnType("INTEGER"); + + b.Property("NativeModelSizeBytes") + .HasColumnType("INTEGER"); + + b.Property("NativeQuantizationKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("NativeStableNgl") + .HasColumnType("INTEGER"); + + b.Property("ProbeSchemaVersion") + .HasColumnType("INTEGER"); + + b.Property("Q8ModelSizeBytes") + .HasColumnType("INTEGER"); + + b.Property("Q8StableNgl") + .HasColumnType("INTEGER"); + + b.Property("QuantizationKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("QuantizedModelFingerprint") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("TEXT"); + + b.Property("SlotsJson") + .IsRequired() + .HasMaxLength(8000) + .HasColumnType("TEXT"); + + b.Property("StaticNgl") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("TensorSplitJson") + .IsRequired() + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("UpdatedUtc") + .HasColumnType("TEXT"); + + b.Property("UsesGpu") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ArchitectureFamilyId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "HardwareFingerprint", "QuantizedModelFingerprint", "QuantizationKey", "DiscoveryTokenTarget") + .IsUnique(); + + b.ToTable("ExecutionPlanProbeCaches"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ImatrixDefinition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("BuildFingerprint") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("CanonicalPath") + .HasMaxLength(2048) + .HasColumnType("TEXT"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("IdentityHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("MetadataJson") + .HasMaxLength(8000) + .HasColumnType("TEXT"); + + b.Property("SourceKind") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("TokenCount") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId", "IdentityHash") + .IsUnique(); + + b.ToTable("ImatrixDefinitions"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.LearnedBaselineTensorQuant", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BaselineCanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("BaselineQuantDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("BaselineQuantId") + .HasColumnType("INTEGER"); + + b.Property("BaselineSourceFileName") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("BaselineSourceKind") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("BaselineSourceRepository") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("FinalQuantType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("TEXT"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupId") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("TensorName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("TensorWeightSchemeId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("BaselineQuantDefinitionId"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "BaselineQuantDefinitionId", "TensorGroupId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "BaselineQuantDefinitionId", "TensorWeightSchemeId", "TensorName") + .IsUnique(); + + b.ToTable("LearnedBaselineTensorQuants"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.QuantizationRun", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CompletedUtc") + .HasColumnType("TEXT"); + + b.Property("DurationMs") + .HasColumnType("INTEGER"); + + b.Property("Error") + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("OutputModelPath") + .HasMaxLength(2048) + .HasColumnType("TEXT"); + + b.Property("StartedUtc") + .HasColumnType("TEXT"); + + b.Property("Succeeded") + .HasColumnType("INTEGER"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ArchitectureFamilyId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("StartedUtc"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.ToTable("QuantizationRuns"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.TensorCombo", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AttnKV") + .HasColumnType("INTEGER"); + + b.Property("AttnOutput") + .HasColumnType("INTEGER"); + + b.Property("AttnQ") + .HasColumnType("INTEGER"); + + b.Property("BaseQuant") + .HasColumnType("INTEGER"); + + b.Property("Embeddings") + .HasColumnType("INTEGER"); + + b.Property("FfnDown") + .HasColumnType("INTEGER"); + + b.Property("FfnUpGate") + .HasColumnType("INTEGER"); + + b.Property("LmHead") + .HasColumnType("INTEGER"); + + b.Property("MoeExperts") + .HasColumnType("INTEGER"); + + b.Property("MoeRouter") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("BaseQuant", "Embeddings", "LmHead", "AttnQ", "AttnKV", "AttnOutput", "FfnUpGate", "FfnDown", "MoeExperts", "MoeRouter") + .IsUnique(); + + b.ToTable("TensorCombos"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.TensorGroupProfile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("FingerprintHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("IsActive") + .HasColumnType("INTEGER"); + + b.Property("SnapshotJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("ArchitectureFamilyId", "FingerprintHash") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "IsActive"); + + b.ToTable("TensorGroupProfiles"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmark", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmarkLearnedSource", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany("LearnedSources") + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.BaselineQuantDefinition", "BaselineQuantDefinition") + .WithMany() + .HasForeignKey("BaselineQuantDefinitionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "SourceLearningBenchmark") + .WithMany() + .HasForeignKey("SourceLearningBenchmarkId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("BaselineQuantDefinition"); + + b.Navigation("SourceLearningBenchmark"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyInteractionRule", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyInteractionRuleGroupState", b => + { + b.HasOne("MQ.DB.Models.DbModels.AnomalyInteractionRule", "Rule") + .WithMany("GroupStates") + .HasForeignKey("RuleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Rule"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyProbeObservation", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "ProbeTensorCombo") + .WithMany() + .HasForeignKey("ProbeTensorComboId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "ReferenceTensorCombo") + .WithMany() + .HasForeignKey("ReferenceTensorComboId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.AnomalyProbeSession", "Session") + .WithMany("Observations") + .HasForeignKey("SessionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("ProbeTensorCombo"); + + b.Navigation("ReferenceTensorCombo"); + + b.Navigation("Session"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyProbeSession", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ArchitectureFamilyModelHash", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BaselineQuantDefinition", b => + { + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("ArchitectureFamily"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BenchmarkRun", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany() + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.CategoryBenchmark", "CategoryBenchmark") + .WithMany() + .HasForeignKey("CategoryBenchmarkId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("CategoryBenchmark"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.CategoryBenchmark", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany("CategorBenchmarks") + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiBenchmark"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ExecutionPlanProbeCache", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ImatrixDefinition", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiModelHash"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.LearnedBaselineTensorQuant", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany() + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.BaselineQuantDefinition", "BaselineQuantDefinition") + .WithMany() + .HasForeignKey("BaselineQuantDefinitionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("BaselineQuantDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.QuantizationRun", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany() + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.TensorGroupProfile", b => + { + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ArchitectureFamily"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmark", b => + { + b.Navigation("CategorBenchmarks"); + + b.Navigation("LearnedSources"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyInteractionRule", b => + { + b.Navigation("GroupStates"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyProbeSession", b => + { + b.Navigation("Observations"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/MQ.DB/Migrations/20260503224957_PredictionEngineAnomalyDetect2.cs b/src/MQ.DB/Migrations/20260503224957_PredictionEngineAnomalyDetect2.cs new file mode 100644 index 0000000..e705c99 --- /dev/null +++ b/src/MQ.DB/Migrations/20260503224957_PredictionEngineAnomalyDetect2.cs @@ -0,0 +1,169 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace MQ.DB.Migrations +{ + /// + public partial class PredictionEngineAnomalyDetect2 : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_AnomalyInteractionRules_ArchitectureFamilyId_TensorGroupProfileId_AiModelHashId_ImatrixDefinitionId_BenchmarkCategory_ReferenceQuantId_GroupSetHash_RuleDirection", + table: "AnomalyInteractionRules"); + + migrationBuilder.AddColumn( + name: "ProbeDisplayName", + table: "AnomalyProbeObservations", + type: "TEXT", + maxLength: 512, + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "ProbeInternalName", + table: "AnomalyProbeObservations", + type: "TEXT", + maxLength: 512, + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "ProbePlanClass", + table: "AnomalyProbeObservations", + type: "TEXT", + maxLength: 64, + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "ReferenceDisplayName", + table: "AnomalyProbeObservations", + type: "TEXT", + maxLength: 512, + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "ReferenceInternalName", + table: "AnomalyProbeObservations", + type: "TEXT", + maxLength: 512, + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "SeedClass", + table: "AnomalyProbeObservations", + type: "TEXT", + maxLength: 64, + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "SeedPriority", + table: "AnomalyProbeObservations", + type: "INTEGER", + nullable: false, + defaultValue: 0); + + migrationBuilder.AddColumn( + name: "CandidateDisplayName", + table: "AnomalyInteractionRules", + type: "TEXT", + maxLength: 512, + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "CandidateInternalName", + table: "AnomalyInteractionRules", + type: "TEXT", + maxLength: 512, + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "ReferenceDisplayName", + table: "AnomalyInteractionRules", + type: "TEXT", + maxLength: 512, + nullable: false, + defaultValue: ""); + + migrationBuilder.AddColumn( + name: "ReferenceInternalName", + table: "AnomalyInteractionRules", + type: "TEXT", + maxLength: 512, + nullable: false, + defaultValue: ""); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyInteractionRules_ArchitectureFamilyId_TensorGroupProfileId_AiModelHashId_ImatrixDefinitionId_BenchmarkCategory_ReferenceQuantId_ReferenceContextKey_GroupSetHash_RuleDirection", + table: "AnomalyInteractionRules", + columns: new[] { "ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "ReferenceQuantId", "ReferenceContextKey", "GroupSetHash", "RuleDirection" }, + unique: true); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_AnomalyInteractionRules_ArchitectureFamilyId_TensorGroupProfileId_AiModelHashId_ImatrixDefinitionId_BenchmarkCategory_ReferenceQuantId_ReferenceContextKey_GroupSetHash_RuleDirection", + table: "AnomalyInteractionRules"); + + migrationBuilder.DropColumn( + name: "ProbeDisplayName", + table: "AnomalyProbeObservations"); + + migrationBuilder.DropColumn( + name: "ProbeInternalName", + table: "AnomalyProbeObservations"); + + migrationBuilder.DropColumn( + name: "ProbePlanClass", + table: "AnomalyProbeObservations"); + + migrationBuilder.DropColumn( + name: "ReferenceDisplayName", + table: "AnomalyProbeObservations"); + + migrationBuilder.DropColumn( + name: "ReferenceInternalName", + table: "AnomalyProbeObservations"); + + migrationBuilder.DropColumn( + name: "SeedClass", + table: "AnomalyProbeObservations"); + + migrationBuilder.DropColumn( + name: "SeedPriority", + table: "AnomalyProbeObservations"); + + migrationBuilder.DropColumn( + name: "CandidateDisplayName", + table: "AnomalyInteractionRules"); + + migrationBuilder.DropColumn( + name: "CandidateInternalName", + table: "AnomalyInteractionRules"); + + migrationBuilder.DropColumn( + name: "ReferenceDisplayName", + table: "AnomalyInteractionRules"); + + migrationBuilder.DropColumn( + name: "ReferenceInternalName", + table: "AnomalyInteractionRules"); + + migrationBuilder.CreateIndex( + name: "IX_AnomalyInteractionRules_ArchitectureFamilyId_TensorGroupProfileId_AiModelHashId_ImatrixDefinitionId_BenchmarkCategory_ReferenceQuantId_GroupSetHash_RuleDirection", + table: "AnomalyInteractionRules", + columns: new[] { "ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "ReferenceQuantId", "GroupSetHash", "RuleDirection" }, + unique: true); + } + } +} diff --git a/src/MQ.DB/Migrations/MagicQuantContextModelSnapshot.cs b/src/MQ.DB/Migrations/MagicQuantContextModelSnapshot.cs new file mode 100644 index 0000000..421ccef --- /dev/null +++ b/src/MQ.DB/Migrations/MagicQuantContextModelSnapshot.cs @@ -0,0 +1,1735 @@ +// +using System; +using MQ.DB.Data; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace MQ.DB.Migrations +{ + [DbContext(typeof(MagicQuantContext))] + partial class MagicQuantContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "10.0.1"); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmark", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("Ngl") + .HasColumnType("INTEGER"); + + b.Property("SizeBytes") + .HasColumnType("INTEGER"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("TokensPerSecond") + .HasColumnType("REAL"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "TensorComboId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "TensorComboId") + .IsUnique(); + + b.ToTable("AiBenchmarks"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmarkLearnedSource", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BaselineCanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("BaselineQuantDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("SourceLearningBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupId") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("BaselineQuantDefinitionId"); + + b.HasIndex("SourceLearningBenchmarkId"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("AiBenchmarkId", "TensorGroupId") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "BaselineQuantDefinitionId"); + + b.ToTable("AiBenchmarkLearnedSources"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiModelHash", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("UniqueHash") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("UniqueHash"); + + b.ToTable("AiModelHashes"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyInteractionRule", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("AppliedPredictionSpaceAdjustmentKld") + .HasColumnType("REAL"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BenchmarkCategory") + .HasColumnType("INTEGER"); + + b.Property("BestActualGainVsTwin") + .HasColumnType("REAL"); + + b.Property("BestPredictionSpaceGap") + .HasColumnType("REAL"); + + b.Property("CandidateDisplayName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("CandidateEffectiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CandidateInternalName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("Confidence") + .HasColumnType("REAL"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("EvidenceCount") + .HasColumnType("INTEGER"); + + b.Property("FullTensorConfigKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("GroupCount") + .HasColumnType("INTEGER"); + + b.Property("GroupSetHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("InactiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("MeanActualGainVsTwin") + .HasColumnType("REAL"); + + b.Property("MeanPredictionSpaceGap") + .HasColumnType("REAL"); + + b.Property("MetadataJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("MovementClassification") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("ReferenceContextKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ReferenceDisplayName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ReferenceEffectiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ReferenceInternalName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ReferenceQuantId") + .HasColumnType("INTEGER"); + + b.Property("RuleDirection") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("RuleStatus") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("RuleType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("ShrinkFactor") + .HasColumnType("REAL"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("UpdatedUtc") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("FullTensorConfigKey"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "RuleDirection", "RuleStatus"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "ReferenceQuantId", "ReferenceContextKey", "GroupSetHash", "RuleDirection") + .IsUnique(); + + b.ToTable("AnomalyInteractionRules"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyInteractionRuleGroupState", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("CandidateQuantId") + .HasColumnType("INTEGER"); + + b.Property("Movement") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("ReferenceQuantId") + .HasColumnType("INTEGER"); + + b.Property("RuleId") + .HasColumnType("TEXT"); + + b.Property("SortOrder") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("RuleId", "TensorGroupId") + .IsUnique(); + + b.ToTable("AnomalyInteractionRuleGroupStates"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyProbeObservation", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("Accepted") + .HasColumnType("INTEGER"); + + b.Property("ActualGainVsTwin") + .HasColumnType("REAL"); + + b.Property("ActualKld") + .HasColumnType("REAL"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("AllActiveGroupsExplicit") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BenchmarkCategory") + .HasColumnType("INTEGER"); + + b.Property("CandidateEffectiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CandidateQuantsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ChangedGroupSetHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ChangedGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("Classification") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("DowngradeCount") + .HasColumnType("INTEGER"); + + b.Property("FailureCode") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("HypothesisLabel") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("InactiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("IsContextualAnomalyProbe") + .HasColumnType("INTEGER"); + + b.Property("Message") + .IsRequired() + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("MovementClassification") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("NetBitDelta") + .HasColumnType("INTEGER"); + + b.Property("OldBf16Isolation") + .HasColumnType("INTEGER"); + + b.Property("PredictedKld") + .HasColumnType("REAL"); + + b.Property("PredictionSpaceGapVsTwin") + .HasColumnType("REAL"); + + b.Property("ProbeDisplayName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ProbeInternalName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ProbePlanClass") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("ProbeTensorComboId") + .HasColumnType("TEXT"); + + b.Property("ProbeTensorConfigKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ProbeType") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("ReferenceActualKld") + .HasColumnType("REAL"); + + b.Property("ReferenceDisplayName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ReferenceEffectiveGroupsJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ReferenceInternalName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("ReferencePredictedKld") + .HasColumnType("REAL"); + + b.Property("ReferenceQuantId") + .HasColumnType("INTEGER"); + + b.Property("ReferenceTensorComboId") + .HasColumnType("TEXT"); + + b.Property("ReferenceTensorConfigKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("RuleDirection") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("SameCount") + .HasColumnType("INTEGER"); + + b.Property("SeedClass") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("SeedPriority") + .HasColumnType("INTEGER"); + + b.Property("SessionId") + .HasColumnType("TEXT"); + + b.Property("SizeSavingsBytes") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("UnknownCount") + .HasColumnType("INTEGER"); + + b.Property("UpgradeCount") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("ProbeTensorComboId"); + + b.HasIndex("ProbeTensorConfigKey"); + + b.HasIndex("ReferenceTensorComboId"); + + b.HasIndex("ReferenceTensorConfigKey"); + + b.HasIndex("SessionId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "ChangedGroupSetHash"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "ReferenceTensorComboId", "ProbeTensorComboId", "ProbeType"); + + b.ToTable("AnomalyProbeObservations"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyProbeSession", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BenchmarkCategory") + .HasColumnType("INTEGER"); + + b.Property("CompletedUtc") + .HasColumnType("TEXT"); + + b.Property("ConfigJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("SourceRunLabel") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("StartedUtc") + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "BenchmarkCategory", "StartedUtc"); + + b.ToTable("AnomalyProbeSessions"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ArchitectureFamily", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("NormalizedName") + .IsRequired() + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("TensorCount") + .HasColumnType("INTEGER"); + + b.Property("TensorSignatureHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("NormalizedName") + .IsUnique(); + + b.HasIndex("TensorSignatureHash", "TensorCount"); + + b.ToTable("ArchitectureFamilies"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ArchitectureFamilyModelHash", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("IsCanonical") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "AiModelHashId") + .IsUnique(); + + b.ToTable("ArchitectureFamilyModelHashes"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BaselineQuantDefinition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BaselineFamily") + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("BaselineName") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("BitRange") + .HasColumnType("INTEGER"); + + b.Property("CanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("DefaultTensorSchemeId") + .HasColumnType("INTEGER"); + + b.Property("DefaultTensorSchemeName") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("DisplayName") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("ExplicitCandidateSortOrder") + .HasColumnType("INTEGER"); + + b.Property("FirstSeenUtc") + .HasColumnType("TEXT"); + + b.Property("IsActiveInCurrentConfig") + .HasColumnType("INTEGER"); + + b.Property("IsCombinationCarrierCandidate") + .HasColumnType("INTEGER"); + + b.Property("IsCustomBaseline") + .HasColumnType("INTEGER"); + + b.Property("IsExplicitGroupCombinationCandidate") + .HasColumnType("INTEGER"); + + b.Property("IsLearningBaseline") + .HasColumnType("INTEGER"); + + b.Property("LastSeenUtc") + .HasColumnType("TEXT"); + + b.Property("LastUpdatedUtc") + .HasColumnType("TEXT"); + + b.Property("NormalizedCanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("NormalizedSourceFileName") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("NormalizedSourceRepository") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("QuantizeBaseArgumentName") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("RequiresImatrix") + .HasColumnType("INTEGER"); + + b.Property("RuntimeBaselineId") + .HasColumnType("INTEGER"); + + b.Property("ShortSourceName") + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("SourceFileName") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("SourceKind") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("SourceOwner") + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("SourceRepository") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("IsActiveInCurrentConfig"); + + b.HasIndex("ArchitectureFamilyId", "NormalizedCanonicalKey") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "RuntimeBaselineId") + .IsUnique(); + + b.HasIndex("RuntimeBaselineId", "ArchitectureFamilyId"); + + b.HasIndex("ArchitectureFamilyId", "NormalizedSourceRepository", "NormalizedSourceFileName") + .IsUnique(); + + b.ToTable("BaselineQuantDefinitions"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BenchmarkRun", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("Category") + .HasColumnType("INTEGER"); + + b.Property("CategoryBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("CompletedUtc") + .HasColumnType("TEXT"); + + b.Property("DurationMs") + .HasColumnType("INTEGER"); + + b.Property("Error") + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("StartedUtc") + .HasColumnType("TEXT"); + + b.Property("Succeeded") + .HasColumnType("INTEGER"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ArchitectureFamilyId"); + + b.HasIndex("CategoryBenchmarkId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("StartedUtc"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("AiBenchmarkId", "Category"); + + b.ToTable("BenchmarkRuns"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.CategoryBenchmark", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("Category") + .HasColumnType("INTEGER"); + + b.Property("Kld") + .HasColumnType("REAL"); + + b.Property("Ppl") + .HasColumnType("REAL"); + + b.Property("PplError") + .HasColumnType("REAL"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.ToTable("CategoryBenchmark"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ExecutionPlanProbeCache", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("DiscoveryTokenTarget") + .HasColumnType("INTEGER"); + + b.Property("GpuMemoryLimitsJson") + .IsRequired() + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("GroupSize") + .HasColumnType("INTEGER"); + + b.Property("HardwareFingerprint") + .IsRequired() + .HasMaxLength(1024) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("MaxCandidateNgl") + .HasColumnType("INTEGER"); + + b.Property("NativeModelSizeBytes") + .HasColumnType("INTEGER"); + + b.Property("NativeQuantizationKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("NativeStableNgl") + .HasColumnType("INTEGER"); + + b.Property("ProbeSchemaVersion") + .HasColumnType("INTEGER"); + + b.Property("Q8ModelSizeBytes") + .HasColumnType("INTEGER"); + + b.Property("Q8StableNgl") + .HasColumnType("INTEGER"); + + b.Property("QuantizationKey") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("QuantizedModelFingerprint") + .IsRequired() + .HasMaxLength(2048) + .HasColumnType("TEXT"); + + b.Property("SlotsJson") + .IsRequired() + .HasMaxLength(8000) + .HasColumnType("TEXT"); + + b.Property("StaticNgl") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("TensorSplitJson") + .IsRequired() + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("UpdatedUtc") + .HasColumnType("TEXT"); + + b.Property("UsesGpu") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ArchitectureFamilyId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "AiModelHashId", "ImatrixDefinitionId", "HardwareFingerprint", "QuantizedModelFingerprint", "QuantizationKey", "DiscoveryTokenTarget") + .IsUnique(); + + b.ToTable("ExecutionPlanProbeCaches"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ImatrixDefinition", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("BuildFingerprint") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("CanonicalPath") + .HasMaxLength(2048) + .HasColumnType("TEXT"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("IdentityHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("MetadataJson") + .HasMaxLength(8000) + .HasColumnType("TEXT"); + + b.Property("SourceKind") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("TokenCount") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiModelHashId", "IdentityHash") + .IsUnique(); + + b.ToTable("ImatrixDefinitions"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.LearnedBaselineTensorQuant", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("BaselineCanonicalKey") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("BaselineQuantDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("BaselineQuantId") + .HasColumnType("INTEGER"); + + b.Property("BaselineSourceFileName") + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("BaselineSourceKind") + .IsRequired() + .HasMaxLength(64) + .HasColumnType("TEXT"); + + b.Property("BaselineSourceRepository") + .HasMaxLength(256) + .HasColumnType("TEXT"); + + b.Property("FinalQuantType") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("TEXT"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupId") + .HasColumnType("INTEGER"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.Property("TensorName") + .IsRequired() + .HasMaxLength(512) + .HasColumnType("TEXT"); + + b.Property("TensorWeightSchemeId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("BaselineQuantDefinitionId"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "BaselineQuantDefinitionId", "TensorGroupId"); + + b.HasIndex("ArchitectureFamilyId", "TensorGroupProfileId", "BaselineQuantDefinitionId", "TensorWeightSchemeId", "TensorName") + .IsUnique(); + + b.ToTable("LearnedBaselineTensorQuants"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.QuantizationRun", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AiBenchmarkId") + .HasColumnType("TEXT"); + + b.Property("AiModelHashId") + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CompletedUtc") + .HasColumnType("TEXT"); + + b.Property("DurationMs") + .HasColumnType("INTEGER"); + + b.Property("Error") + .HasMaxLength(4000) + .HasColumnType("TEXT"); + + b.Property("ImatrixDefinitionId") + .HasColumnType("INTEGER"); + + b.Property("OutputModelPath") + .HasMaxLength(2048) + .HasColumnType("TEXT"); + + b.Property("StartedUtc") + .HasColumnType("TEXT"); + + b.Property("Succeeded") + .HasColumnType("INTEGER"); + + b.Property("TensorComboId") + .HasColumnType("TEXT"); + + b.Property("TensorGroupProfileId") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("AiBenchmarkId"); + + b.HasIndex("AiModelHashId"); + + b.HasIndex("ArchitectureFamilyId"); + + b.HasIndex("ImatrixDefinitionId"); + + b.HasIndex("StartedUtc"); + + b.HasIndex("TensorComboId"); + + b.HasIndex("TensorGroupProfileId"); + + b.ToTable("QuantizationRuns"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.TensorCombo", b => + { + b.Property("Id") + .HasColumnType("TEXT"); + + b.Property("AttnKV") + .HasColumnType("INTEGER"); + + b.Property("AttnOutput") + .HasColumnType("INTEGER"); + + b.Property("AttnQ") + .HasColumnType("INTEGER"); + + b.Property("BaseQuant") + .HasColumnType("INTEGER"); + + b.Property("Embeddings") + .HasColumnType("INTEGER"); + + b.Property("FfnDown") + .HasColumnType("INTEGER"); + + b.Property("FfnUpGate") + .HasColumnType("INTEGER"); + + b.Property("LmHead") + .HasColumnType("INTEGER"); + + b.Property("MoeExperts") + .HasColumnType("INTEGER"); + + b.Property("MoeRouter") + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.HasIndex("BaseQuant", "Embeddings", "LmHead", "AttnQ", "AttnKV", "AttnOutput", "FfnUpGate", "FfnDown", "MoeExperts", "MoeRouter") + .IsUnique(); + + b.ToTable("TensorCombos"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.TensorGroupProfile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("ArchitectureFamilyId") + .HasColumnType("INTEGER"); + + b.Property("CreatedUtc") + .HasColumnType("TEXT"); + + b.Property("FingerprintHash") + .IsRequired() + .HasMaxLength(128) + .HasColumnType("TEXT"); + + b.Property("IsActive") + .HasColumnType("INTEGER"); + + b.Property("SnapshotJson") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("ArchitectureFamilyId", "FingerprintHash") + .IsUnique(); + + b.HasIndex("ArchitectureFamilyId", "IsActive"); + + b.ToTable("TensorGroupProfiles"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmark", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmarkLearnedSource", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany("LearnedSources") + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.BaselineQuantDefinition", "BaselineQuantDefinition") + .WithMany() + .HasForeignKey("BaselineQuantDefinitionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "SourceLearningBenchmark") + .WithMany() + .HasForeignKey("SourceLearningBenchmarkId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("BaselineQuantDefinition"); + + b.Navigation("SourceLearningBenchmark"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyInteractionRule", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyInteractionRuleGroupState", b => + { + b.HasOne("MQ.DB.Models.DbModels.AnomalyInteractionRule", "Rule") + .WithMany("GroupStates") + .HasForeignKey("RuleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("Rule"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyProbeObservation", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "ProbeTensorCombo") + .WithMany() + .HasForeignKey("ProbeTensorComboId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "ReferenceTensorCombo") + .WithMany() + .HasForeignKey("ReferenceTensorComboId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.AnomalyProbeSession", "Session") + .WithMany("Observations") + .HasForeignKey("SessionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("ProbeTensorCombo"); + + b.Navigation("ReferenceTensorCombo"); + + b.Navigation("Session"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyProbeSession", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ArchitectureFamilyModelHash", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BaselineQuantDefinition", b => + { + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade); + + b.Navigation("ArchitectureFamily"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.BenchmarkRun", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany() + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.CategoryBenchmark", "CategoryBenchmark") + .WithMany() + .HasForeignKey("CategoryBenchmarkId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("CategoryBenchmark"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.CategoryBenchmark", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany("CategorBenchmarks") + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiBenchmark"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ExecutionPlanProbeCache", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.ImatrixDefinition", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("AiModelHash"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.LearnedBaselineTensorQuant", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany() + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.BaselineQuantDefinition", "BaselineQuantDefinition") + .WithMany() + .HasForeignKey("BaselineQuantDefinitionId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("BaselineQuantDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.QuantizationRun", b => + { + b.HasOne("MQ.DB.Models.DbModels.AiBenchmark", "AiBenchmark") + .WithMany() + .HasForeignKey("AiBenchmarkId") + .OnDelete(DeleteBehavior.SetNull); + + b.HasOne("MQ.DB.Models.DbModels.AiModelHash", "AiModelHash") + .WithMany() + .HasForeignKey("AiModelHashId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.ImatrixDefinition", "ImatrixDefinition") + .WithMany() + .HasForeignKey("ImatrixDefinitionId") + .OnDelete(DeleteBehavior.Restrict); + + b.HasOne("MQ.DB.Models.DbModels.TensorCombo", "TensorCombo") + .WithMany() + .HasForeignKey("TensorComboId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.HasOne("MQ.DB.Models.DbModels.TensorGroupProfile", "TensorGroupProfile") + .WithMany() + .HasForeignKey("TensorGroupProfileId") + .OnDelete(DeleteBehavior.Restrict) + .IsRequired(); + + b.Navigation("AiBenchmark"); + + b.Navigation("AiModelHash"); + + b.Navigation("ArchitectureFamily"); + + b.Navigation("ImatrixDefinition"); + + b.Navigation("TensorCombo"); + + b.Navigation("TensorGroupProfile"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.TensorGroupProfile", b => + { + b.HasOne("MQ.DB.Models.DbModels.ArchitectureFamily", "ArchitectureFamily") + .WithMany() + .HasForeignKey("ArchitectureFamilyId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.Navigation("ArchitectureFamily"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AiBenchmark", b => + { + b.Navigation("CategorBenchmarks"); + + b.Navigation("LearnedSources"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyInteractionRule", b => + { + b.Navigation("GroupStates"); + }); + + modelBuilder.Entity("MQ.DB.Models.DbModels.AnomalyProbeSession", b => + { + b.Navigation("Observations"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/MQ.DB/Models/BaselineQuants.cs b/src/MQ.DB/Models/BaselineQuants.cs new file mode 100644 index 0000000..3b26606 --- /dev/null +++ b/src/MQ.DB/Models/BaselineQuants.cs @@ -0,0 +1,700 @@ +using System.Collections.Immutable; +using MQ.DB; + +namespace MQ.DB.Models; + +public record BaselineQuants( + byte UniqueId, + bool RequiresImatrix, + ImmutableArray Names, + string QuantizeBaseArgumentName, + TensorWeightScheme PrimaryTensorWeightScheme, + ImmutableArray LearnedMatchTensorWeightSchemes, + ImmutableArray BannedGroupIds, + bool IsLearningBaseline, + bool IsCombinationCarrierCandidate, + bool IsExplicitGroupCombinationCandidate, + bool IsHighPrecisionExactAlias, + byte BitRange, + bool IsCustomBaseline = false, + string CanonicalKey = "", + string SourceKind = "standard", + string? SourceOwner = null, + string? SourceRepository = null, + string? SourceFileName = null, + string? ShortSourceName = null, + int ExplicitCandidateSortOrder = int.MaxValue) +{ + public const byte NativeSourceUniqueId = 250; + private const byte FirstDynamicCustomBaselineId = 100; + + private static readonly object DynamicLock = new(); + private static readonly List DynamicCustomBaselines = new(); + private static HashSet? EnabledStandardLearningBaselineIds; + private static HashSet? EnabledStandardCombinationCarrierIds; + private static HashSet? EnabledStandardExplicitCandidateIds; + + public TensorWeightScheme? DefaultTensorScheme => PrimaryTensorWeightScheme; + public ImmutableArray TensorWeightSchemes => LearnedMatchTensorWeightSchemes; + public bool IsPureBaselineCandidate => IsLearningBaseline; + public bool IsHighPrecisionExplicitCandidate => IsHighPrecisionExactAlias; + + public bool IsExternalRepositoryBaseline => + IsCustomBaseline && + !string.IsNullOrWhiteSpace(SourceRepository) && + !string.IsNullOrWhiteSpace(SourceFileName); + + private static BaselineQuants Create( + byte uniqueId, + bool requiresImatrix, + string name, + string quantizeBaseArgumentName, + TensorWeightScheme primaryTensorWeightScheme, + ImmutableArray learnedMatchTensorWeightSchemes, + ImmutableArray bannedGroupIds, + bool isLearningBaseline, + bool isCombinationCarrierCandidate, + bool isExplicitGroupCombinationCandidate, + bool isHighPrecisionExactAlias, + byte bitRange, + int explicitCandidateSortOrder = int.MaxValue, + bool isCustomBaseline = false, + string? canonicalKey = null, + string sourceKind = "standard", + string? sourceOwner = null, + string? sourceRepository = null, + string? sourceFileName = null, + string? shortSourceName = null) + { + return new BaselineQuants( + uniqueId, + requiresImatrix, + [name], + quantizeBaseArgumentName, + primaryTensorWeightScheme, + learnedMatchTensorWeightSchemes, + bannedGroupIds, + isLearningBaseline, + isCombinationCarrierCandidate, + isExplicitGroupCombinationCandidate, + isHighPrecisionExactAlias, + bitRange, + isCustomBaseline, + canonicalKey ?? $"standard:{name.ToLowerInvariant()}", + sourceKind, + sourceOwner, + sourceRepository, + sourceFileName, + shortSourceName, + explicitCandidateSortOrder); + } + + + public static readonly BaselineQuants Q8_0 = + Create(0, false, "Q8_0", "Q8_0", TensorWeightScheme.Q8_0, [TensorWeightScheme.Q8_0], [], true, true, true, + false, 8, 16); + + public static readonly BaselineQuants Q6_K = + Create(1, false, "Q6_K", "Q6_K", TensorWeightScheme.Q6_K, [TensorWeightScheme.Q6_K], [], true, false, true, + false, 6, 15); + + public static readonly BaselineQuants Q5_K = + Create(2, false, "Q5_K", "Q5_K", TensorWeightScheme.Q5_K, [TensorWeightScheme.Q5_K], [], true, false, true, + false, 5, 14); + + public static readonly BaselineQuants Q5_K_S = + Create(13, false, "Q5_K_S", "Q5_K_S", TensorWeightScheme.Q5_K_S, [TensorWeightScheme.Q5_K_S], [], true, false, + true, false, 5, 13); + + + public static readonly BaselineQuants Q4_K_M = + Create(3, false, "Q4_K_M", "Q4_K_M", TensorWeightScheme.Q4_K, [TensorWeightScheme.Q4_K], [], true, false, true, + false, 4, 12); + + public static readonly BaselineQuants Q4_K_S = + Create(14, false, "Q4_K_S", "Q4_K_S", TensorWeightScheme.Q4_K_S, [TensorWeightScheme.Q4_K_S], [], true, false, + true, false, 4, 11); + + + public static readonly BaselineQuants IQ4_NL = + Create(5, false, "IQ4_NL", "IQ4_NL", TensorWeightScheme.IQ4_NL, [TensorWeightScheme.IQ4_NL], [], true, false, + true, false, 4, 10); + + public static readonly BaselineQuants IQ4_XS = + Create(6, false, "IQ4_XS", "IQ4_XS", TensorWeightScheme.IQ4_XS, [TensorWeightScheme.IQ4_XS], [], true, false, + true, false, 4, 9); + + public static readonly BaselineQuants MXFP4_MOE = + Create(15, false, "MXFP4_MOE", "MXFP4_MOE", TensorWeightScheme.MXFP4, + [TensorWeightScheme.MXFP4, TensorWeightScheme.IQ3_S, TensorWeightScheme.IQ3_XS], [], false, false, false, + false, 4, 8); + + public static readonly BaselineQuants IQ3_M = + Create(17, true, "IQ3_M", "IQ3_M", TensorWeightScheme.IQ3_S, [TensorWeightScheme.IQ3_S], [], true, false, true, + false, 3, 7); + + + public static readonly BaselineQuants IQ3_S = + Create(7, true, "IQ3_S", "IQ3_S", TensorWeightScheme.IQ3_S, [TensorWeightScheme.IQ3_S], [], true, false, true, + false, 3, 6); + + public static readonly BaselineQuants IQ3_XS = + Create(8, true, "IQ3_XS", "IQ3_XS", TensorWeightScheme.IQ3_XS, [TensorWeightScheme.IQ3_XS], [], true, false, + true, false, 3, 5); + + public static readonly BaselineQuants IQ3_XXS = + Create(9, true, "IQ3_XXS", "IQ3_XXS", TensorWeightScheme.IQ3_XXS, [TensorWeightScheme.IQ3_XXS], [], true, false, + true, false, 3, 4); + + public static readonly BaselineQuants IQ2_M = + Create(16, true, "IQ2_M", "IQ2_M", TensorWeightScheme.IQ2_S, [TensorWeightScheme.IQ2_S], [], true, false, true, + false, 2, 3); + + + public static readonly BaselineQuants IQ2_S = + Create(10, true, "IQ2_S", "IQ2_S", TensorWeightScheme.IQ2_S, [TensorWeightScheme.IQ2_S], [], true, false, true, + false, 2, 2); + + public static readonly BaselineQuants IQ2_XS = + Create(11, true, "IQ2_XS", "IQ2_XS", TensorWeightScheme.IQ2_XS, [TensorWeightScheme.IQ2_XS], [], true, false, + true, false, 2, 1); + + public static readonly BaselineQuants IQ2_XXS = + Create(12, true, "IQ2_XXS", "IQ2_XXS", TensorWeightScheme.IQ2_XXS, [TensorWeightScheme.IQ2_XXS], [], true, + false, true, false, 2, 0); + + public static readonly BaselineQuants IQ1_S = + Create(18, true, "IQ1_S", "IQ1_S", TensorWeightScheme.IQ1_S, [TensorWeightScheme.IQ1_S], [], true, + false, true, false, 1, -2); + + public static readonly BaselineQuants IQ1_M = + Create(19, true, "IQ1_M", "IQ1_M", TensorWeightScheme.IQ1_M, [TensorWeightScheme.IQ1_M], [], true, + false, true, false, 1, -1); + + public static readonly BaselineQuants BF16_Hybrid = + Create(201, false, "BF16", "BF16", TensorWeightScheme.BF16, [TensorWeightScheme.BF16], [], false, false, false, + true, 16, int.MaxValue, false, "alias:bf16", "exact_alias", null, null, null, null); + + public static readonly BaselineQuants F16_Hybrid = + Create(202, false, "F16", "F16", TensorWeightScheme.F16, [TensorWeightScheme.F16], [], false, false, false, + true, 16, int.MaxValue, false, "alias:f16", "exact_alias", null, null, null, null); + + private static readonly ImmutableArray StandardBaselines = + [ + Q8_0, + Q6_K, + Q5_K, + Q5_K_S, + Q4_K_M, + Q4_K_S, + IQ4_NL, + IQ4_XS, + MXFP4_MOE, + IQ3_M, + IQ3_S, + IQ3_XS, + IQ3_XXS, + IQ2_M, + IQ2_S, + IQ2_XS, + IQ2_XXS, + IQ1_S, + IQ1_M + ]; + + private static readonly ImmutableArray ExactAliases = + [ + BF16_Hybrid, + F16_Hybrid + ]; + + public static IReadOnlyList All => GetAllRecognizedBaselines(); + + public static BaselineQuants CreateDynamicCustomBaseline( + byte uniqueId, + string displayName, + string quantizeBaseArgumentName, + string sourceRepository, + string sourceFileName, + string shortSourceName, + string sourceOwner, + string sourceKind, + string canonicalKey, + TensorWeightScheme primaryTensorWeightScheme, + ImmutableArray learnedMatchTensorWeightSchemes, + IReadOnlyCollection bannedGroupIds, + bool requiresImatrix, + bool isLearningBaseline, + bool isCombinationCarrierCandidate, + bool isExplicitGroupCombinationCandidate, + byte bitRange, + int explicitCandidateSortOrder) + { + return new BaselineQuants( + uniqueId, + requiresImatrix, + [displayName, primaryTensorWeightScheme.Names[0]], + quantizeBaseArgumentName, + primaryTensorWeightScheme, + learnedMatchTensorWeightSchemes, + bannedGroupIds?.Distinct().OrderBy(x => x).ToImmutableArray() ?? ImmutableArray.Empty, + isLearningBaseline, + isCombinationCarrierCandidate, + isExplicitGroupCombinationCandidate, + false, + bitRange, + true, + canonicalKey, + sourceKind, + sourceOwner, + sourceRepository, + sourceFileName, + shortSourceName, + explicitCandidateSortOrder); + } + + public static void ResetDynamicCustomBaselines() + { + lock (DynamicLock) + { + DynamicCustomBaselines.Clear(); + } + } + + public static byte GetFirstAvailableDynamicBaselineId() + { + var used = GetAllRecognizedBaselines().Select(x => x.UniqueId).ToHashSet(); + for (byte id = FirstDynamicCustomBaselineId; id < 200; id++) + { + if (!used.Contains(id)) + return id; + } + + throw new InvalidOperationException("No free dynamic baseline ids remain in the configured range."); + } + + public static void RegisterDynamicCustomBaseline(BaselineQuants baseline) + { + if (!baseline.IsCustomBaseline) + throw new InvalidOperationException("Only custom baselines can be dynamically registered."); + + lock (DynamicLock) + { + var existingDynamic = DynamicCustomBaselines.FirstOrDefault(x => + x.UniqueId == baseline.UniqueId || + string.Equals(x.CanonicalKey, baseline.CanonicalKey, StringComparison.Ordinal)); + if (existingDynamic != null) + { + DynamicCustomBaselines.Remove(existingDynamic); + } + else + { + var builtInCollision = StandardBaselines.Concat(ExactAliases) + .FirstOrDefault(x => + x.UniqueId == baseline.UniqueId || string.Equals(x.CanonicalKey, baseline.CanonicalKey, + StringComparison.Ordinal)); + + if (builtInCollision != null) + throw new InvalidOperationException( + $"Dynamic baseline collision detected against built-in baseline '{builtInCollision.Names[0]}' for id/key '{baseline.UniqueId}/{baseline.CanonicalKey}'."); + } + + DynamicCustomBaselines.Add(baseline); + } + } + + public static void ConfigureStandardRoleFilters( + IReadOnlyCollection? enabledLearningBaselineIds, + IReadOnlyCollection? enabledCombinationCarrierIds, + IReadOnlyCollection? enabledExplicitCandidateIds) + { + EnabledStandardLearningBaselineIds = + enabledLearningBaselineIds == null ? null : enabledLearningBaselineIds.ToHashSet(); + EnabledStandardCombinationCarrierIds = + enabledCombinationCarrierIds == null ? null : enabledCombinationCarrierIds.ToHashSet(); + EnabledStandardExplicitCandidateIds = + enabledExplicitCandidateIds == null ? null : enabledExplicitCandidateIds.ToHashSet(); + } + + public static void ConfigureStandardPolicy( + bool includeStandardLearningBaselines, + bool includeStandardCombinationCarriers, + bool includeStandardGroupCandidates, + bool alwaysIncludeQ8Anchor, + IReadOnlyCollection? standardLearningBaselineAllowList, + IReadOnlyCollection? standardCarrierAllowList, + IReadOnlyCollection? standardGroupCandidateAllowList) + { + HashSet? learning = includeStandardLearningBaselines + ? ResolveNamesToIdsOrNull(standardLearningBaselineAllowList) + : new HashSet(); + + HashSet? carriers = includeStandardCombinationCarriers + ? ResolveNamesToIdsOrNull(standardCarrierAllowList) + : new HashSet(); + + HashSet? explicitCandidates = includeStandardGroupCandidates + ? ResolveNamesToIdsOrNull(standardGroupCandidateAllowList) + : new HashSet(); + + if (alwaysIncludeQ8Anchor) + { + learning ??= new HashSet(); + carriers ??= new HashSet(); + explicitCandidates ??= new HashSet(); + learning.Add(Q8_0.UniqueId); + carriers.Add(Q8_0.UniqueId); + explicitCandidates.Add(Q8_0.UniqueId); + } + + ConfigureStandardRoleFilters(learning, carriers, explicitCandidates); + } + + private static HashSet? ResolveNamesToIdsOrNull(IReadOnlyCollection? names) + { + if (names == null || names.Count == 0) + return null; + + var set = new HashSet(); + foreach (var raw in names) + { + var item = ResolveBuiltInStandardBaseline(raw ?? string.Empty); + if (item != null) + set.Add(item.UniqueId); + } + + return set; + } + + public sealed class ExternalBaselineRegistration + { + public string CanonicalKey { get; set; } = string.Empty; + public string DisplayName { get; set; } = string.Empty; + public string QuantizeBaseArgumentName { get; set; } = string.Empty; + public string Repository { get; set; } = string.Empty; + public string RepositoryFileName { get; set; } = string.Empty; + public string OwnerShortName { get; set; } = string.Empty; + public string BaselineFamilyName { get; set; } = string.Empty; + public TensorWeightScheme TensorScheme { get; set; } = default!; + public bool RequiresImatrix { get; set; } + public bool AddAsLearningBaseline { get; set; } + public bool AddAsCombinationCarrier { get; set; } + public bool AddAsGroupCandidate { get; set; } + public byte BitRange { get; set; } + public IReadOnlyCollection BannedGroupIds { get; set; } = Array.Empty(); + } + + public static BaselineQuants RegisterCustomExternalBaseline(ExternalBaselineRegistration registration) + { + var sortOrder = StandardBaselines + .FirstOrDefault(x => + string.Equals(x.Names[0], registration.BaselineFamilyName, StringComparison.OrdinalIgnoreCase)) + ?.ExplicitCandidateSortOrder ?? int.MaxValue; + + var baseline = CreateDynamicCustomBaseline( + GetFirstAvailableDynamicBaselineId(), + registration.DisplayName, + registration.QuantizeBaseArgumentName, + registration.Repository, + registration.RepositoryFileName, + registration.OwnerShortName, + registration.OwnerShortName, + "huggingface_repo", + registration.CanonicalKey, + registration.TensorScheme, + [registration.TensorScheme], + registration.BannedGroupIds, + registration.RequiresImatrix, + registration.AddAsLearningBaseline, + registration.AddAsCombinationCarrier, + registration.AddAsGroupCandidate, + registration.BitRange, + sortOrder); + + RegisterDynamicCustomBaseline(baseline); + return baseline; + } + + public static BaselineQuants GetNativeQuant() + { + var nativeScheme = TensorWeightScheme.GetCurrentNativePrecisionScheme(); + + return new BaselineQuants( + NativeSourceUniqueId, + false, + [nativeScheme.Names[0]], + nativeScheme.Names[0], + nativeScheme, + [nativeScheme], + [], + false, + false, + false, + true, + 16, + false, + $"native:{nativeScheme.Names[0].ToLowerInvariant()}", + "native_exact_alias", + null, + null, + null, + null, + int.MaxValue); + } + + public static BaselineQuants GetBF16Quant() => GetNativeQuant(); + + public static IReadOnlyList GetBuiltInStandardBaselines() => + StandardBaselines.OrderBy(x => x.UniqueId).ToList(); + + public static BaselineQuants? ResolveBuiltInStandardBaseline(string name) + { + if (string.IsNullOrWhiteSpace(name)) + return null; + + return StandardBaselines.FirstOrDefault(x => + x.Names.Any(n => string.Equals(n, name, StringComparison.OrdinalIgnoreCase)) || + string.Equals(x.PrimaryTensorWeightScheme.Names[0], name, StringComparison.OrdinalIgnoreCase)); + } + + /// + /// Resolves user-facing standard-role configuration with canonical baseline names + /// taking precedence over shared tensor-scheme aliases. Keep the legacy resolver + /// unchanged because external baseline family normalization relies on its historical + /// scheme-first registry ordering. + /// + public static BaselineQuants? ResolveBuiltInStandardRoleBaseline(string name) + { + if (string.IsNullOrWhiteSpace(name)) + return null; + + var exactName = StandardBaselines.FirstOrDefault(x => + x.Names.Any(n => string.Equals(n, name, StringComparison.OrdinalIgnoreCase))); + + return exactName ?? ResolveBuiltInStandardBaseline(name); + } + + public static IReadOnlyList GetAllRecognizedBaselines() => + StandardBaselines + .Concat(DynamicCustomBaselines.OrderBy(x => x.UniqueId)) + .Concat(ExactAliases) + .OrderBy(x => x.UniqueId) + .ToList(); + + private static IEnumerable FilterStandardByRole( + IEnumerable source, + HashSet? enabledIds) + { + return enabledIds == null ? source : source.Where(x => enabledIds.Contains(x.UniqueId)); + } + + public static IReadOnlyList GetLearningBaselines(bool hasUsableImatrix) + { + var standard = FilterStandardByRole(StandardBaselines.Where(x => x.IsLearningBaseline), + EnabledStandardLearningBaselineIds); + var custom = DynamicCustomBaselines.Where(x => x.IsLearningBaseline); + + return standard + .Concat(custom) + .Where(x => hasUsableImatrix || !x.RequiresImatrix) + .OrderBy(x => x.UniqueId) + .ToList(); + } + + public static IReadOnlyList GetPureBaselineCandidates(bool hasUsableImatrix) => + GetLearningBaselines(hasUsableImatrix); + + public static IReadOnlyList GetCombinationCarrierBaselines(bool hasUsableImatrix) + { + var standard = FilterStandardByRole(StandardBaselines.Where(x => x.IsCombinationCarrierCandidate), + EnabledStandardCombinationCarrierIds); + var custom = DynamicCustomBaselines.Where(x => x.IsCombinationCarrierCandidate); + + var result = standard + .Concat(custom) + .Where(x => hasUsableImatrix || !x.RequiresImatrix) + .OrderBy(x => x.UniqueId) + .ToList(); + + return result.Count == 0 ? new[] { Q8_0 } : result; + } + + public static IReadOnlyList GetGroupCombinationCandidates(bool hasUsableImatrix, + bool allowHighPrecisionHybrids) + { + var standard = FilterStandardByRole(StandardBaselines.Where(x => x.IsExplicitGroupCombinationCandidate), + EnabledStandardExplicitCandidateIds); + var custom = DynamicCustomBaselines.Where(x => x.IsExplicitGroupCombinationCandidate); + + return standard + .Concat(custom) + .Where(x => hasUsableImatrix || !x.RequiresImatrix) + .OrderBy(x => x.ExplicitCandidateSortOrder) + .ThenBy(x => x.UniqueId) + .ToList(); + } + + public static IReadOnlyList GetGroupCombinationCandidatesSmallestFirst(bool hasUsableImatrix, + bool allowHighPrecisionHybrids) => + GetGroupCombinationCandidates(hasUsableImatrix, allowHighPrecisionHybrids) + .OrderBy(x => x.BitRange) + .ThenBy(x => x.ExplicitCandidateSortOrder) + .ThenBy(x => x.UniqueId) + .ToList(); + + public static IReadOnlyList GetExactHighPrecisionAliases(bool allowHighPrecisionHybrids) + { + if (!allowHighPrecisionHybrids) + return Array.Empty(); + + return ExactAliases.OrderBy(x => x.UniqueId).ToList(); + } + + public const byte TensorConfigNullSlotValue = 0; + + public static BaselineQuants GetDefaultExplicitFallbackBaseline() => Q8_0; + public static bool IsNullTensorConfigGroupSlot(byte storedValue) => storedValue == TensorConfigNullSlotValue; + + public static byte EncodeTensorConfigGroupSlot(BaselineQuants baseline) => + EncodeTensorConfigGroupSlotBaselineId(baseline.UniqueId); + + public static byte EncodeTensorConfigGroupSlot(TensorWeightScheme exactScheme) => + EncodeTensorConfigGroupSlotBaselineId(GetExactOverrideStorageId(exactScheme)); + + public static byte EncodeTensorConfigGroupSlotBaselineId(byte baselineId) + { + if (baselineId == byte.MaxValue) + throw new InvalidOperationException("Baseline id 255 cannot be encoded into a tensor-config group slot."); + + return checked((byte)(baselineId + 1)); + } + + public static byte DecodeTensorConfigGroupSlotToBaselineId(byte storedValue) + { + if (IsNullTensorConfigGroupSlot(storedValue)) + throw new InvalidOperationException( + "Tensor-config group slot 0 represents NULL and cannot be decoded as a baseline id."); + + return checked((byte)(storedValue - 1)); + } + + public static BaselineQuants DecodeTensorConfigGroupSlotToBaseline(byte storedValue) => + FromId(DecodeTensorConfigGroupSlotToBaselineId(storedValue)); + + public static bool IsNativeExactAlias(BaselineQuants baseline) => IsNativeExactAlias(baseline.UniqueId); + + public static bool IsNativeExactAlias(byte baselineId) + { + return baselineId == NativeSourceUniqueId || + baselineId == BF16_Hybrid.UniqueId || + baselineId == F16_Hybrid.UniqueId; + } + + public static byte CanonicalLearningBaselineId(BaselineQuants baseline) => + CanonicalLearningBaselineId(baseline.UniqueId); + + public static byte CanonicalLearningBaselineId(byte baselineId) + { + return IsNativeExactAlias(baselineId) + ? NativeSourceUniqueId + : baselineId; + } + + public static byte GetExactOverrideStorageId(TensorWeightScheme scheme) + { + if (scheme.UniqueId == TensorWeightScheme.BF16.UniqueId) + return BF16_Hybrid.UniqueId; + + if (scheme.UniqueId == TensorWeightScheme.F16.UniqueId) + return F16_Hybrid.UniqueId; + + if (scheme.UniqueId == TensorWeightScheme.GetCurrentNativePrecisionScheme().UniqueId) + return NativeSourceUniqueId; + + throw new InvalidOperationException( + $"Tensor scheme '{scheme.Names[0]}' does not have a supported exact-override storage baseline id."); + } + + public static TensorWeightScheme ResolveExactOverrideScheme(byte baselineId) + { + return baselineId switch + { + NativeSourceUniqueId => TensorWeightScheme.GetCurrentNativePrecisionScheme(), + 201 => TensorWeightScheme.BF16, + 202 => TensorWeightScheme.F16, + _ => throw new InvalidOperationException($"Baseline id '{baselineId}' is not an exact override alias.") + }; + } + + public static void ValidateIntegrityOrThrow() + { + var all = GetAllRecognizedBaselines(); + + var invalidBaselines = all + .Where(x => !x.IsHighPrecisionExactAlias) + .Where(x => x.LearnedMatchTensorWeightSchemes.IsDefaultOrEmpty) + .Select(x => x.Names.IsDefaultOrEmpty ? $"id:{x.UniqueId}" : x.Names[0]) + .ToList(); + + if (invalidBaselines.Count > 0) + { + throw new InvalidOperationException( + "Every non-alias BaselineQuants entry must define at least one TensorWeightScheme. Missing for: " + + string.Join(", ", invalidBaselines)); + } + + var duplicateIds = all + .GroupBy(x => x.UniqueId) + .Where(g => g.Count() > 1) + .Select(g => g.Key) + .ToList(); + + if (duplicateIds.Count > 0) + throw new InvalidOperationException($"Duplicate baseline ids detected: {string.Join(", ", duplicateIds)}"); + + var duplicateKeys = all + .GroupBy(x => x.CanonicalKey, StringComparer.Ordinal) + .Where(g => g.Count() > 1) + .Select(g => g.Key) + .ToList(); + + if (duplicateKeys.Count > 0) + throw new InvalidOperationException( + $"Duplicate baseline canonical keys detected: {string.Join(", ", duplicateKeys)}"); + } + + public static BaselineQuants FromId(byte id) + { + if (id == NativeSourceUniqueId) + return GetNativeQuant(); + + var found = GetAllRecognizedBaselines().FirstOrDefault(x => x.UniqueId == id); + if (found == null) + throw new InvalidOperationException($"Unknown baseline quant id '{id}'."); + + return found; + } + + public static BaselineQuants FromTensorSchemeId(byte schemeId) + { + if (schemeId == TensorWeightScheme.BF16.UniqueId) + return BF16_Hybrid; + + if (schemeId == TensorWeightScheme.F16.UniqueId) + return F16_Hybrid; + + if (schemeId == TensorWeightScheme.GetCurrentNativePrecisionScheme().UniqueId) + return GetNativeQuant(); + + var found = GetAllRecognizedBaselines() + .Where(x => !x.IsHighPrecisionExactAlias) + .FirstOrDefault(x => + x.PrimaryTensorWeightScheme.UniqueId == schemeId || + x.LearnedMatchTensorWeightSchemes.Any(s => s.UniqueId == schemeId)); + + if (found == null) + throw new InvalidOperationException($"Unknown tensor scheme id '{schemeId}' for baseline conversion."); + + return found; + } +} diff --git a/src/MQ.DB/Models/BenchmarkResult.cs b/src/MQ.DB/Models/BenchmarkResult.cs new file mode 100644 index 0000000..f5281d1 --- /dev/null +++ b/src/MQ.DB/Models/BenchmarkResult.cs @@ -0,0 +1,15 @@ +namespace MQ.DB.Models; + +public class BenchmarkResult +{ + public LlamaBenchMetrics? LlamaBench { get; set; } + + public Dictionary Perplexity { get; set; } = + new(StringComparer.OrdinalIgnoreCase); + + /// + /// Persisted into bench_metrics.json so disk-only reuse can still sync DB later + /// even if the temporary GGUF has already been deleted. + /// + public ulong? ModelSizeBytes { get; set; } +} \ No newline at end of file diff --git a/src/MQ.DB/Models/DbModels/AiBenchmark.cs b/src/MQ.DB/Models/DbModels/AiBenchmark.cs new file mode 100644 index 0000000..b97c32f --- /dev/null +++ b/src/MQ.DB/Models/DbModels/AiBenchmark.cs @@ -0,0 +1,139 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using MQ.DB.Interfaces; + +namespace MQ.DB.Models.DbModels; + +public enum BenchmarkCategory +{ + General = 1, + Math = 2, + Code = 3, +} + +public class AiBenchmark : ISQLiteEntity +{ + public Guid Id { get; set; } = Guid.NewGuid(); + + /// + /// n-N gpu layers + /// + public byte Ngl { get; set; } + + /// + /// Size of the model in bytes at this combination. + /// + public ulong SizeBytes { get; set; } + + public double TokensPerSecond { get; set; } + + public int ArchitectureFamilyId { get; set; } + public ArchitectureFamily ArchitectureFamily { get; set; } = default!; + + public int TensorGroupProfileId { get; set; } + public TensorGroupProfile TensorGroupProfile { get; set; } = default!; + + /// + /// foreign key + /// + public Guid TensorComboId { get; set; } + + public TensorCombo TensorCombo { get; set; } = default!; + + /// + /// foreign key + /// + public uint AiModelHashId { get; set; } + + public AiModelHash AiModelHash { get; set; } = default!; + + public int? ImatrixDefinitionId { get; set; } + public ImatrixDefinition? ImatrixDefinition { get; set; } + + public List CategorBenchmarks { get; set; } = new(); + public List LearnedSources { get; set; } = new(); + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + + builder.Property(x => x.Id) + .ValueGeneratedNever(); + + builder.HasIndex(x => new { x.ArchitectureFamilyId, x.TensorGroupProfileId, x.AiModelHashId, x.ImatrixDefinitionId, x.TensorComboId }) + .IsUnique(); + + builder.HasIndex(x => new { x.ArchitectureFamilyId, x.TensorGroupProfileId, x.TensorComboId }); + + builder.HasOne(x => x.ArchitectureFamily) + .WithMany() + .HasForeignKey(x => x.ArchitectureFamilyId) + .OnDelete(DeleteBehavior.Cascade); + + builder.HasOne(x => x.TensorGroupProfile) + .WithMany() + .HasForeignKey(x => x.TensorGroupProfileId) + .OnDelete(DeleteBehavior.Restrict); + + builder.HasOne(x => x.TensorCombo) + .WithMany() + .HasForeignKey(x => x.TensorComboId) + .OnDelete(DeleteBehavior.Restrict); + + builder.HasOne(x => x.AiModelHash) + .WithMany() + .HasForeignKey(x => x.AiModelHashId) + .OnDelete(DeleteBehavior.Cascade); + + builder.HasOne(x => x.ImatrixDefinition) + .WithMany() + .HasForeignKey(x => x.ImatrixDefinitionId) + .OnDelete(DeleteBehavior.Restrict); + + builder.HasMany(x => x.CategorBenchmarks) + .WithOne(x => x.AiBenchmark) + .HasForeignKey(x => x.AiBenchmarkId) + .OnDelete(DeleteBehavior.Cascade); + + builder.HasMany(x => x.LearnedSources) + .WithOne(x => x.AiBenchmark) + .HasForeignKey(x => x.AiBenchmarkId) + .OnDelete(DeleteBehavior.Cascade); + } +} + +public class CategoryBenchmark : ISQLiteEntity +{ + public Guid Id { get; set; } = Guid.NewGuid(); + + /// + /// foreign key to AiBenchmark + /// + public Guid AiBenchmarkId { get; set; } + + public AiBenchmark AiBenchmark { get; set; } = default!; + + /// + /// Byte version to BenchmarkCategory enum in C# + /// + public byte Category { get; set; } + + public double Kld { get; set; } + public double Ppl { get; set; } + public double PplError { get; set; } + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + + builder.Property(x => x.Id) + .ValueGeneratedNever(); + + builder.HasIndex(x => x.AiBenchmarkId); + + builder.HasOne(x => x.AiBenchmark) + .WithMany(x => x.CategorBenchmarks) + .HasForeignKey(x => x.AiBenchmarkId) + .OnDelete(DeleteBehavior.Cascade); + } +} diff --git a/src/MQ.DB/Models/DbModels/AiBenchmarkLearnedSource.cs b/src/MQ.DB/Models/DbModels/AiBenchmarkLearnedSource.cs new file mode 100644 index 0000000..1c755d2 --- /dev/null +++ b/src/MQ.DB/Models/DbModels/AiBenchmarkLearnedSource.cs @@ -0,0 +1,74 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using MQ.DB.Interfaces; + +namespace MQ.DB.Models.DbModels; + +public class AiBenchmarkLearnedSource : ISQLiteEntity +{ + public Guid Id { get; set; } = Guid.NewGuid(); + + public Guid AiBenchmarkId { get; set; } + public AiBenchmark AiBenchmark { get; set; } = default!; + + public int ArchitectureFamilyId { get; set; } + public ArchitectureFamily ArchitectureFamily { get; set; } = default!; + + public int TensorGroupProfileId { get; set; } + public TensorGroupProfile TensorGroupProfile { get; set; } = default!; + + public Guid TensorComboId { get; set; } + public TensorCombo TensorCombo { get; set; } = default!; + + public byte TensorGroupId { get; set; } + + public int BaselineQuantDefinitionId { get; set; } + public BaselineQuantDefinition BaselineQuantDefinition { get; set; } = default!; + + public Guid? SourceLearningBenchmarkId { get; set; } + public AiBenchmark? SourceLearningBenchmark { get; set; } + + public string BaselineCanonicalKey { get; set; } = string.Empty; + public DateTime CreatedUtc { get; set; } = DateTime.UtcNow; + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + builder.Property(x => x.Id).ValueGeneratedNever(); + builder.Property(x => x.BaselineCanonicalKey).HasMaxLength(512).IsRequired(); + + builder.HasIndex(x => new { x.AiBenchmarkId, x.TensorGroupId }).IsUnique(); + builder.HasIndex(x => new { x.ArchitectureFamilyId, x.TensorGroupProfileId, x.BaselineQuantDefinitionId }); + builder.HasIndex(x => x.TensorComboId); + + builder.HasOne(x => x.AiBenchmark) + .WithMany(x => x.LearnedSources) + .HasForeignKey(x => x.AiBenchmarkId) + .OnDelete(DeleteBehavior.Cascade); + + builder.HasOne(x => x.ArchitectureFamily) + .WithMany() + .HasForeignKey(x => x.ArchitectureFamilyId) + .OnDelete(DeleteBehavior.Cascade); + + builder.HasOne(x => x.TensorGroupProfile) + .WithMany() + .HasForeignKey(x => x.TensorGroupProfileId) + .OnDelete(DeleteBehavior.Cascade); + + builder.HasOne(x => x.TensorCombo) + .WithMany() + .HasForeignKey(x => x.TensorComboId) + .OnDelete(DeleteBehavior.Restrict); + + builder.HasOne(x => x.BaselineQuantDefinition) + .WithMany() + .HasForeignKey(x => x.BaselineQuantDefinitionId) + .OnDelete(DeleteBehavior.Restrict); + + builder.HasOne(x => x.SourceLearningBenchmark) + .WithMany() + .HasForeignKey(x => x.SourceLearningBenchmarkId) + .OnDelete(DeleteBehavior.SetNull); + } +} diff --git a/src/MQ.DB/Models/DbModels/AiModelHash.cs b/src/MQ.DB/Models/DbModels/AiModelHash.cs new file mode 100644 index 0000000..2f6e1f4 --- /dev/null +++ b/src/MQ.DB/Models/DbModels/AiModelHash.cs @@ -0,0 +1,23 @@ +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using MQ.DB.Interfaces; + +namespace MQ.DB.Models.DbModels; + +/// +/// The created blake3 hash ID associated to a proper. +/// Additionally this has a cascading delete effect. +/// If one of these rows is ever deleted, all other table +/// rows that reference this Id as a foreign key must be deleted +/// alongside this and for it to occur safely. +/// +public class AiModelHash : ISQLiteEntity +{ + public uint Id { get; set; } + public string UniqueHash { get; set; } = null!; // Required; assigned by EF or model registration. + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + builder.HasIndex(h => h.UniqueHash); + } +} diff --git a/src/MQ.DB/Models/DbModels/AnomalyProbeSession.cs b/src/MQ.DB/Models/DbModels/AnomalyProbeSession.cs new file mode 100644 index 0000000..ed84601 --- /dev/null +++ b/src/MQ.DB/Models/DbModels/AnomalyProbeSession.cs @@ -0,0 +1,231 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using MQ.DB.Interfaces; + +namespace MQ.DB.Models.DbModels; + +public class AnomalyProbeSession : ISQLiteEntity +{ + public Guid Id { get; set; } = Guid.NewGuid(); + public int ArchitectureFamilyId { get; set; } + public ArchitectureFamily ArchitectureFamily { get; set; } = default!; + public int TensorGroupProfileId { get; set; } + public TensorGroupProfile TensorGroupProfile { get; set; } = default!; + public uint AiModelHashId { get; set; } + public AiModelHash AiModelHash { get; set; } = default!; + public int? ImatrixDefinitionId { get; set; } + public ImatrixDefinition? ImatrixDefinition { get; set; } + public byte BenchmarkCategory { get; set; } = (byte)MQ.DB.Models.DbModels.BenchmarkCategory.General; + public DateTime StartedUtc { get; set; } = DateTime.UtcNow; + public DateTime? CompletedUtc { get; set; } + public string SourceRunLabel { get; set; } = string.Empty; + public string ConfigJson { get; set; } = string.Empty; + public List Observations { get; set; } = new(); + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + builder.Property(x => x.Id).ValueGeneratedNever(); + builder.Property(x => x.SourceRunLabel).HasMaxLength(256); + builder.Property(x => x.ConfigJson).HasColumnType("TEXT"); + builder.HasIndex(x => new { x.ArchitectureFamilyId, x.TensorGroupProfileId, x.AiModelHashId, x.ImatrixDefinitionId, x.BenchmarkCategory, x.StartedUtc }); + builder.HasOne(x => x.ArchitectureFamily).WithMany().HasForeignKey(x => x.ArchitectureFamilyId).OnDelete(DeleteBehavior.Cascade); + builder.HasOne(x => x.TensorGroupProfile).WithMany().HasForeignKey(x => x.TensorGroupProfileId).OnDelete(DeleteBehavior.Restrict); + builder.HasOne(x => x.AiModelHash).WithMany().HasForeignKey(x => x.AiModelHashId).OnDelete(DeleteBehavior.Cascade); + builder.HasOne(x => x.ImatrixDefinition).WithMany().HasForeignKey(x => x.ImatrixDefinitionId).OnDelete(DeleteBehavior.Restrict); + builder.HasMany(x => x.Observations).WithOne(x => x.Session).HasForeignKey(x => x.SessionId).OnDelete(DeleteBehavior.Cascade); + } +} + +public class AnomalyProbeObservation : ISQLiteEntity +{ + public Guid Id { get; set; } = Guid.NewGuid(); + public Guid SessionId { get; set; } + public AnomalyProbeSession Session { get; set; } = default!; + public int ArchitectureFamilyId { get; set; } + public ArchitectureFamily ArchitectureFamily { get; set; } = default!; + public int TensorGroupProfileId { get; set; } + public TensorGroupProfile TensorGroupProfile { get; set; } = default!; + public uint AiModelHashId { get; set; } + public AiModelHash AiModelHash { get; set; } = default!; + public int? ImatrixDefinitionId { get; set; } + public ImatrixDefinition? ImatrixDefinition { get; set; } + public byte BenchmarkCategory { get; set; } = (byte)MQ.DB.Models.DbModels.BenchmarkCategory.General; + public Guid? ReferenceTensorComboId { get; set; } + public TensorCombo? ReferenceTensorCombo { get; set; } + public Guid? ProbeTensorComboId { get; set; } + public TensorCombo? ProbeTensorCombo { get; set; } + public string ProbeType { get; set; } = string.Empty; + public string Classification { get; set; } = string.Empty; + public string HypothesisLabel { get; set; } = string.Empty; + public string MovementClassification { get; set; } = string.Empty; + public string ChangedGroupSetHash { get; set; } = string.Empty; + public string ChangedGroupsJson { get; set; } = string.Empty; + public string CandidateQuantsJson { get; set; } = string.Empty; + public string ReferenceEffectiveGroupsJson { get; set; } = string.Empty; + public string CandidateEffectiveGroupsJson { get; set; } = string.Empty; + public string InactiveGroupsJson { get; set; } = string.Empty; + public string ReferenceTensorConfigKey { get; set; } = string.Empty; + public string ProbeTensorConfigKey { get; set; } = string.Empty; + public string ReferenceDisplayName { get; set; } = string.Empty; + public string ProbeDisplayName { get; set; } = string.Empty; + public string ReferenceInternalName { get; set; } = string.Empty; + public string ProbeInternalName { get; set; } = string.Empty; + public string SeedClass { get; set; } = string.Empty; + public int SeedPriority { get; set; } + public string ProbePlanClass { get; set; } = string.Empty; + public bool IsContextualAnomalyProbe { get; set; } + public bool OldBf16Isolation { get; set; } + public bool AllActiveGroupsExplicit { get; set; } + public byte ReferenceQuantId { get; set; } + public double ActualKld { get; set; } + public double PredictedKld { get; set; } + public double ReferenceActualKld { get; set; } + public double ReferencePredictedKld { get; set; } + public double ActualGainVsTwin { get; set; } + public double PredictionSpaceGapVsTwin { get; set; } + public ulong SizeSavingsBytes { get; set; } + public int UpgradeCount { get; set; } + public int DowngradeCount { get; set; } + public int SameCount { get; set; } + public int UnknownCount { get; set; } + public int NetBitDelta { get; set; } + public string RuleDirection { get; set; } = string.Empty; + public bool Accepted { get; set; } + public string FailureCode { get; set; } = string.Empty; + public string Message { get; set; } = string.Empty; + public DateTime CreatedUtc { get; set; } = DateTime.UtcNow; + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + builder.Property(x => x.Id).ValueGeneratedNever(); + builder.Property(x => x.ProbeType).HasMaxLength(64); + builder.Property(x => x.Classification).HasMaxLength(64); + builder.Property(x => x.HypothesisLabel).HasMaxLength(128); + builder.Property(x => x.MovementClassification).HasMaxLength(64); + builder.Property(x => x.ChangedGroupSetHash).HasMaxLength(128); + builder.Property(x => x.ChangedGroupsJson).HasColumnType("TEXT"); + builder.Property(x => x.CandidateQuantsJson).HasColumnType("TEXT"); + builder.Property(x => x.ReferenceEffectiveGroupsJson).HasColumnType("TEXT"); + builder.Property(x => x.CandidateEffectiveGroupsJson).HasColumnType("TEXT"); + builder.Property(x => x.InactiveGroupsJson).HasColumnType("TEXT"); + builder.Property(x => x.ReferenceTensorConfigKey).HasMaxLength(128); + builder.Property(x => x.ProbeTensorConfigKey).HasMaxLength(128); + builder.Property(x => x.ReferenceDisplayName).HasMaxLength(512); + builder.Property(x => x.ProbeDisplayName).HasMaxLength(512); + builder.Property(x => x.ReferenceInternalName).HasMaxLength(512); + builder.Property(x => x.ProbeInternalName).HasMaxLength(512); + builder.Property(x => x.SeedClass).HasMaxLength(64); + builder.Property(x => x.ProbePlanClass).HasMaxLength(64); + builder.HasIndex(x => x.ReferenceTensorConfigKey); + builder.HasIndex(x => x.ProbeTensorConfigKey); + builder.Property(x => x.RuleDirection).HasMaxLength(64); + builder.Property(x => x.FailureCode).HasMaxLength(128); + builder.Property(x => x.Message).HasMaxLength(4000); + builder.HasIndex(x => x.SessionId); + builder.HasIndex(x => new { x.ArchitectureFamilyId, x.TensorGroupProfileId, x.AiModelHashId, x.ImatrixDefinitionId, x.BenchmarkCategory, x.ChangedGroupSetHash }); + builder.HasIndex(x => new { x.ArchitectureFamilyId, x.TensorGroupProfileId, x.AiModelHashId, x.ImatrixDefinitionId, x.BenchmarkCategory, x.ReferenceTensorComboId, x.ProbeTensorComboId, x.ProbeType }); + builder.HasOne(x => x.Session).WithMany(x => x.Observations).HasForeignKey(x => x.SessionId).OnDelete(DeleteBehavior.Cascade); + builder.HasOne(x => x.ArchitectureFamily).WithMany().HasForeignKey(x => x.ArchitectureFamilyId).OnDelete(DeleteBehavior.Cascade); + builder.HasOne(x => x.TensorGroupProfile).WithMany().HasForeignKey(x => x.TensorGroupProfileId).OnDelete(DeleteBehavior.Restrict); + builder.HasOne(x => x.AiModelHash).WithMany().HasForeignKey(x => x.AiModelHashId).OnDelete(DeleteBehavior.Cascade); + builder.HasOne(x => x.ImatrixDefinition).WithMany().HasForeignKey(x => x.ImatrixDefinitionId).OnDelete(DeleteBehavior.Restrict); + builder.HasOne(x => x.ReferenceTensorCombo).WithMany().HasForeignKey(x => x.ReferenceTensorComboId).OnDelete(DeleteBehavior.Restrict); + builder.HasOne(x => x.ProbeTensorCombo).WithMany().HasForeignKey(x => x.ProbeTensorComboId).OnDelete(DeleteBehavior.Restrict); + } +} + +public class AnomalyInteractionRule : ISQLiteEntity +{ + public Guid Id { get; set; } = Guid.NewGuid(); + public int ArchitectureFamilyId { get; set; } + public ArchitectureFamily ArchitectureFamily { get; set; } = default!; + public int TensorGroupProfileId { get; set; } + public TensorGroupProfile TensorGroupProfile { get; set; } = default!; + public uint AiModelHashId { get; set; } + public AiModelHash AiModelHash { get; set; } = default!; + public int? ImatrixDefinitionId { get; set; } + public ImatrixDefinition? ImatrixDefinition { get; set; } + public byte BenchmarkCategory { get; set; } = (byte)MQ.DB.Models.DbModels.BenchmarkCategory.General; + public byte ReferenceQuantId { get; set; } + public string ReferenceContextKey { get; set; } = string.Empty; + public string ReferenceEffectiveGroupsJson { get; set; } = string.Empty; + public string CandidateEffectiveGroupsJson { get; set; } = string.Empty; + public string InactiveGroupsJson { get; set; } = string.Empty; + public string FullTensorConfigKey { get; set; } = string.Empty; + public string ReferenceDisplayName { get; set; } = string.Empty; + public string CandidateDisplayName { get; set; } = string.Empty; + public string ReferenceInternalName { get; set; } = string.Empty; + public string CandidateInternalName { get; set; } = string.Empty; + public string RuleType { get; set; } = string.Empty; + public string RuleDirection { get; set; } = string.Empty; + public string RuleStatus { get; set; } = string.Empty; + public string MovementClassification { get; set; } = string.Empty; + public string GroupSetHash { get; set; } = string.Empty; + public int GroupCount { get; set; } + public double MeanActualGainVsTwin { get; set; } + public double BestActualGainVsTwin { get; set; } + public double MeanPredictionSpaceGap { get; set; } + public double BestPredictionSpaceGap { get; set; } + public double AppliedPredictionSpaceAdjustmentKld { get; set; } + public int EvidenceCount { get; set; } + public double Confidence { get; set; } + public double ShrinkFactor { get; set; } + public string Status { get; set; } = string.Empty; + public DateTime CreatedUtc { get; set; } = DateTime.UtcNow; + public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow; + public string MetadataJson { get; set; } = string.Empty; + public List GroupStates { get; set; } = new(); + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + builder.Property(x => x.Id).ValueGeneratedNever(); + builder.Property(x => x.ReferenceContextKey).HasMaxLength(512); + builder.Property(x => x.ReferenceEffectiveGroupsJson).HasColumnType("TEXT"); + builder.Property(x => x.CandidateEffectiveGroupsJson).HasColumnType("TEXT"); + builder.Property(x => x.InactiveGroupsJson).HasColumnType("TEXT"); + builder.Property(x => x.FullTensorConfigKey).HasMaxLength(128); + builder.Property(x => x.ReferenceDisplayName).HasMaxLength(512); + builder.Property(x => x.CandidateDisplayName).HasMaxLength(512); + builder.Property(x => x.ReferenceInternalName).HasMaxLength(512); + builder.Property(x => x.CandidateInternalName).HasMaxLength(512); + builder.HasIndex(x => x.FullTensorConfigKey); + builder.Property(x => x.RuleType).HasMaxLength(64); + builder.Property(x => x.RuleDirection).HasMaxLength(64); + builder.Property(x => x.RuleStatus).HasMaxLength(64); + builder.Property(x => x.MovementClassification).HasMaxLength(64); + builder.Property(x => x.GroupSetHash).HasMaxLength(128); + builder.Property(x => x.Status).HasMaxLength(64); + builder.Property(x => x.MetadataJson).HasColumnType("TEXT"); + builder.HasIndex(x => new { x.ArchitectureFamilyId, x.TensorGroupProfileId, x.AiModelHashId, x.ImatrixDefinitionId, x.BenchmarkCategory, x.RuleDirection, x.RuleStatus }); + builder.HasIndex(x => new { x.ArchitectureFamilyId, x.TensorGroupProfileId, x.AiModelHashId, x.ImatrixDefinitionId, x.BenchmarkCategory, x.ReferenceQuantId, x.ReferenceContextKey, x.GroupSetHash, x.RuleDirection }).IsUnique(); + builder.HasOne(x => x.ArchitectureFamily).WithMany().HasForeignKey(x => x.ArchitectureFamilyId).OnDelete(DeleteBehavior.Cascade); + builder.HasOne(x => x.TensorGroupProfile).WithMany().HasForeignKey(x => x.TensorGroupProfileId).OnDelete(DeleteBehavior.Restrict); + builder.HasOne(x => x.AiModelHash).WithMany().HasForeignKey(x => x.AiModelHashId).OnDelete(DeleteBehavior.Cascade); + builder.HasOne(x => x.ImatrixDefinition).WithMany().HasForeignKey(x => x.ImatrixDefinitionId).OnDelete(DeleteBehavior.Restrict); + builder.HasMany(x => x.GroupStates).WithOne(x => x.Rule).HasForeignKey(x => x.RuleId).OnDelete(DeleteBehavior.Cascade); + } +} + +public class AnomalyInteractionRuleGroupState : ISQLiteEntity +{ + public Guid Id { get; set; } = Guid.NewGuid(); + public Guid RuleId { get; set; } + public AnomalyInteractionRule Rule { get; set; } = default!; + public byte TensorGroupId { get; set; } + public byte CandidateQuantId { get; set; } + public byte ReferenceQuantId { get; set; } + public string Movement { get; set; } = string.Empty; + public int SortOrder { get; set; } + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + builder.Property(x => x.Id).ValueGeneratedNever(); + builder.Property(x => x.Movement).HasMaxLength(64); + builder.HasIndex(x => new { x.RuleId, x.TensorGroupId }).IsUnique(); + builder.HasOne(x => x.Rule).WithMany(x => x.GroupStates).HasForeignKey(x => x.RuleId).OnDelete(DeleteBehavior.Cascade); + } +} \ No newline at end of file diff --git a/src/MQ.DB/Models/DbModels/ArchitectureFamily.cs b/src/MQ.DB/Models/DbModels/ArchitectureFamily.cs new file mode 100644 index 0000000..93f7099 --- /dev/null +++ b/src/MQ.DB/Models/DbModels/ArchitectureFamily.cs @@ -0,0 +1,24 @@ +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using MQ.DB.Interfaces; + +namespace MQ.DB.Models.DbModels; + +public class ArchitectureFamily : ISQLiteEntity +{ + public int Id { get; set; } + public string NormalizedName { get; set; } = string.Empty; + public string DisplayName { get; set; } = string.Empty; + public string TensorSignatureHash { get; set; } = string.Empty; + public int TensorCount { get; set; } + public DateTime CreatedUtc { get; set; } = DateTime.UtcNow; + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + builder.HasIndex(x => x.NormalizedName).IsUnique(); + builder.HasIndex(x => new { x.TensorSignatureHash, x.TensorCount }); + builder.Property(x => x.NormalizedName).HasMaxLength(256).IsRequired(); + builder.Property(x => x.DisplayName).HasMaxLength(256).IsRequired(); + builder.Property(x => x.TensorSignatureHash).HasMaxLength(128).IsRequired(); + } +} diff --git a/src/MQ.DB/Models/DbModels/ArchitectureFamilyModelHash.cs b/src/MQ.DB/Models/DbModels/ArchitectureFamilyModelHash.cs new file mode 100644 index 0000000..1ea94ed --- /dev/null +++ b/src/MQ.DB/Models/DbModels/ArchitectureFamilyModelHash.cs @@ -0,0 +1,31 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using MQ.DB.Interfaces; + +namespace MQ.DB.Models.DbModels; + +public class ArchitectureFamilyModelHash : ISQLiteEntity +{ + public int Id { get; set; } + public int ArchitectureFamilyId { get; set; } + public ArchitectureFamily ArchitectureFamily { get; set; } = default!; + public uint AiModelHashId { get; set; } + public AiModelHash AiModelHash { get; set; } = default!; + public bool IsCanonical { get; set; } + public DateTime CreatedUtc { get; set; } = DateTime.UtcNow; + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + builder.HasIndex(x => new { x.ArchitectureFamilyId, x.AiModelHashId }).IsUnique(); + builder.HasIndex(x => x.AiModelHashId).IsUnique(); + builder.HasOne(x => x.ArchitectureFamily) + .WithMany() + .HasForeignKey(x => x.ArchitectureFamilyId) + .OnDelete(DeleteBehavior.Cascade); + builder.HasOne(x => x.AiModelHash) + .WithMany() + .HasForeignKey(x => x.AiModelHashId) + .OnDelete(DeleteBehavior.Cascade); + } +} diff --git a/src/MQ.DB/Models/DbModels/BaselineQuantDefinition.cs b/src/MQ.DB/Models/DbModels/BaselineQuantDefinition.cs new file mode 100644 index 0000000..1a058e7 --- /dev/null +++ b/src/MQ.DB/Models/DbModels/BaselineQuantDefinition.cs @@ -0,0 +1,79 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using MQ.DB.Interfaces; + +namespace MQ.DB.Models.DbModels; + +public class BaselineQuantDefinition : ISQLiteEntity +{ + public int Id { get; set; } + + /// + /// Null for built-in standard/exact aliases. Non-null for architecture-family-scoped custom baselines. + /// + public int? ArchitectureFamilyId { get; set; } + public ArchitectureFamily? ArchitectureFamily { get; set; } + + /// + /// Compact id used inside TensorCombo slots. + /// + public byte RuntimeBaselineId { get; set; } + + public string CanonicalKey { get; set; } = string.Empty; + public string NormalizedCanonicalKey { get; set; } = string.Empty; + public string BaselineName { get; set; } = string.Empty; + public string DisplayName { get; set; } = string.Empty; + public string QuantizeBaseArgumentName { get; set; } = string.Empty; + public byte DefaultTensorSchemeId { get; set; } + public string DefaultTensorSchemeName { get; set; } = string.Empty; + public string SourceKind { get; set; } = string.Empty; + public string? SourceOwner { get; set; } + public string? SourceRepository { get; set; } + public string? NormalizedSourceRepository { get; set; } + public string? SourceFileName { get; set; } + public string? NormalizedSourceFileName { get; set; } + public string? ShortSourceName { get; set; } + public string? BaselineFamily { get; set; } + public bool IsCustomBaseline { get; set; } + public bool IsLearningBaseline { get; set; } + public bool IsCombinationCarrierCandidate { get; set; } + public bool IsExplicitGroupCombinationCandidate { get; set; } + public bool RequiresImatrix { get; set; } + public byte BitRange { get; set; } + public int ExplicitCandidateSortOrder { get; set; } + public bool IsActiveInCurrentConfig { get; set; } = true; + public DateTime FirstSeenUtc { get; set; } = DateTime.UtcNow; + public DateTime LastSeenUtc { get; set; } = DateTime.UtcNow; + public DateTime LastUpdatedUtc { get; set; } = DateTime.UtcNow; + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + + builder.Property(x => x.CanonicalKey).HasMaxLength(512).IsRequired(); + builder.Property(x => x.NormalizedCanonicalKey).HasMaxLength(512).IsRequired(); + builder.Property(x => x.BaselineName).HasMaxLength(128).IsRequired(); + builder.Property(x => x.DisplayName).HasMaxLength(128).IsRequired(); + builder.Property(x => x.QuantizeBaseArgumentName).HasMaxLength(64).IsRequired(); + builder.Property(x => x.DefaultTensorSchemeName).HasMaxLength(64).IsRequired(); + builder.Property(x => x.SourceKind).HasMaxLength(64).IsRequired(); + builder.Property(x => x.SourceOwner).HasMaxLength(128); + builder.Property(x => x.SourceRepository).HasMaxLength(256); + builder.Property(x => x.NormalizedSourceRepository).HasMaxLength(256); + builder.Property(x => x.SourceFileName).HasMaxLength(512); + builder.Property(x => x.NormalizedSourceFileName).HasMaxLength(512); + builder.Property(x => x.ShortSourceName).HasMaxLength(64); + builder.Property(x => x.BaselineFamily).HasMaxLength(128); + + builder.HasIndex(x => new { x.ArchitectureFamilyId, x.RuntimeBaselineId }).IsUnique(); + builder.HasIndex(x => new { x.ArchitectureFamilyId, x.NormalizedCanonicalKey }).IsUnique(); + builder.HasIndex(x => new { x.ArchitectureFamilyId, x.NormalizedSourceRepository, x.NormalizedSourceFileName }).IsUnique(); + builder.HasIndex(x => new { x.RuntimeBaselineId, x.ArchitectureFamilyId }); + builder.HasIndex(x => x.IsActiveInCurrentConfig); + + builder.HasOne(x => x.ArchitectureFamily) + .WithMany() + .HasForeignKey(x => x.ArchitectureFamilyId) + .OnDelete(DeleteBehavior.Cascade); + } +} diff --git a/src/MQ.DB/Models/DbModels/BenchmarkRun.cs b/src/MQ.DB/Models/DbModels/BenchmarkRun.cs new file mode 100644 index 0000000..96a1b47 --- /dev/null +++ b/src/MQ.DB/Models/DbModels/BenchmarkRun.cs @@ -0,0 +1,105 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using MQ.DB.Interfaces; + +namespace MQ.DB.Models.DbModels; + +public class BenchmarkRun : ISQLiteEntity +{ + public Guid Id { get; set; } + + public int ArchitectureFamilyId { get; set; } + public ArchitectureFamily ArchitectureFamily { get; set; } = default!; + + public int TensorGroupProfileId { get; set; } + public TensorGroupProfile TensorGroupProfile { get; set; } = default!; + + public uint AiModelHashId { get; set; } + public AiModelHash AiModelHash { get; set; } = default!; + + public int? ImatrixDefinitionId { get; set; } + public ImatrixDefinition? ImatrixDefinition { get; set; } + + public Guid TensorComboId { get; set; } + public TensorCombo TensorCombo { get; set; } = default!; + + public Guid AiBenchmarkId { get; set; } + public AiBenchmark AiBenchmark { get; set; } = default!; + + /// + /// Nullable until the CategoryBenchmark row is created/persisted. + /// + public Guid? CategoryBenchmarkId { get; set; } + public CategoryBenchmark? CategoryBenchmark { get; set; } + + /// + /// Snapshot of the category for convenience and resilience. + /// Stored as the byte value of BenchmarkCategory. + /// + public byte Category { get; set; } + + public DateTime StartedUtc { get; set; } + public DateTime CompletedUtc { get; set; } + + public long DurationMs { get; set; } + + public bool Succeeded { get; set; } + + public string? Error { get; set; } + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + + builder.Property(x => x.Id) + .ValueGeneratedNever(); + + builder.HasIndex(x => x.ArchitectureFamilyId); + builder.HasIndex(x => x.TensorGroupProfileId); + builder.HasIndex(x => x.AiModelHashId); + builder.HasIndex(x => x.ImatrixDefinitionId); + builder.HasIndex(x => x.TensorComboId); + builder.HasIndex(x => x.AiBenchmarkId); + builder.HasIndex(x => x.CategoryBenchmarkId); + builder.HasIndex(x => x.StartedUtc); + builder.HasIndex(x => new { x.AiBenchmarkId, x.Category }); + + builder.Property(x => x.Error) + .HasMaxLength(4000); + + builder.HasOne(x => x.ArchitectureFamily) + .WithMany() + .HasForeignKey(x => x.ArchitectureFamilyId) + .OnDelete(DeleteBehavior.Cascade); + + builder.HasOne(x => x.TensorGroupProfile) + .WithMany() + .HasForeignKey(x => x.TensorGroupProfileId) + .OnDelete(DeleteBehavior.Restrict); + + builder.HasOne(x => x.AiModelHash) + .WithMany() + .HasForeignKey(x => x.AiModelHashId) + .OnDelete(DeleteBehavior.Cascade); + + builder.HasOne(x => x.ImatrixDefinition) + .WithMany() + .HasForeignKey(x => x.ImatrixDefinitionId) + .OnDelete(DeleteBehavior.Restrict); + + builder.HasOne(x => x.TensorCombo) + .WithMany() + .HasForeignKey(x => x.TensorComboId) + .OnDelete(DeleteBehavior.Restrict); + + builder.HasOne(x => x.AiBenchmark) + .WithMany() + .HasForeignKey(x => x.AiBenchmarkId) + .OnDelete(DeleteBehavior.Cascade); + + builder.HasOne(x => x.CategoryBenchmark) + .WithMany() + .HasForeignKey(x => x.CategoryBenchmarkId) + .OnDelete(DeleteBehavior.SetNull); + } +} diff --git a/src/MQ.DB/Models/DbModels/ExecutionPlanProbeCache.cs b/src/MQ.DB/Models/DbModels/ExecutionPlanProbeCache.cs new file mode 100644 index 0000000..d7cdf87 --- /dev/null +++ b/src/MQ.DB/Models/DbModels/ExecutionPlanProbeCache.cs @@ -0,0 +1,95 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using MQ.DB.Interfaces; + +namespace MQ.DB.Models.DbModels; + +public class ExecutionPlanProbeCache : ISQLiteEntity +{ + public Guid Id { get; set; } = Guid.NewGuid(); + + public int ArchitectureFamilyId { get; set; } + public ArchitectureFamily ArchitectureFamily { get; set; } = default!; + + public int TensorGroupProfileId { get; set; } + public TensorGroupProfile TensorGroupProfile { get; set; } = default!; + + public uint AiModelHashId { get; set; } + public AiModelHash AiModelHash { get; set; } = default!; + + public int? ImatrixDefinitionId { get; set; } + public ImatrixDefinition? ImatrixDefinition { get; set; } + + public string HardwareFingerprint { get; set; } = string.Empty; + public string QuantizedModelFingerprint { get; set; } = string.Empty; + public string QuantizationKey { get; set; } = string.Empty; + public int DiscoveryTokenTarget { get; set; } + + public int StaticNgl { get; set; } + public bool UsesGpu { get; set; } + public int GroupSize { get; set; } + public string SlotsJson { get; set; } = "[]"; + + public int ProbeSchemaVersion { get; set; } = 2; + public ulong Q8ModelSizeBytes { get; set; } + public int Q8StableNgl { get; set; } + public ulong NativeModelSizeBytes { get; set; } + public int NativeStableNgl { get; set; } + public string NativeQuantizationKey { get; set; } = string.Empty; + public int MaxCandidateNgl { get; set; } + public string GpuMemoryLimitsJson { get; set; } = "{}"; + public string TensorSplitJson { get; set; } = "{}"; + + public DateTime CreatedUtc { get; set; } = DateTime.UtcNow; + public DateTime UpdatedUtc { get; set; } = DateTime.UtcNow; + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + builder.Property(x => x.Id).ValueGeneratedNever(); + + builder.Property(x => x.HardwareFingerprint).HasMaxLength(1024); + builder.Property(x => x.QuantizedModelFingerprint).HasMaxLength(2048); + builder.Property(x => x.QuantizationKey).HasMaxLength(128); + builder.Property(x => x.NativeQuantizationKey).HasMaxLength(128); + builder.Property(x => x.SlotsJson).HasMaxLength(8000); + builder.Property(x => x.GpuMemoryLimitsJson).HasMaxLength(4000); + builder.Property(x => x.TensorSplitJson).HasMaxLength(4000); + + builder.HasIndex(x => x.ArchitectureFamilyId); + builder.HasIndex(x => x.TensorGroupProfileId); + builder.HasIndex(x => x.AiModelHashId); + builder.HasIndex(x => x.ImatrixDefinitionId); + builder.HasIndex(x => new + { + x.ArchitectureFamilyId, + x.TensorGroupProfileId, + x.AiModelHashId, + x.ImatrixDefinitionId, + x.HardwareFingerprint, + x.QuantizedModelFingerprint, + x.QuantizationKey, + x.DiscoveryTokenTarget + }).IsUnique(); + + builder.HasOne(x => x.ArchitectureFamily) + .WithMany() + .HasForeignKey(x => x.ArchitectureFamilyId) + .OnDelete(DeleteBehavior.Cascade); + + builder.HasOne(x => x.TensorGroupProfile) + .WithMany() + .HasForeignKey(x => x.TensorGroupProfileId) + .OnDelete(DeleteBehavior.Restrict); + + builder.HasOne(x => x.AiModelHash) + .WithMany() + .HasForeignKey(x => x.AiModelHashId) + .OnDelete(DeleteBehavior.Cascade); + + builder.HasOne(x => x.ImatrixDefinition) + .WithMany() + .HasForeignKey(x => x.ImatrixDefinitionId) + .OnDelete(DeleteBehavior.Restrict); + } +} diff --git a/src/MQ.DB/Models/DbModels/ImatrixDefinition.cs b/src/MQ.DB/Models/DbModels/ImatrixDefinition.cs new file mode 100644 index 0000000..23945f0 --- /dev/null +++ b/src/MQ.DB/Models/DbModels/ImatrixDefinition.cs @@ -0,0 +1,35 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using MQ.DB.Interfaces; + +namespace MQ.DB.Models.DbModels; + +public class ImatrixDefinition : ISQLiteEntity +{ + public int Id { get; set; } + public uint AiModelHashId { get; set; } + public AiModelHash AiModelHash { get; set; } = default!; + public string IdentityHash { get; set; } = string.Empty; + public string? CanonicalPath { get; set; } + public string SourceKind { get; set; } = "none"; + public DateTime CreatedUtc { get; set; } = DateTime.UtcNow; + public string? MetadataJson { get; set; } + public int? TokenCount { get; set; } + public string? BuildFingerprint { get; set; } + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + builder.HasIndex(x => new { x.AiModelHashId, x.IdentityHash }).IsUnique(); + builder.Property(x => x.IdentityHash).HasMaxLength(128); + builder.Property(x => x.CanonicalPath).HasMaxLength(2048); + builder.Property(x => x.SourceKind).HasMaxLength(64); + builder.Property(x => x.MetadataJson).HasMaxLength(8000); + builder.Property(x => x.BuildFingerprint).HasMaxLength(512); + + builder.HasOne(x => x.AiModelHash) + .WithMany() + .HasForeignKey(x => x.AiModelHashId) + .OnDelete(DeleteBehavior.Cascade); + } +} \ No newline at end of file diff --git a/src/MQ.DB/Models/DbModels/LearnedBaselineTensorQuant.cs b/src/MQ.DB/Models/DbModels/LearnedBaselineTensorQuant.cs new file mode 100644 index 0000000..3568e36 --- /dev/null +++ b/src/MQ.DB/Models/DbModels/LearnedBaselineTensorQuant.cs @@ -0,0 +1,112 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using MQ.DB.Interfaces; + +namespace MQ.DB.Models.DbModels; + +public class LearnedBaselineTensorQuant : ISQLiteEntity +{ + public Guid Id { get; set; } = Guid.NewGuid(); + + public int ArchitectureFamilyId { get; set; } + public ArchitectureFamily ArchitectureFamily { get; set; } = default!; + + public int TensorGroupProfileId { get; set; } + public TensorGroupProfile TensorGroupProfile { get; set; } = default!; + + public int BaselineQuantDefinitionId { get; set; } + public BaselineQuantDefinition BaselineQuantDefinition { get; set; } = default!; + + public Guid TensorComboId { get; set; } + public TensorCombo TensorCombo { get; set; } = default!; + + public Guid AiBenchmarkId { get; set; } + public AiBenchmark AiBenchmark { get; set; } = default!; + + /// + /// Exact source model hash used when learning happened. Reuse is scoped by architecture family/profile. + /// + public uint AiModelHashId { get; set; } + public AiModelHash AiModelHash { get; set; } = default!; + + /// + /// Snapshot of the compact runtime id at learning time. + /// + public byte BaselineQuantId { get; set; } + + public string BaselineCanonicalKey { get; set; } = string.Empty; + public string BaselineSourceKind { get; set; } = string.Empty; + public string? BaselineSourceRepository { get; set; } + public string? BaselineSourceFileName { get; set; } + + public byte TensorWeightSchemeId { get; set; } + public byte TensorGroupId { get; set; } + + public string TensorName { get; set; } = string.Empty; + public string FinalQuantType { get; set; } = string.Empty; + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + builder.Property(x => x.Id).ValueGeneratedNever(); + + builder.Property(x => x.BaselineCanonicalKey).HasMaxLength(512).IsRequired(); + builder.Property(x => x.BaselineSourceKind).HasMaxLength(64).IsRequired(); + builder.Property(x => x.BaselineSourceRepository).HasMaxLength(256); + builder.Property(x => x.BaselineSourceFileName).HasMaxLength(512); + builder.Property(x => x.TensorName).HasMaxLength(512).IsRequired(); + builder.Property(x => x.FinalQuantType).HasMaxLength(32).IsRequired(); + + builder.HasIndex(x => new + { + x.ArchitectureFamilyId, + x.TensorGroupProfileId, + x.BaselineQuantDefinitionId, + x.TensorWeightSchemeId, + x.TensorName + }) + .IsUnique(); + + builder.HasIndex(x => new + { + x.ArchitectureFamilyId, + x.TensorGroupProfileId, + x.BaselineQuantDefinitionId, + x.TensorGroupId + }); + + builder.HasIndex(x => x.AiBenchmarkId); + builder.HasIndex(x => x.AiModelHashId); + builder.HasIndex(x => x.TensorComboId); + + builder.HasOne(x => x.ArchitectureFamily) + .WithMany() + .HasForeignKey(x => x.ArchitectureFamilyId) + .OnDelete(DeleteBehavior.Cascade); + + builder.HasOne(x => x.TensorGroupProfile) + .WithMany() + .HasForeignKey(x => x.TensorGroupProfileId) + .OnDelete(DeleteBehavior.Restrict); + + builder.HasOne(x => x.BaselineQuantDefinition) + .WithMany() + .HasForeignKey(x => x.BaselineQuantDefinitionId) + .OnDelete(DeleteBehavior.Restrict); + + builder.HasOne(x => x.TensorCombo) + .WithMany() + .HasForeignKey(x => x.TensorComboId) + .OnDelete(DeleteBehavior.Restrict); + + builder.HasOne(x => x.AiModelHash) + .WithMany() + .HasForeignKey(x => x.AiModelHashId) + .OnDelete(DeleteBehavior.Cascade); + + builder.HasOne(x => x.AiBenchmark) + .WithMany() + .HasForeignKey(x => x.AiBenchmarkId) + .OnDelete(DeleteBehavior.Cascade); + } +} diff --git a/src/MQ.DB/Models/DbModels/QuantizationRun.cs b/src/MQ.DB/Models/DbModels/QuantizationRun.cs new file mode 100644 index 0000000..7adec02 --- /dev/null +++ b/src/MQ.DB/Models/DbModels/QuantizationRun.cs @@ -0,0 +1,97 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using MQ.DB.Interfaces; + +namespace MQ.DB.Models.DbModels; + +public class QuantizationRun : ISQLiteEntity +{ + public Guid Id { get; set; } = Guid.NewGuid(); + + public int ArchitectureFamilyId { get; set; } + public ArchitectureFamily ArchitectureFamily { get; set; } = default!; + + public int TensorGroupProfileId { get; set; } + public TensorGroupProfile TensorGroupProfile { get; set; } = default!; + + public uint AiModelHashId { get; set; } + public AiModelHash AiModelHash { get; set; } = default!; + + public int? ImatrixDefinitionId { get; set; } + public ImatrixDefinition? ImatrixDefinition { get; set; } + + public Guid TensorComboId { get; set; } + public TensorCombo TensorCombo { get; set; } = default!; + + /// + /// Nullable because a quantization can fail before a benchmark row exists. + /// + public Guid? AiBenchmarkId { get; set; } + public AiBenchmark? AiBenchmark { get; set; } + + public DateTime StartedUtc { get; set; } + public DateTime CompletedUtc { get; set; } + + /// + /// Total wall clock duration in milliseconds. + /// + public long DurationMs { get; set; } + + public bool Succeeded { get; set; } + + public string? Error { get; set; } + + public string? OutputModelPath { get; set; } + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + + builder.Property(x => x.Id) + .ValueGeneratedNever(); + + builder.HasIndex(x => x.ArchitectureFamilyId); + builder.HasIndex(x => x.TensorGroupProfileId); + builder.HasIndex(x => x.AiModelHashId); + builder.HasIndex(x => x.ImatrixDefinitionId); + builder.HasIndex(x => x.TensorComboId); + builder.HasIndex(x => x.AiBenchmarkId); + builder.HasIndex(x => x.StartedUtc); + + builder.Property(x => x.Error) + .HasMaxLength(4000); + + builder.Property(x => x.OutputModelPath) + .HasMaxLength(2048); + + builder.HasOne(x => x.ArchitectureFamily) + .WithMany() + .HasForeignKey(x => x.ArchitectureFamilyId) + .OnDelete(DeleteBehavior.Cascade); + + builder.HasOne(x => x.TensorGroupProfile) + .WithMany() + .HasForeignKey(x => x.TensorGroupProfileId) + .OnDelete(DeleteBehavior.Restrict); + + builder.HasOne(x => x.AiModelHash) + .WithMany() + .HasForeignKey(x => x.AiModelHashId) + .OnDelete(DeleteBehavior.Cascade); + + builder.HasOne(x => x.ImatrixDefinition) + .WithMany() + .HasForeignKey(x => x.ImatrixDefinitionId) + .OnDelete(DeleteBehavior.Restrict); + + builder.HasOne(x => x.TensorCombo) + .WithMany() + .HasForeignKey(x => x.TensorComboId) + .OnDelete(DeleteBehavior.Restrict); + + builder.HasOne(x => x.AiBenchmark) + .WithMany() + .HasForeignKey(x => x.AiBenchmarkId) + .OnDelete(DeleteBehavior.SetNull); + } +} diff --git a/src/MQ.DB/Models/DbModels/TensorCombo.cs b/src/MQ.DB/Models/DbModels/TensorCombo.cs new file mode 100644 index 0000000..f4eb937 --- /dev/null +++ b/src/MQ.DB/Models/DbModels/TensorCombo.cs @@ -0,0 +1,60 @@ +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using MQ.DB.Interfaces; + +namespace MQ.DB.Models.DbModels; + +public class TensorCombo : ISQLiteEntity +{ + public TensorCombo() + { + } + + // Connection to TensorConfig + public TensorCombo(TensorConfig c) + { + BaseQuant = c.BaseQuant; + Embeddings = c.Embeddings; + LmHead = c.LmHead; + AttnQ = c.AttnQ; + AttnKV = c.AttnKV; + AttnOutput = c.AttnOutput; + FfnUpGate = c.FfnUpGate; + FfnDown = c.FfnDown; + MoeExperts = c.MoeExperts; + MoeRouter = c.MoeRouter; + } + + public Guid Id { get; set; } = Guid.NewGuid(); + public readonly byte BaseQuant; + public readonly byte Embeddings; + public readonly byte LmHead; + public readonly byte AttnQ; + public readonly byte AttnKV; + public readonly byte AttnOutput; + public readonly byte FfnUpGate; + public readonly byte FfnDown; + public readonly byte MoeExperts; + public readonly byte MoeRouter; + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + + builder.Property(x => x.Id) + .ValueGeneratedNever(); + + builder.HasIndex(x => new + { + x.BaseQuant, + x.Embeddings, + x.LmHead, + x.AttnQ, + x.AttnKV, + x.AttnOutput, + x.FfnUpGate, + x.FfnDown, + x.MoeExperts, + x.MoeRouter + }).IsUnique();; + } +} \ No newline at end of file diff --git a/src/MQ.DB/Models/DbModels/TensorGroupProfile.cs b/src/MQ.DB/Models/DbModels/TensorGroupProfile.cs new file mode 100644 index 0000000..7516401 --- /dev/null +++ b/src/MQ.DB/Models/DbModels/TensorGroupProfile.cs @@ -0,0 +1,30 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; +using MQ.DB.Interfaces; + +namespace MQ.DB.Models.DbModels; + +public class TensorGroupProfile : ISQLiteEntity +{ + public int Id { get; set; } + public int ArchitectureFamilyId { get; set; } + public ArchitectureFamily ArchitectureFamily { get; set; } = default!; + public string FingerprintHash { get; set; } = string.Empty; + public string SnapshotJson { get; set; } = string.Empty; + public DateTime CreatedUtc { get; set; } = DateTime.UtcNow; + public bool IsActive { get; set; } = true; + + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + builder.Property(x => x.FingerprintHash).HasMaxLength(128).IsRequired(); + builder.Property(x => x.SnapshotJson).IsRequired(); + builder.HasIndex(x => new { x.ArchitectureFamilyId, x.FingerprintHash }).IsUnique(); + builder.HasIndex(x => new { x.ArchitectureFamilyId, x.IsActive }); + + builder.HasOne(x => x.ArchitectureFamily) + .WithMany() + .HasForeignKey(x => x.ArchitectureFamilyId) + .OnDelete(DeleteBehavior.Cascade); + } +} diff --git a/src/MQ.DB/Models/HybridQuant.cs b/src/MQ.DB/Models/HybridQuant.cs new file mode 100644 index 0000000..cf1ce63 --- /dev/null +++ b/src/MQ.DB/Models/HybridQuant.cs @@ -0,0 +1,210 @@ +namespace MQ.DB.Models; + +public enum HybridTensorOverrideMode +{ + LearnedBaselineCandidate = 1, + ExactTensorScheme = 2, +} + +public class HybridQuant +{ + public BaselineQuants BaseQuant { get; set; } = default!; + public List Tensors { get; set; } = new(); + + public HybridQuant() { } + + public HybridQuant(TensorConfig c) + { + BaseQuant = BaselineQuants.FromId(c.BaseQuant); + + AddIfNotNull(TReg.Embeddings, c.Embeddings); + AddIfNotNull(TReg.LmHead, c.LmHead); + AddIfNotNull(TReg.AttnQ, c.AttnQ); + AddIfNotNull(TReg.AttnKV, c.AttnKV); + AddIfNotNull(TReg.AttnOutput, c.AttnOutput); + AddIfNotNull(TReg.FfnUpGate, c.FfnUpGate); + AddIfNotNull(TReg.FfnDown, c.FfnDown); + AddIfNotNull(TReg.MoeExperts, c.MoeExperts); + AddIfNotNull(TReg.MoeRouter, c.MoeRouter); + } + + private void AddIfNotNull(TensorGroup group, byte storedId) + { + if (BaselineQuants.IsNullTensorConfigGroupSlot(storedId)) + return; + + var decodedBaselineId = BaselineQuants.DecodeTensorConfigGroupSlotToBaselineId(storedId); + + if (BaselineQuants.IsNativeExactAlias(decodedBaselineId)) + { + var exactScheme = BaselineQuants.ResolveExactOverrideScheme(decodedBaselineId); + Tensors.Add(HybridTensor.CreateExact(group, exactScheme)); + return; + } + + var candidate = BaselineQuants.FromId(decodedBaselineId); + Tensors.Add(HybridTensor.CreateLearned(group, candidate)); + } + + public HybridQuant Clone() + { + return new HybridQuant + { + BaseQuant = BaseQuant, + Tensors = Tensors + .Select(t => t.Clone()) + .ToList() + }; + } + + public HybridTensor? TryGetTensor(TensorGroup group) => + Tensors.FirstOrDefault(x => x.TGroup.UniqueId == group.UniqueId); + + public HybridTensor GetRequiredTensor(TensorGroup group) => + TryGetTensor(group) ?? throw new InvalidOperationException($"HybridQuant does not contain group '{group.Name}'."); + + public void SetExactOverride(TensorGroup group, TensorWeightScheme exactScheme) + { + RemoveGroupIfPresent(group); + Tensors.Add(HybridTensor.CreateExact(group, exactScheme)); + } + + public void SetLearnedCandidateOverride(TensorGroup group, BaselineQuants candidateBaseline) + { + RemoveGroupIfPresent(group); + Tensors.Add(HybridTensor.CreateLearned(group, candidateBaseline)); + } + + public void RemoveGroupIfPresent(TensorGroup group) + { + Tensors.RemoveAll(x => x.TGroup.UniqueId == group.UniqueId); + } + + public static HybridQuant CreatePureBaseline(BaselineQuants baseQuant) + { + return new HybridQuant + { + BaseQuant = baseQuant, + Tensors = new List() + }; + } + + public static HybridQuant CreateExactBlanket( + BaselineQuants baseQuant, + IEnumerable groups, + TensorWeightScheme exactScheme) + { + return new HybridQuant + { + BaseQuant = baseQuant, + Tensors = groups + .Select(g => HybridTensor.CreateExact(g, exactScheme)) + .ToList() + }; + } + + public static HybridQuant CreateLearnedCandidateBlanket( + BaselineQuants baseQuant, + IEnumerable groups, + BaselineQuants candidateBaseline) + { + return new HybridQuant + { + BaseQuant = baseQuant, + Tensors = groups + .Select(g => HybridTensor.CreateLearned(g, candidateBaseline)) + .ToList() + }; + } + + public static explicit operator HybridQuant(TensorConfig c) => new HybridQuant(c); +} + +public class HybridTensor +{ + public TensorGroup TGroup { get; set; } = null!; + public HybridTensorOverrideMode OverrideMode { get; set; } + public BaselineQuants? CandidateBaseline { get; set; } + public TensorWeightScheme? ExactTensorScheme { get; set; } + public TensorWeightScheme MaterializedTensorScheme { get; set; } = default!; + + // Compatibility alias retained for older call sites. + public TensorWeightScheme TensorType + { + get => MaterializedTensorScheme; + set => MaterializedTensorScheme = value; + } + + public static HybridTensor CreateLearned(TensorGroup group, BaselineQuants candidateBaseline) + { + if (BaselineQuants.IsNativeExactAlias(candidateBaseline)) + { + throw new InvalidOperationException( + $"Baseline '{candidateBaseline.Names[0]}' is an exact/native alias and cannot be used as a learned baseline candidate."); + } + + return new HybridTensor + { + TGroup = group, + OverrideMode = HybridTensorOverrideMode.LearnedBaselineCandidate, + CandidateBaseline = candidateBaseline, + ExactTensorScheme = null, + MaterializedTensorScheme = candidateBaseline.DefaultTensorScheme ?? TensorWeightScheme.GetCurrentNativePrecisionScheme() + }; + } + + public static HybridTensor CreateExact(TensorGroup group, TensorWeightScheme exactScheme) + { + return new HybridTensor + { + TGroup = group, + OverrideMode = HybridTensorOverrideMode.ExactTensorScheme, + CandidateBaseline = null, + ExactTensorScheme = exactScheme, + MaterializedTensorScheme = exactScheme + }; + } + + public HybridTensor Clone() + { + return new HybridTensor + { + TGroup = TGroup, + OverrideMode = OverrideMode, + CandidateBaseline = CandidateBaseline, + ExactTensorScheme = ExactTensorScheme, + MaterializedTensorScheme = MaterializedTensorScheme + }; + } + + public void ValidateOrThrow() + { + if (TGroup == null) + throw new InvalidOperationException("HybridTensor.TGroup is required."); + + switch (OverrideMode) + { + case HybridTensorOverrideMode.LearnedBaselineCandidate: + if (CandidateBaseline == null) + throw new InvalidOperationException($"HybridTensor for group '{TGroup.Name}' is missing CandidateBaseline."); + + if (ExactTensorScheme != null) + throw new InvalidOperationException($"HybridTensor for group '{TGroup.Name}' cannot specify ExactTensorScheme when OverrideMode is LearnedBaselineCandidate."); + + if (BaselineQuants.IsNativeExactAlias(CandidateBaseline)) + throw new InvalidOperationException($"HybridTensor for group '{TGroup.Name}' cannot use exact/native alias '{CandidateBaseline.Names[0]}' as a learned baseline candidate."); + break; + + case HybridTensorOverrideMode.ExactTensorScheme: + if (ExactTensorScheme == null) + throw new InvalidOperationException($"HybridTensor for group '{TGroup.Name}' is missing ExactTensorScheme."); + + if (CandidateBaseline != null) + throw new InvalidOperationException($"HybridTensor for group '{TGroup.Name}' cannot specify CandidateBaseline when OverrideMode is ExactTensorScheme."); + break; + + default: + throw new InvalidOperationException($"HybridTensor for group '{TGroup.Name}' has unknown OverrideMode '{OverrideMode}'."); + } + } +} diff --git a/src/MQ.DB/Models/IsolationRules.cs b/src/MQ.DB/Models/IsolationRules.cs new file mode 100644 index 0000000..6b22ce9 --- /dev/null +++ b/src/MQ.DB/Models/IsolationRules.cs @@ -0,0 +1,34 @@ +namespace MQ.DB.Models; + +public static class IsolationRules +{ + /// + /// If the smallest explicit non-imatrix probe for a tensor group cannot shrink + /// the total model by at least this much versus the carrier-base-only sample, + /// we stop isolated explicit quant exploration for that group for this run. + /// + public const double MinimumIsolationReductionToContinue = 0.04d; + + /// + /// If a base-only baseline sample only shrinks the model by less than this versus + /// the native BF16/F16/F32 source, it can be disabled as a future combination baseline. + /// + public const double MinimumMeaningfulBaseOnlyReductionRatio = 0.01d; + + /// + /// Hard damage cutoff for isolated tensor options. + /// 5% = 0.05 ratio. + /// + public const double MaximumIsolationPplDeltaRatio = 0.05d; + + /// + /// Hard KLD cutoff for isolated tensor options. + /// + public const double MaximumIsolationKld = 0.10d; + + /// + /// Used when comparing floating-point metrics so near-identical values do not + /// cause unstable eliminations. + /// + public const double MetricComparisonEpsilon = 1e-9d; +} \ No newline at end of file diff --git a/src/MQ.DB/Models/LlamaBenchMetrics.cs b/src/MQ.DB/Models/LlamaBenchMetrics.cs new file mode 100644 index 0000000..f696689 --- /dev/null +++ b/src/MQ.DB/Models/LlamaBenchMetrics.cs @@ -0,0 +1,10 @@ +namespace MQ.DB.Models; + +public class LlamaBenchMetrics +{ + public string? LogPath { get; set; } + public string? Backend { get; set; } + public int? Ngl { get; set; } + public string? Test { get; set; } + public double? Tps { get; set; } +} \ No newline at end of file diff --git a/src/MQ.DB/Models/LlamaBinaries.cs b/src/MQ.DB/Models/LlamaBinaries.cs new file mode 100644 index 0000000..e1a8a36 --- /dev/null +++ b/src/MQ.DB/Models/LlamaBinaries.cs @@ -0,0 +1,37 @@ +using System.Runtime.InteropServices; + +namespace MQ.DB.Models; + +public class LlamaBinaries +{ + public string Bench { get; } + public string Ppl { get; } + public string Cli { get; } + + public LlamaBinaries(string? root) + { + var binDir = !string.IsNullOrWhiteSpace(Cache.LlamaBin) ? Cache.LlamaBin + : !string.IsNullOrWhiteSpace(root) ? Path.Combine(root, "build", "bin") + : throw new InvalidOperationException("Set the llama.cpp binary directory before constructing LlamaBinaries."); + Bench = Path.Combine(binDir, "llama-bench"); + Ppl = Path.Combine(binDir, "llama-perplexity"); + Cli = Path.Combine(binDir, "llama-cli"); + + // Windows check (.exe) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + Bench += ".exe"; Ppl += ".exe"; Cli += ".exe"; + } + } + + public void Validate() + { + var missing = new List(); + if (!File.Exists(Bench)) missing.Add(Bench); + if (!File.Exists(Ppl)) missing.Add(Ppl); + if (!File.Exists(Cli)) missing.Add(Cli); + + if (missing.Any()) + throw new FileNotFoundException($"Missing llama.cpp binaries:\n{string.Join("\n", missing)}"); + } +} diff --git a/src/MQ.DB/Models/PplMetrics.cs b/src/MQ.DB/Models/PplMetrics.cs new file mode 100644 index 0000000..234433f --- /dev/null +++ b/src/MQ.DB/Models/PplMetrics.cs @@ -0,0 +1,9 @@ +namespace MQ.DB.Models; + +public class PplMetrics +{ + public string? LogPath { get; set; } + public double Ppl { get; set; } + public double PplError { get; set; } + public double? Kld { get; set; } +} \ No newline at end of file diff --git a/src/MQ.DB/Models/RequiredSamplePlan.cs b/src/MQ.DB/Models/RequiredSamplePlan.cs new file mode 100644 index 0000000..470ef0c --- /dev/null +++ b/src/MQ.DB/Models/RequiredSamplePlan.cs @@ -0,0 +1,55 @@ +namespace MQ.DB.Models; + +public enum RequiredSampleKind +{ + PureBaseline = 1, + BaseOnlyIsolation = 2, + GroupIsolationProbe = 3, + GroupIsolationContinuation = 4 +} + +public sealed class RequiredSamplePlan +{ + public RequiredSampleKind Kind { get; set; } + public string Key { get; set; } = string.Empty; + public string Description { get; set; } = string.Empty; + public HybridQuant Quant { get; set; } = default!; + public byte? TargetGroupId { get; set; } + + // Legacy name kept for compatibility: this now stores the tested baseline-family candidate id. + public byte? TestedSchemeId { get; set; } + + public byte? TestedCandidateId + { + get => TestedSchemeId; + set => TestedSchemeId = value; + } + + public string? TestedCandidateCanonicalKey { get; set; } + public byte? TestedBaselineId { get; set; } + public string? TestedBaselineCanonicalKey { get; set; } + public bool IsSmallestProbe { get; set; } +} + +public sealed class RequiredSampleGenerationResult +{ + public List Plans { get; set; } = new(); + public int PureBaselineCount { get; set; } + public int BaseOnlyIsolationCount { get; set; } + public int GroupIsolationCount { get; set; } + public int TotalCount => Plans.Count; + + public RequiredSampleGenerationResult MergeWith(RequiredSampleGenerationResult other) + { + var merged = new RequiredSampleGenerationResult + { + PureBaselineCount = PureBaselineCount + other.PureBaselineCount, + BaseOnlyIsolationCount = BaseOnlyIsolationCount + other.BaseOnlyIsolationCount, + GroupIsolationCount = GroupIsolationCount + other.GroupIsolationCount + }; + + merged.Plans.AddRange(Plans); + merged.Plans.AddRange(other.Plans); + return merged; + } +} diff --git a/src/MQ.DB/Models/SystemInfo.cs b/src/MQ.DB/Models/SystemInfo.cs new file mode 100644 index 0000000..076858b --- /dev/null +++ b/src/MQ.DB/Models/SystemInfo.cs @@ -0,0 +1,37 @@ +namespace MQ.DB.Models; + +public enum GpuVendor +{ + Nvidia = 1, + Amd = 2, + Intel = 3, + Cpu = 4, + Unknown = 0 +} + +public class SystemInfo +{ + public List GpuInfo { get; set; } = new List(); + public double RamGb { get; set; } + public int ThreadCount { get; set; } +} + +public class GpuInfo +{ + public GpuVendor GpuVendor { get; set; } + public string GpuName { get; set; } = "Unknown"; + public double VramGb { get; set; } + public string? UniqueId { get; set; } +} + +public static class MagicConstants +{ + public const string MagicQuantFolder = "MagicQuant"; + public const string EnvName = "MagicQuant-Env"; + public const string LlamaRepoName = "llama.cpp"; + public const string SuccessJson = "install_success.json"; + + // Windows Python Embed URL + public const string WinPythonUrl = "https://www.python.org/ftp/python/3.12.3/python-3.12.3-embed-amd64.zip"; + public const string WinPythonZip = "python-3.12.3-embed.zip"; +} \ No newline at end of file diff --git a/src/MQ.DB/Models/TensorConfigs.cs b/src/MQ.DB/Models/TensorConfigs.cs new file mode 100644 index 0000000..59a8257 --- /dev/null +++ b/src/MQ.DB/Models/TensorConfigs.cs @@ -0,0 +1,95 @@ +using System; +using System.Runtime.InteropServices; + +namespace MQ.DB.Models; + +[StructLayout(LayoutKind.Sequential, Pack = 1)] +public readonly struct TensorConfig +{ + public readonly byte BaseQuant; + public readonly byte Embeddings; + public readonly byte LmHead; + public readonly byte AttnQ; + public readonly byte AttnKV; + public readonly byte AttnOutput; + public readonly byte FfnUpGate; + public readonly byte FfnDown; + public readonly byte MoeExperts; + public readonly byte MoeRouter; + + public TensorConfig( + byte baseQuant, + byte embeddings, + byte lmHead, + byte attnQ, + byte attnKV, + byte attnOutput, + byte ffnUpGate, + byte ffnDown, + byte moeExperts, + byte moeRouter) + { + BaseQuant = baseQuant; + Embeddings = embeddings; + LmHead = lmHead; + AttnQ = attnQ; + AttnKV = attnKV; + AttnOutput = attnOutput; + FfnUpGate = ffnUpGate; + FfnDown = ffnDown; + MoeExperts = moeExperts; + MoeRouter = moeRouter; + } + + // Converting constructor: HybridQuant -> TensorConfig + public TensorConfig(HybridQuant h) + : this( + baseQuant: checked((byte)h.BaseQuant.UniqueId), + embeddings: GetStoredIdOrDefault(h, TReg.Embeddings), + lmHead: GetStoredIdOrDefault(h, TReg.LmHead), + attnQ: GetStoredIdOrDefault(h, TReg.AttnQ), + attnKV: GetStoredIdOrDefault(h, TReg.AttnKV), + attnOutput: GetStoredIdOrDefault(h, TReg.AttnOutput), + ffnUpGate: GetStoredIdOrDefault(h, TReg.FfnUpGate), + ffnDown: GetStoredIdOrDefault(h, TReg.FfnDown), + moeExperts: GetStoredIdOrDefault(h, TReg.MoeExperts), + moeRouter: GetStoredIdOrDefault(h, TReg.MoeRouter)) + { } + + private static byte GetStoredIdOrDefault(HybridQuant h, TensorGroup group) + { + if (h.Tensors == null || h.Tensors.Count == 0) + return BaselineQuants.TensorConfigNullSlotValue; + + HybridTensor? found = null; + + for (int i = 0; i < h.Tensors.Count; i++) + { + var t = h.Tensors[i]; + if (t?.TGroup == null) + continue; + + if (t.TGroup.UniqueId != group.UniqueId) + continue; + + if (found != null) + throw new InvalidOperationException($"HybridQuant contains duplicate entries for group '{group.Name}' (UniqueId={group.UniqueId})."); + + found = t; + } + + if (found == null) + return BaselineQuants.TensorConfigNullSlotValue; + + found.ValidateOrThrow(); + + return found.OverrideMode switch + { + HybridTensorOverrideMode.LearnedBaselineCandidate => BaselineQuants.EncodeTensorConfigGroupSlot(found.CandidateBaseline!), + HybridTensorOverrideMode.ExactTensorScheme => BaselineQuants.EncodeTensorConfigGroupSlot(found.ExactTensorScheme!), + _ => throw new InvalidOperationException($"Unknown HybridTensorOverrideMode '{found.OverrideMode}'.") + }; + } + + public static explicit operator TensorConfig(HybridQuant h) => new TensorConfig(h); +} diff --git a/src/MQ.DB/Models/TensorGroup.cs b/src/MQ.DB/Models/TensorGroup.cs new file mode 100644 index 0000000..f5814a8 --- /dev/null +++ b/src/MQ.DB/Models/TensorGroup.cs @@ -0,0 +1,411 @@ +using System.Collections.Immutable; +using System.Text.RegularExpressions; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace MQ.DB.Models; + +public class TensorGroupInfo +{ + public TensorGroup Group { get; set; } = null!; +} + +/// +/// Represents a categorized group of tensors with a unique name and matching patterns. +/// +/// Important: +/// The group identity itself is intentionally owned by C#. +/// The regex patterns are loaded from tensor_groups.yaml. +/// +/// This keeps MagicQuant's benchmark identity stable while allowing tensor-name +/// matching rules to evolve without recompiling the application. +/// +public record TensorGroup(byte UniqueId, string Name, ImmutableArray Tensors) +{ + /// + /// Helper to map the group name to a single-character identifier for CLI or UI display. + /// + public char ShortCode => Name switch + { + "embeddings" => 'E', + "lm_head" => 'H', + "attn_q" => 'Q', + "attn_kv" => 'K', + "attn_output" => 'O', + "ffn_up_gate" => 'U', + "ffn_down" => 'D', + "moe_experts" => 'X', + "moe_router" => 'R', + _ => '?' + }; +} + +/// +/// Tensor Registry. +/// +/// The semantic tensor groups are fixed here on purpose. +/// The matching regex patterns are loaded from tensor_groups.yaml and cached on first use. +/// +/// Design rule: +/// MagicQuant should not silently invent tensor-group behavior at runtime. +/// New groups should be deliberate architecture/benchmark decisions. +/// Pattern changes, however, are config/schema-level changes and belong in YAML. +/// +public static class TReg +{ + private const string DefaultYamlFileName = "tensor_groups.yaml"; + + private static readonly object CacheLock = new(); + + private static TensorGroupYamlFile? _yamlCache; + + private static readonly Dictionary GroupCache = + new(StringComparer.OrdinalIgnoreCase); + + private static readonly Dictionary> GroupRegexCache = + new(StringComparer.OrdinalIgnoreCase); + + private static ImmutableArray? _baseQuantExceptionPatternsCache; + + private static ImmutableArray? _baseQuantExceptionRegexCache; + + /// + /// Future extension point. + /// + /// Right now this can remain null and the loader will resolve tensor_groups.yaml + /// from the application output directory. + /// + /// Later, CLI/config code can set this before first access if users provide + /// an override location. + /// + public static string? TensorGroupsYamlPathOverride { get; set; } + + public static TensorGroup Embeddings => GetRequiredGroup(0, "embeddings"); + + public static TensorGroup LmHead => GetRequiredGroup(1, "lm_head"); + + public static TensorGroup AttnQ => GetRequiredGroup(2, "attn_q"); + + public static TensorGroup AttnKV => GetRequiredGroup(3, "attn_kv"); + + public static TensorGroup AttnOutput => GetRequiredGroup(4, "attn_output"); + + public static TensorGroup FfnUpGate => GetRequiredGroup(5, "ffn_up_gate"); + + public static TensorGroup FfnDown => GetRequiredGroup(6, "ffn_down"); + + public static TensorGroup MoeExperts => GetRequiredGroup(7, "moe_experts"); + + public static TensorGroup MoeRouter => GetRequiredGroup(8, "moe_router"); + + /// + /// Provides a complete list of all registered tensor groups. + /// + /// This remains fixed by design. The regex patterns inside each group come + /// from tensor_groups.yaml. + /// + public static ImmutableArray All => + [ + Embeddings, + LmHead, + AttnQ, + AttnKV, + AttnOutput, + FfnUpGate, + FfnDown, + MoeExperts, + MoeRouter + ]; + + /// + /// Look up a group by its string name, useful when parsing external configs. + /// + public static TensorGroup? GetByName(string name) => + All.FirstOrDefault(g => g.Name.Equals(name, StringComparison.OrdinalIgnoreCase)); + + /// + /// Gets compiled regexes for a semantic tensor group. + /// + /// This is useful when matching many tensors repeatedly and avoids recompiling + /// the same patterns over and over. + /// + public static ImmutableArray GetRegexesForGroup(TensorGroup group) + { + lock (CacheLock) + { + if (GroupRegexCache.TryGetValue(group.Name, out var cached)) + return cached; + + var regexes = group.Tensors + .Select(CreateRegex) + .ToImmutableArray(); + + GroupRegexCache[group.Name] = regexes; + return regexes; + } + } + + /// + /// Gets regex patterns for tensors that are explicitly allowed to fall back to BaseQuant. + /// + /// These are not semantic tensor groups. They are only checked after a tensor fails to + /// match any registered semantic group. + /// + public static ImmutableArray GetBaseQuantExceptionPatterns() + { + lock (CacheLock) + { + if (_baseQuantExceptionPatternsCache is not null) + return _baseQuantExceptionPatternsCache.Value; + + var yaml = LoadYamlIfNeeded(); + + var patterns = yaml.BaseQuantExceptions?.Patterns ?? []; + + var cleanedPatterns = patterns + .Where(p => !string.IsNullOrWhiteSpace(p)) + .Select(p => p.Trim()) + .Distinct(StringComparer.Ordinal) + .ToImmutableArray(); + + _baseQuantExceptionPatternsCache = cleanedPatterns; + return cleanedPatterns; + } + } + + /// + /// Gets compiled regexes for tensors that are explicitly allowed to fall back to BaseQuant. + /// + public static ImmutableArray GetBaseQuantExceptionRegexes() + { + lock (CacheLock) + { + if (_baseQuantExceptionRegexCache is not null) + return _baseQuantExceptionRegexCache.Value; + + var regexes = GetBaseQuantExceptionPatterns() + .Select(CreateRegex) + .ToImmutableArray(); + + _baseQuantExceptionRegexCache = regexes; + return regexes; + } + } + + /// + /// Returns true when the tensor is explicitly allowed to remain outside all semantic + /// tensor groups and fall back to the artifact BaseQuant. + /// + /// Important: + /// This should only be called after normal group matching returns zero matches. + /// It must not be used to resolve group collisions. + /// + public static bool IsBaseQuantException(string tensorName) + { + foreach (var regex in GetBaseQuantExceptionRegexes()) + { + if (regex.IsMatch(tensorName)) + return true; + } + + return false; + } + + /// + /// Finds all semantic tensor groups that match the provided tensor name. + /// + /// If this returns: + /// - 0 groups: caller may then check IsBaseQuantException. + /// - 1 group: tensor is safely categorized. + /// - 2+ groups: caller should treat this as an ambiguity/collision error. + /// + public static ImmutableArray FindMatchingGroups(string tensorName) + { + var matches = ImmutableArray.CreateBuilder(); + + foreach (var group in All) + { + var regexes = GetRegexesForGroup(group); + + foreach (var regex in regexes) + { + if (!regex.IsMatch(tensorName)) + continue; + + matches.Add(group); + break; + } + } + + return matches.ToImmutable(); + } + + /// + /// Clears the loaded YAML and materialized group/regex caches. + /// + /// This is mainly useful for tests or future reload behavior. + /// Normal production runs should not need to call this. + /// + public static void ClearCache() + { + lock (CacheLock) + { + _yamlCache = null; + GroupCache.Clear(); + GroupRegexCache.Clear(); + _baseQuantExceptionPatternsCache = null; + _baseQuantExceptionRegexCache = null; + } + } + + private static TensorGroup GetRequiredGroup(byte uniqueId, string name) + { + lock (CacheLock) + { + if (GroupCache.TryGetValue(name, out var cached)) + return cached; + + var yaml = LoadYamlIfNeeded(); + + if (yaml.Groups is null || yaml.Groups.Count == 0) + { + throw new InvalidOperationException( + $"Tensor group YAML did not define any groups. File: {ResolveTensorGroupsYamlPath()}"); + } + + if (!yaml.Groups.TryGetValue(name, out var groupDef)) + { + throw new InvalidOperationException( + $"Required tensor group '{name}' was not found in {ResolveTensorGroupsYamlPath()}."); + } + + if (groupDef.Patterns is null || groupDef.Patterns.Count == 0) + { + throw new InvalidOperationException( + $"Tensor group '{name}' exists in {ResolveTensorGroupsYamlPath()}, but it has no patterns."); + } + + var cleanedPatterns = groupDef.Patterns + .Where(p => !string.IsNullOrWhiteSpace(p)) + .Select(p => p.Trim()) + .Distinct(StringComparer.Ordinal) + .ToImmutableArray(); + + if (cleanedPatterns.Length == 0) + { + throw new InvalidOperationException( + $"Tensor group '{name}' exists in {ResolveTensorGroupsYamlPath()}, but all patterns were empty."); + } + + var group = new TensorGroup(uniqueId, name, cleanedPatterns); + GroupCache[name] = group; + + return group; + } + } + + private static TensorGroupYamlFile LoadYamlIfNeeded() + { + if (_yamlCache is not null) + return _yamlCache; + + var path = ResolveTensorGroupsYamlPath(); + + if (!File.Exists(path)) + { + throw new FileNotFoundException( + $"Could not find {DefaultYamlFileName}. Expected it at: {path}", + path); + } + + var yamlText = File.ReadAllText(path); + + if (string.IsNullOrWhiteSpace(yamlText)) + { + throw new InvalidOperationException( + $"Tensor group YAML file is empty: {path}"); + } + + var deserializer = new DeserializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .IgnoreUnmatchedProperties() + .Build(); + + TensorGroupYamlFile? parsed; + + try + { + parsed = deserializer.Deserialize(yamlText); + } + catch (Exception ex) + { + throw new InvalidOperationException( + $"Failed to parse tensor group YAML file: {path}", + ex); + } + + if (parsed is null) + { + throw new InvalidOperationException( + $"Tensor group YAML parsed to null: {path}"); + } + + if (parsed.SchemaVersion <= 0) + { + throw new InvalidOperationException( + $"Tensor group YAML must define a positive schema_version. File: {path}"); + } + + _yamlCache = parsed; + return _yamlCache; + } + + private static string ResolveTensorGroupsYamlPath() + { + if (!string.IsNullOrWhiteSpace(TensorGroupsYamlPathOverride)) + return Path.GetFullPath(TensorGroupsYamlPathOverride); + + return Path.Combine(AppContext.BaseDirectory, DefaultYamlFileName); + } + + private static Regex CreateRegex(string pattern) + { + try + { + return new Regex( + pattern, + RegexOptions.Compiled | + RegexOptions.CultureInvariant); + } + catch (Exception ex) + { + throw new InvalidOperationException( + $"Invalid tensor group regex pattern: {pattern}", + ex); + } + } + + private sealed class TensorGroupYamlFile + { + public int SchemaVersion { get; set; } + + public Dictionary Groups { get; set; } = + new(StringComparer.OrdinalIgnoreCase); + + public BaseQuantExceptionYamlDefinition? BaseQuantExceptions { get; set; } + } + + private sealed class TensorGroupYamlDefinition + { + public string? Description { get; set; } + + public List Patterns { get; set; } = []; + } + + private sealed class BaseQuantExceptionYamlDefinition + { + public string? Description { get; set; } + + public List Patterns { get; set; } = []; + } +} diff --git a/src/MQ.DB/Models/TensorGroupSynergy.cs b/src/MQ.DB/Models/TensorGroupSynergy.cs new file mode 100644 index 0000000..1c71b21 --- /dev/null +++ b/src/MQ.DB/Models/TensorGroupSynergy.cs @@ -0,0 +1,77 @@ +using System.Collections.Immutable; + +namespace MQ.DB.Models; + +/// +/// Represents a static semantic relationship between tensor groups that are +/// independently tracked but expected to behave as a connected optimization unit. +/// +/// A synergy group does not replace normal tensor group identity. +/// It exists to describe known architectural coupling between groups. +/// +public record TensorGroupSynergy( + byte UniqueId, + string Name, + ImmutableArray Groups) +{ + public bool Contains(TensorGroup group) => + Groups.Any(g => g.UniqueId == group.UniqueId); + + public bool Contains(string groupName) => + Groups.Any(g => g.Name.Equals(groupName, StringComparison.OrdinalIgnoreCase)); + + public bool ContainsAny(IEnumerable groups) + { + foreach (var group in groups) + { + if (Contains(group)) + return true; + } + + return false; + } + + public bool ContainsAll(IEnumerable groups) + { + foreach (var group in groups) + { + if (!Contains(group)) + return false; + } + + return true; + } +} + +/// +/// Static registry for tensor-group relationships that should receive synergy-aware +/// second-chance review after normal bad-trade pruning. +/// +/// Keep this intentionally small and explicit. Synergy is an architectural exception, +/// not a fuzzy runtime heuristic. +/// +public static class TensorGroupSynergies +{ + /// + /// Feed-forward projection groups are independently measurable, but their + /// downstream hybrid behavior can be coupled enough that a candidate surviving + /// in one group deserves a KLD-only second chance in the other when the only + /// normal bad-trade objection was PPL volatility. + /// + public static TensorGroupSynergy FeedForwardUpGateDown { get; } = new( + UniqueId: 0, + Name: "ffn_up_gate+ffn_down", + Groups: ImmutableArray.Create(TReg.FfnUpGate, TReg.FfnDown)); + + public static ImmutableArray All { get; } = + ImmutableArray.Create(FeedForwardUpGateDown); + + public static ImmutableArray GetContainingSynergies(TensorGroup group) => + All.Where(x => x.Contains(group)).ToImmutableArray(); + + public static bool IsInAnySynergy(TensorGroup group) => + All.Any(x => x.Contains(group)); + + public static bool AreInSameSynergy(TensorGroup left, TensorGroup right) => + All.Any(x => x.Contains(left) && x.Contains(right)); +} diff --git a/src/MQ.DB/Models/TensorWeight.cs b/src/MQ.DB/Models/TensorWeight.cs new file mode 100644 index 0000000..f306acb --- /dev/null +++ b/src/MQ.DB/Models/TensorWeight.cs @@ -0,0 +1,47 @@ +namespace MQ.DB.Models; + +public class TensorWeight +{ + public TensorWeight(byte uniqueId, bool requiresImatrix, string[] names) + { + Names = names.ToList(); + UniqueId = uniqueId; + RequiresImatrix = requiresImatrix; + } + + /// + /// + /// + /// Leave null for basically everything. Only provide Bf16 or F16 or + /// so on for those that are categorized together, which is Unique. + /// + /// + public string GetName(string? name = null) + { + if (Names != null && Names.Any()) + { + if (Names.Count == 1) + { + return Names.First(); + } + else if(Names.Count > 1 && !string.IsNullOrEmpty(name)) + { + return Names.First(x => x.Equals(name, StringComparison.InvariantCultureIgnoreCase)); + } + else + { + throw new Exception("Tensor Weight had more than one name, but provided override name was null or didn't match any stored."); + } + } + else + { + throw new Exception("No strings in the TensorWeight Names variable."); + } + } + + public List? Names { get; } + public byte UniqueId { get; } + + public bool RequiresImatrix { get; } + +} diff --git a/src/MQ.DB/Models/TensorWeightScheme.cs b/src/MQ.DB/Models/TensorWeightScheme.cs new file mode 100644 index 0000000..292e47c --- /dev/null +++ b/src/MQ.DB/Models/TensorWeightScheme.cs @@ -0,0 +1,295 @@ +using System.Collections.Immutable; +using MQ.DB; + +namespace MQ.DB.Models; + +public sealed class TensorWeightScheme +{ + public byte UniqueId { get; } + public bool RequiresImatrix { get; } + public ImmutableArray Names { get; } + public ushort? BlockNeo { get; } + public bool IsEligibleForBaseline { get; } + + private TensorWeightScheme( + byte uniqueId, + bool requiresImatrix, + ImmutableArray names, + ushort? blockNeo, + bool isEligibleForBaseline = true) + { + UniqueId = uniqueId; + RequiresImatrix = requiresImatrix; + Names = names; + BlockNeo = blockNeo; + IsEligibleForBaseline = isEligibleForBaseline; + + } + + public static void ValidateSmallestConfiguration() + { + var ordered = GetSmallestInOrder(); + + if (ordered.Length == 0) + throw new InvalidOperationException("TensorWeightScheme.GetSmallestInOrder() must return at least one item."); + + var duplicateIds = ordered + .GroupBy(x => x.UniqueId) + .Where(g => g.Count() > 1) + .Select(g => g.First().Names[0]) + .ToList(); + + if (duplicateIds.Count > 0) + { + throw new InvalidOperationException( + $"TensorWeightScheme.GetSmallestInOrder() contains duplicates: {string.Join(", ", duplicateIds)}"); + } + + var knownIds = All.Select(x => x.UniqueId).ToHashSet(); + var unknown = ordered + .Where(x => !knownIds.Contains(x.UniqueId)) + .Select(x => x.Names[0]) + .Distinct() + .ToList(); + + if (unknown.Count > 0) + { + throw new InvalidOperationException( + $"TensorWeightScheme.GetSmallestInOrder() includes unknown schemes: {string.Join(", ", unknown)}"); + } + } + + + /// + /// This labels the models that're supposed to quantize the smallest + /// in order. Top of array being the smallest, the further down, + /// it becomes larger in order of expected quantization size. + /// + /// + public static TensorWeightScheme[] GetSmallestInOrder() + { + return [IQ4_XS, IQ4_NL, Q4_K, Q6_K, Q8_0, BF16, F16, F32]; + } + + public static TensorWeightScheme GetCurrentNativePrecisionScheme() + { + return (Cache.TorchType ?? Cache.MainTorchType.BF16) switch + { + Cache.MainTorchType.BF16 => BF16, + Cache.MainTorchType.F16 => F16, + Cache.MainTorchType.F32 => F32, + _ => BF16 + }; + } + + public static bool IsNativePrecisionScheme(TensorWeightScheme scheme) + { + return scheme.UniqueId == BF16.UniqueId || + scheme.UniqueId == F16.UniqueId || + scheme.UniqueId == F32.UniqueId; + } + + + public static TensorWeightScheme FromId(byte id) + { + var found = All.FirstOrDefault(x => x.UniqueId == id); + if (found == null) + throw new InvalidOperationException($"Unknown tensor weight scheme id '{id}'."); + return found; + } + + // Compatibility shim for any older code still referencing BF16_F16. + public static TensorWeightScheme BF16_F16 => GetCurrentNativePrecisionScheme(); + + public static readonly TensorWeightScheme NULL = + new(0, false, ["NULL"], null, isEligibleForBaseline: false); + + public static readonly TensorWeightScheme BF16 = + new(1, false, ["BF16", "BFLOAT16"], null, isEligibleForBaseline: false); + + /*public static readonly TensorWeightScheme MXFP4 = + new( + 2, + false, + ["MXFP4"], + new[] + { + TReg.AttnQ, + TReg.MoeRouter, + TReg.MoeExperts + }, + 32, + isEligibleForBaseline: false);*/ + + public static readonly TensorWeightScheme Q8_0 = + new(3, false, ["Q8_0"], null); + + public static readonly TensorWeightScheme Q6_K = + new(4, false, ["Q6_K"], 256); + + public static readonly TensorWeightScheme Q5_K = + new(5, false, ["Q5_K"], 256); + + public static readonly TensorWeightScheme Q5_K_S = + new(18, false, ["Q5_K_S"], 256); + + public static readonly TensorWeightScheme IQ4_XS = + new(6, false, ["IQ4_XS"], 32); + + public static readonly TensorWeightScheme IQ4_NL = + new(7, false, ["IQ4_NL"], 32); + + public static readonly TensorWeightScheme IQ3_S = + new( + 8, + true, + ["IQ3_S"], + 32 + ); + + public static readonly TensorWeightScheme IQ3_XS = + new( + 9, + true, + ["IQ3_XS"], + 32 + ); + + public static readonly TensorWeightScheme IQ3_XXS = + new( + 10, + true, + ["IQ3_XXS"], + 32 + ); + + public static readonly TensorWeightScheme IQ2_S = + new( + 11, + true, + ["IQ2_S"], + 32 + ); + + public static readonly TensorWeightScheme IQ2_XS = + new( + 12, + true, + ["IQ2_XS"], + 32 + ); + + public static readonly TensorWeightScheme IQ2_XXS = + new( + 13, + true, + ["IQ2_XXS"], + 32 + ); + + public static readonly TensorWeightScheme IQ1_S = + new( + 20, + true, + ["IQ1_S"], + 256 + ); + + public static readonly TensorWeightScheme IQ1_M = + new( + 21, + true, + ["IQ1_M"], + 256 + ); + + public static readonly TensorWeightScheme Q4_K = + new( + 14, + false, + ["Q4_K"], + 32 + ); + + public static readonly TensorWeightScheme Q4_K_S = + new( + 17, + false, + ["Q4_K_S"], + 32 + ); + + public static readonly TensorWeightScheme MXFP4 = + new( + 19, + false, + ["MXFP4"], + 32 + ); + + + public static readonly TensorWeightScheme Q2_K = + new( + 13, + true, + ["Q2_K"], + 32 + ); + + public static readonly TensorWeightScheme F16 = + new(15, false, ["F16", "FLOAT16", "FP16", "HALF"], null, isEligibleForBaseline: false); + + public static readonly TensorWeightScheme F32 = + new(16, false, ["F32", "FLOAT32", "FP32", "FLOAT"], null, isEligibleForBaseline: false); + + // This is the set used by hybrid search / combination generation. + public static readonly ImmutableArray All_Allowed_Hybrid_Quants = + [ + NULL, + BF16, + //F16, + Q8_0, + Q6_K, + Q5_K, + IQ4_XS, + IQ4_NL, + MXFP4, + IQ3_S, + IQ3_XS, + IQ3_XXS, + IQ2_S, + IQ2_XS, + IQ2_XXS, + IQ1_S, + IQ1_M, + Q4_K, + Q4_K_S, + Q5_K_S + ]; + + // This is the true registry of everything known. + public static readonly ImmutableArray All = + [ + NULL, + BF16, + F16, + F32, + MXFP4, + Q8_0, + Q6_K, + Q5_K, + IQ4_XS, + IQ4_NL, + IQ3_S, + IQ3_XS, + IQ3_XXS, + IQ2_S, + IQ2_XS, + IQ2_XXS, + IQ1_S, + IQ1_M, + Q4_K, + Q4_K_S, + Q5_K_S + ]; +} diff --git a/src/MQ.DB/packages.lock.json b/src/MQ.DB/packages.lock.json new file mode 100644 index 0000000..dba06cc --- /dev/null +++ b/src/MQ.DB/packages.lock.json @@ -0,0 +1,365 @@ +{ + "version": 1, + "dependencies": { + "net10.0": { + "Microsoft.Data.Sqlite": { + "type": "Direct", + "requested": "[10.0.11, )", + "resolved": "10.0.11", + "contentHash": "7je7UELzm131GiLYc4PpZvfKXIgIyzPM+v+tjcd/nbnuWRfgcONYKzDTqJlURxwVCFsVnlpmq6y6yn4qvR8QXQ==", + "dependencies": { + "Microsoft.Data.Sqlite.Core": "10.0.11", + "SQLitePCLRaw.bundle_e_sqlite3": "2.1.12", + "SQLitePCLRaw.core": "2.1.12" + } + }, + "Microsoft.EntityFrameworkCore": { + "type": "Direct", + "requested": "[10.0.11, )", + "resolved": "10.0.11", + "contentHash": "VOSGU8en6HZJs8t7UMFN+9vGcRgVOOn6fA44Ngcg2NyvJ3P1KE94iAb0XzaVaGhXGtt+qaM/VtEn0/hzluQJeg==", + "dependencies": { + "Microsoft.EntityFrameworkCore.Abstractions": "10.0.11", + "Microsoft.EntityFrameworkCore.Analyzers": "10.0.11", + "Microsoft.Extensions.Caching.Memory": "10.0.11", + "Microsoft.Extensions.Logging": "10.0.11" + } + }, + "Microsoft.EntityFrameworkCore.Design": { + "type": "Direct", + "requested": "[10.0.11, )", + "resolved": "10.0.11", + "contentHash": "0zlzPs/jtrp2jGNZSxHLd0bRgDB/TlCDT17pnt8hovTVgCmC6qbX1277/goAlnZbRip1mZp1TVjjG+tj9PQ/Uw==", + "dependencies": { + "Humanizer.Core": "2.14.1", + "Microsoft.Build.Framework": "18.0.2", + "Microsoft.CodeAnalysis.CSharp": "5.0.0", + "Microsoft.CodeAnalysis.CSharp.Workspaces": "5.0.0", + "Microsoft.CodeAnalysis.Workspaces.MSBuild": "5.0.0", + "Microsoft.EntityFrameworkCore.Relational": "10.0.11", + "Microsoft.Extensions.Caching.Memory": "10.0.11", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.DependencyModel": "10.0.11", + "Microsoft.Extensions.Logging": "10.0.11", + "Mono.TextTemplating": "3.0.0", + "Newtonsoft.Json": "13.0.4" + } + }, + "Microsoft.EntityFrameworkCore.Sqlite": { + "type": "Direct", + "requested": "[10.0.11, )", + "resolved": "10.0.11", + "contentHash": "jc7iVrhQyInR3loraMESfEFaFOtQOB1mRKHjX6QYC9o7YDbfMNbAPnIwlpffnFwhXd6/27FKaaV+sWSoLd4F1g==", + "dependencies": { + "Microsoft.EntityFrameworkCore.Sqlite.Core": "10.0.11", + "Microsoft.Extensions.Caching.Memory": "10.0.11", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.DependencyModel": "10.0.11", + "Microsoft.Extensions.Logging": "10.0.11", + "SQLitePCLRaw.bundle_e_sqlite3": "2.1.12", + "SQLitePCLRaw.core": "2.1.12" + } + }, + "YamlDotNet": { + "type": "Direct", + "requested": "[17.0.1, )", + "resolved": "17.0.1", + "contentHash": "qVir5fehR/W5nTJyoJUibypETXaW4iRAF9cQa0FQIC9TJ3VC0qDOwm4o/RxANewj8KzPF8WMF2abBfUgi6LC4w==" + }, + "Humanizer.Core": { + "type": "Transitive", + "resolved": "2.14.1", + "contentHash": "lQKvtaTDOXnoVJ20ibTuSIOf2i0uO0MPbDhd1jm238I+U/2ZnRENj0cktKZhtchBMtCUSRQ5v4xBCUbKNmyVMw==" + }, + "Microsoft.Build.Framework": { + "type": "Transitive", + "resolved": "18.0.2", + "contentHash": "sOSb+0J4G/jCBW/YqmRuL0eOMXgfw1KQLdC9TkbvfA5xs7uNm+PBQXJCOzSJGXtZcZrtXozcwxPmUiRUbmd7FA==" + }, + "Microsoft.CodeAnalysis.Analyzers": { + "type": "Transitive", + "resolved": "3.11.0", + "contentHash": "v/EW3UE8/lbEYHoC2Qq7AR/DnmvpgdtAMndfQNmpuIMx/Mto8L5JnuCfdBYtgvalQOtfNCnxFejxuRrryvUTsg==" + }, + "Microsoft.CodeAnalysis.Common": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "ZXRAdvH6GiDeHRyd3q/km8Z44RoM6FBWHd+gen/la81mVnAdHTEsEkO5J0TCNXBymAcx5UYKt5TvgKBhaLJEow==", + "dependencies": { + "Microsoft.CodeAnalysis.Analyzers": "3.11.0" + } + }, + "Microsoft.CodeAnalysis.CSharp": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "5DSyJ9bk+ATuDy7fp2Zt0mJStDVKbBoiz1DyfAwSa+k4H4IwykAUcV3URelw5b8/iVbfSaOwkwmPUZH6opZKCw==", + "dependencies": { + "Microsoft.CodeAnalysis.Analyzers": "3.11.0", + "Microsoft.CodeAnalysis.Common": "[5.0.0]" + } + }, + "Microsoft.CodeAnalysis.CSharp.Workspaces": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "Al/Q8B+yO8odSqGVpSvrShMFDvlQdIBU//F3E6Rb0YdiLSALE9wh/pvozPNnfmh5HDnvU+mkmSjpz4hQO++jaA==", + "dependencies": { + "Humanizer.Core": "2.14.1", + "Microsoft.CodeAnalysis.Analyzers": "3.11.0", + "Microsoft.CodeAnalysis.CSharp": "[5.0.0]", + "Microsoft.CodeAnalysis.Common": "[5.0.0]", + "Microsoft.CodeAnalysis.Workspaces.Common": "[5.0.0]", + "System.Composition": "9.0.0" + } + }, + "Microsoft.CodeAnalysis.Workspaces.Common": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "ZbUmIvT6lqTNKiv06Jl5wf0MTMi1vQ1oH7ou4CLcs2C/no/L7EhP3T8y3XXvn9VbqMcJaJnEsNA1jwYUMgc5jg==", + "dependencies": { + "Humanizer.Core": "2.14.1", + "Microsoft.CodeAnalysis.Analyzers": "3.11.0", + "Microsoft.CodeAnalysis.Common": "[5.0.0]", + "System.Composition": "9.0.0" + } + }, + "Microsoft.CodeAnalysis.Workspaces.MSBuild": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "/G+LVoAGMz6Ae8nm+PGLxSw+F5RjYx/J7irbTO5uKAPw1bxHyQJLc/YOnpDxt+EpPtYxvC9wvBsg/kETZp1F9Q==", + "dependencies": { + "Humanizer.Core": "2.14.1", + "Microsoft.Build.Framework": "17.11.31", + "Microsoft.CodeAnalysis.Analyzers": "3.11.0", + "Microsoft.CodeAnalysis.Workspaces.Common": "[5.0.0]", + "Microsoft.Extensions.DependencyInjection": "9.0.0", + "Microsoft.Extensions.Logging": "9.0.0", + "Microsoft.Extensions.Logging.Abstractions": "9.0.0", + "Microsoft.Extensions.Options": "9.0.0", + "Microsoft.Extensions.Primitives": "9.0.0", + "Microsoft.VisualStudio.SolutionPersistence": "1.0.52", + "Newtonsoft.Json": "13.0.3", + "System.Composition": "9.0.0" + } + }, + "Microsoft.Data.Sqlite.Core": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "hubA20AGenQ4Sx0ElWaPpB8DISjXpdx463+1zOGRslsT0e/t/06ITv+pHsop8CcJ0d8PZLfgnT7juCDVD79Dkw==", + "dependencies": { + "SQLitePCLRaw.core": "2.1.12" + } + }, + "Microsoft.EntityFrameworkCore.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "6auJR+9+9VunznKfH7WGrHMrnrmA0F7JZ22EXzwXvVhjfnbu9Xq7NSIWaOf3KJsOanM2qf5ajJ2JR5TlcPZTLA==" + }, + "Microsoft.EntityFrameworkCore.Analyzers": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "Bv7X4wSSnzCQED9WYXKJ8fwgyvKwf0xZM1GO8xkf6CF9zl+UBnvjxmcPnokJRy0JKjc1SlHSzzhx1HcL4jitTQ==" + }, + "Microsoft.EntityFrameworkCore.Relational": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "grznnTJgEYxaWpdKAsTzg6j+89jHgCXWYp+QGtlX5O92+w/VuhWM6JLPYb+uw8M9VhGUvOTsO76dYOy9vNPd5Q==", + "dependencies": { + "Microsoft.EntityFrameworkCore": "10.0.11", + "Microsoft.Extensions.Caching.Memory": "10.0.11", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.Logging": "10.0.11" + } + }, + "Microsoft.EntityFrameworkCore.Sqlite.Core": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "ywTQKt32xnVhCzjEQAqFufpEyXkOUfvW/EC/s4xnS8Xaor2xXE+TMUyzhgACqXtZEU5IR95y94RDzHto55Fx7w==", + "dependencies": { + "Microsoft.Data.Sqlite.Core": "10.0.11", + "Microsoft.EntityFrameworkCore.Relational": "10.0.11", + "Microsoft.Extensions.Caching.Memory": "10.0.11", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.DependencyModel": "10.0.11", + "Microsoft.Extensions.Logging": "10.0.11", + "SQLitePCLRaw.core": "2.1.12" + } + }, + "Microsoft.Extensions.Caching.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "vUl798SmruTqqlt/xH2gDk3tJlhk6k3HdOXAHirlRfbNKDym4g/kRpUL9S4sl6F6FsOTOMW+ZsDapqlZMOOiEw==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Caching.Memory": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "el1g0mBEbDBGY2bT9mcSfrTWO8QlPdq2nOCnvQugioOFwHV+bVBMeiakoI0dNOdj8d6Hi9K6HY2xzRUWJiDR3w==", + "dependencies": { + "Microsoft.Extensions.Caching.Abstractions": "10.0.11", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11", + "Microsoft.Extensions.Logging.Abstractions": "10.0.11", + "Microsoft.Extensions.Options": "10.0.11", + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "fVi053xdpda9Em7vSkmgVxO/PtgC2m78ekReKWsgcyskqY0U82Bz/MONwxpGzI0hElYKJfw+fupqMVeKW3fSaA==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "PSmotV19c7E3lKed++uYo1kSiXFI+uTl37CBSrhq+CfLC3FCHjG7R91+xPnNehQfHS1b0Tzo/CCLPWH3qaEheg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "/a1aJz4m7ylhEDf25ugQChLQoN5XwoGjWw/BoR/ZWWKsO1v4DdJElS1uyngahz4B/eOzjFk1KNTkarRLE5wsIg==" + }, + "Microsoft.Extensions.DependencyModel": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "PJPtFYsZ+r+uz9qqXWUTEyKeJ1EiBGIJtqavkg9ZXijjGSFAk4Fgi5sqIxj+uAyLZwEKgexDUQXhWhvU6l3+og==" + }, + "Microsoft.Extensions.Logging": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "nUOJwgFkSiLHiVGFpU22pIJtuWYewuSYQ3JVuP/gdK8ASMT807Px+TYQiRWs6uSsOmoyFTaVCwKXTasczV6BpA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "10.0.11", + "Microsoft.Extensions.Logging.Abstractions": "10.0.11", + "Microsoft.Extensions.Options": "10.0.11" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "Ljd0Uxoq5XpScD2Bg0nM/r3mwx7Ao5Uq24eo2ARxbGvqJ7Zht6rt2cJtwVRH4Cv+1ZVMdXz6TB43KbpmsxRrvQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11" + } + }, + "Microsoft.Extensions.Options": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "eY1GAKcTfD2maP27J84X9IovT3yjHJ2dVDzPmDg6/XqYvt3jMzJhtfQCLjG9pVsZGAd+8DQ2QrjaDcs2+VQLGw==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11", + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "SXcz+kF+4Oo9b1+55zntpJFYfwb1jw66ioxptyNOOTDc8g2FHnBFWjZpsWfCvZIhzr0x+4e2trVTs4OKwQfBtw==" + }, + "Microsoft.VisualStudio.SolutionPersistence": { + "type": "Transitive", + "resolved": "1.0.52", + "contentHash": "oNv2JtYXhpdJrX63nibx1JT3uCESOBQ1LAk7Dtz/sr0+laW0KRM6eKp4CZ3MHDR2siIkKsY8MmUkeP5DKkQQ5w==" + }, + "Mono.TextTemplating": { + "type": "Transitive", + "resolved": "3.0.0", + "contentHash": "YqueG52R/Xej4VVbKuRIodjiAhV0HR/XVbLbNrJhCZnzjnSjgMJ/dCdV0akQQxavX6hp/LC6rqLGLcXeQYU7XA==", + "dependencies": { + "System.CodeDom": "6.0.0" + } + }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.4", + "contentHash": "pdgNNMai3zv51W5aq268sujXUyx7SNdE2bj1wZcWjAQrKMFZV260lbqYop1d2GM67JI1huLRwxo9ZqnfF/lC6A==" + }, + "SQLitePCLRaw.bundle_e_sqlite3": { + "type": "Transitive", + "resolved": "2.1.12", + "contentHash": "mAgscpQMLw5/nfA1Q5oJVAT29yROUo1ifZGbbTpx/lwZpSxMUGoYbKfmvdm8oXER+RzxqBmmQzeBEVKfeHv2nw==", + "dependencies": { + "SQLitePCLRaw.lib.e_sqlite3": "2.1.12", + "SQLitePCLRaw.provider.e_sqlite3": "2.1.12" + } + }, + "SQLitePCLRaw.core": { + "type": "Transitive", + "resolved": "2.1.12", + "contentHash": "ETpNw9DY3ckWLgRRAeCHj+GKOuPi61aeczkXhgHexUvqoZBAYg8RYESE2J7O1M7+o6QbdSEZwrw9bfqztUVWXg==" + }, + "SQLitePCLRaw.lib.e_sqlite3": { + "type": "Transitive", + "resolved": "2.1.12", + "contentHash": "fWi8Dbknuhgg72fWinIdjXVaqO1hHL4YBBwVLnr7e1c9TAZwJ0QE38j9syW1hwx6HaqEVTwI+O07WPdZn8Rp0w==" + }, + "SQLitePCLRaw.provider.e_sqlite3": { + "type": "Transitive", + "resolved": "2.1.12", + "contentHash": "W3oH4XIfCzFrgUSDKHhN6N+dgzA5YHOR2VxX8GB6Qy7CyrJJgxPEG8NirgYWlPQC5P2jz2knSsexWu4tDUL33g==", + "dependencies": { + "SQLitePCLRaw.core": "2.1.12" + } + }, + "System.CodeDom": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "CPc6tWO1LAer3IzfZufDBRL+UZQcj5uS207NHALQzP84Vp/z6wF0Aa0YZImOQY8iStY0A2zI/e3ihKNPfUm8XA==" + }, + "System.Composition": { + "type": "Transitive", + "resolved": "9.0.0", + "contentHash": "3Djj70fFTraOarSKmRnmRy/zm4YurICm+kiCtI0dYRqGJnLX6nJ+G3WYuFJ173cAPax/gh96REcbNiVqcrypFQ==", + "dependencies": { + "System.Composition.AttributedModel": "9.0.0", + "System.Composition.Convention": "9.0.0", + "System.Composition.Hosting": "9.0.0", + "System.Composition.Runtime": "9.0.0", + "System.Composition.TypedParts": "9.0.0" + } + }, + "System.Composition.AttributedModel": { + "type": "Transitive", + "resolved": "9.0.0", + "contentHash": "iri00l/zIX9g4lHMY+Nz0qV1n40+jFYAmgsaiNn16xvt2RDwlqByNG4wgblagnDYxm3YSQQ0jLlC/7Xlk9CzyA==" + }, + "System.Composition.Convention": { + "type": "Transitive", + "resolved": "9.0.0", + "contentHash": "+vuqVP6xpi582XIjJi6OCsIxuoTZfR0M7WWufk3uGDeCl3wGW6KnpylUJ3iiXdPByPE0vR5TjJgR6hDLez4FQg==", + "dependencies": { + "System.Composition.AttributedModel": "9.0.0" + } + }, + "System.Composition.Hosting": { + "type": "Transitive", + "resolved": "9.0.0", + "contentHash": "OFqSeFeJYr7kHxDfaViGM1ymk7d4JxK//VSoNF9Ux0gpqkLsauDZpu89kTHHNdCWfSljbFcvAafGyBoY094btQ==", + "dependencies": { + "System.Composition.Runtime": "9.0.0" + } + }, + "System.Composition.Runtime": { + "type": "Transitive", + "resolved": "9.0.0", + "contentHash": "w1HOlQY1zsOWYussjFGZCEYF2UZXgvoYnS94NIu2CBnAGMbXFAX8PY8c92KwUItPmowal68jnVLBCzdrWLeEKA==" + }, + "System.Composition.TypedParts": { + "type": "Transitive", + "resolved": "9.0.0", + "contentHash": "aRZlojCCGEHDKqh43jaDgaVpYETsgd7Nx4g1zwLKMtv4iTo0627715ajEFNpEEBTgLmvZuv8K0EVxc3sM4NWJA==", + "dependencies": { + "System.Composition.AttributedModel": "9.0.0", + "System.Composition.Hosting": "9.0.0", + "System.Composition.Runtime": "9.0.0" + } + } + } + } +} \ No newline at end of file diff --git a/src/MQ.DB/tensor_groups.yaml b/src/MQ.DB/tensor_groups.yaml new file mode 100644 index 0000000..0388c0d --- /dev/null +++ b/src/MQ.DB/tensor_groups.yaml @@ -0,0 +1,392 @@ +schema_version: 1 + +groups: + embeddings: + description: "Token embedding matrices." + patterns: + # Top-level GGUF / llama.cpp forms. + - "^token_embd\\.weight$" + + # Common HF / framework forms. + - "^model\\.embed_tokens\\.weight$" + - "^embed_tokens\\.weight$" + - "^tok_embeddings\\.weight$" + - "^word_embeddings\\.weight$" + - "^transformer\\.wte\\.weight$" + - "^wte\\.weight$" + + lm_head: + description: "Final output/logit projection tensors. Keep these anchored so blk.N.attn_output.weight never collides with lm_head." + patterns: + # Top-level GGUF output head only. + # Important: do NOT use unanchored output\.weight here, because it also + # matches blk.N.attn_output.weight when Regex.IsMatch is used. + - "^output\\.weight$" + + # Common HF / framework forms. + - "^lm_head\\.weight$" + - "^final_logits_proj\\.weight$" + - "^model\\.embed_out\\.weight$" + - "^lm_head\\.decoder\\.weight$" + + attn_q: + description: "Attention query projection tensors. Fused QKV tensors are assigned here as the representative attention-projection owner because MagicQuant currently has no separate attn_qkv group." + patterns: + # llama.cpp GGUF query form. + - "^blk\\..*\\.attn_q\\.weight$" + + # Qwen3.6 / hybrid attention form. + # This is a fused query/key/value tensor. It is material and should not be + # an exception. If MagicQuant later gains a dedicated attn_qkv group, move + # this pattern there. + - "^blk\\..*\\.attn_qkv\\.weight$" + + # LLaMA / Qwen / Mistral / modern HF forms. + - ".*q_proj.*weight$" + - ".*self_attn\\.q_proj\\.weight$" + + # BERT / encoder style forms. + - ".*query\\.weight$" + - ".*attention\\.self\\.query\\.weight$" + + # T5 / miscellaneous forms. + - ".*SelfAttention\\.q\\.weight$" + + # Fused QKV HF forms. + # These may intentionally match broader attention tensors and should be + # handled carefully by the learning/ambiguity layer. + - ".*c_attn\\.weight$" + - ".*query_key_value\\.weight$" + + attn_kv: + description: "Attention key/value projection tensors." + patterns: + # llama.cpp GGUF forms. + - "^blk\\..*\\.attn_k\\.weight$" + - "^blk\\..*\\.attn_v\\.weight$" + + # LLaMA / Qwen / Mistral / modern HF forms. + - ".*k_proj.*weight$" + - ".*v_proj.*weight$" + - ".*self_attn\\.k_proj\\.weight$" + - ".*self_attn\\.v_proj\\.weight$" + + # BERT / encoder style forms. + - ".*key\\.weight$" + - ".*value\\.weight$" + - ".*attention\\.self\\.key\\.weight$" + - ".*attention\\.self\\.value\\.weight$" + + # T5 / encoder-decoder style forms. + - ".*SelfAttention\\.k\\.weight$" + - ".*SelfAttention\\.v\\.weight$" + - ".*EncDecAttention\\.k\\.weight$" + - ".*EncDecAttention\\.v\\.weight$" + + attn_output: + description: "Attention output projection tensors." + patterns: + # llama.cpp GGUF form. + - "^blk\\..*\\.attn_output\\.weight$" + + # LLaMA / Qwen / Mistral / modern HF forms. + - ".*out_proj.*weight$" + - ".*o_proj.*weight$" + - ".*self_attn\\.out_proj\\.weight$" + + # GPT-style attention output. + # Note: c_proj can also appear in MLPs on some architectures, so this may + # require architecture-aware disambiguation or strict ambiguity reporting. + - ".*attn.*c_proj\\.weight$" + - ".*attention.*c_proj\\.weight$" + + # BERT / encoder style forms. + - ".*attention\\.output\\.dense\\.weight$" + - ".*self_attention\\.dense\\.weight$" + - ".*attention\\.proj\\.weight$" + + # T5 / miscellaneous forms. + - ".*SelfAttention\\.o\\.weight$" + + ffn_up_gate: + description: "Dense FFN up/gate tensors only. Expert-path tensors are intentionally excluded and should be owned by moe_experts." + patterns: + # llama.cpp GGUF dense FFN forms. + - "^blk\\..*\\.ffn_up\\.weight$" + - "^blk\\..*\\.ffn_gate\\.weight$" + + # BERT / encoder style forms. + - ".*intermediate\\.dense\\.weight$" + + # GPT / MLP style forms. + - ".*mlp.*c_fc\\.weight$" + - ".*mlp.*fc1\\.weight$" + - ".*fc1\\.weight$" + - ".*fc_in\\.weight$" + - ".*dense_h_to_4h\\.weight$" + + # T5 / gated FFN style forms. + - ".*wi\\.weight$" + - ".*wi_0\\.weight$" + - ".*wi_1\\.weight$" + - ".*DenseReluDense\\.wi_0\\.weight$" + - ".*DenseReluDense\\.wi_1\\.weight$" + + # LLaMA / Qwen / Mistral / modern dense MLP forms. + - ".*mlp\\.up_proj\\.weight$" + - ".*mlp\\.gate_proj\\.weight$" + + # Do NOT add expert/shared-expert patterns here. + # Qwen3.5 / Qwen3.6 expert forms such as: + # blk.N.ffn_up_exps.weight + # blk.N.ffn_gate_exps.weight + # blk.N.ffn_up_shexp.weight + # blk.N.ffn_gate_shexp.weight + # are MoE expert-path tensors and belong to moe_experts. + # GGUF MoE routed/shared expert FFN up/gate payloads. + - "^blk\\..*\\.ffn_up_exps\\.weight$" + - "^blk\\..*\\.ffn_gate_exps\\.weight$" + - "^blk\\..*\\.ffn_up_shexp\\.weight$" + - "^blk\\..*\\.ffn_gate_shexp\\.weight$" + + # HF / framework expert forms. + - ".*experts?\\..*wi_0.*" + - ".*experts?\\..*wi_1.*" + - ".*experts?\\..*fc1.*" + - ".*experts?\\..*dense_h_to_4h.*" + - ".*experts?\\..*up_proj.*" + - ".*experts?\\..*gate_proj.*" + - ".*mlp\\.experts\\.gate_up_proj.*" + - ".*mlp\\.experts\\.gate_proj.*" + - ".*mlp\\.experts\\.up_proj.*" + - ".*mlp\\.shared_expert\\.gate_proj\\.weight$" + - ".*mlp\\.shared_expert\\.up_proj\\.weight$" + - ".*layers\\..*\\.experts\\.gate_up_proj.*" + - ".*layers\\..*\\.experts\\.gate_proj.*" + - ".*layers\\..*\\.experts\\.up_proj.*" + + ffn_down: + description: "Dense FFN down-projection tensors only. Expert-path tensors are intentionally excluded and should be owned by moe_experts." + patterns: + # llama.cpp GGUF dense FFN form. + - "^blk\\..*\\.ffn_down\\.weight$" + + # BERT / encoder style form. + # Keep this scoped to avoid catching attention.output.dense if that should + # be owned by attn_output. + - ".*mlp.*output\\.dense\\.weight$" + - ".*ffn.*output\\.dense\\.weight$" + + # GPT / MLP style forms. + # c_proj can also appear as attention output on some architectures. + # Scope it toward MLP where possible to avoid attn_output ambiguity. + - ".*mlp.*c_proj\\.weight$" + - ".*mlp.*fc2\\.weight$" + - ".*fc2\\.weight$" + - ".*fc_out\\.weight$" + - ".*dense_4h_to_h\\.weight$" + + # T5 / gated FFN style forms. + - ".*wo\\.weight$" + - ".*DenseReluDense\\.wo\\.weight$" + + # LLaMA / Qwen / Mistral / modern dense MLP form. + - ".*mlp\\.down_proj\\.weight$" + + # Do NOT add expert/shared-expert patterns here. + # Qwen3.5 / Qwen3.6 expert forms such as: + # blk.N.ffn_down_exps.weight + # blk.N.ffn_down_shexp.weight + # are MoE expert-path tensors and belong to moe_experts. + # GGUF MoE routed/shared expert FFN down payloads. + - "^blk\\..*\\.ffn_down_exps\\.weight$" + - "^blk\\..*\\.ffn_down_shexp\\.weight$" + + # HF / framework expert forms. + - ".*experts?\\..*wo.*" + - ".*experts?\\..*fc2.*" + - ".*experts?\\..*dense_4h_to_h.*" + - ".*experts?\\..*down_proj.*" + - ".*mlp\\.experts\\.down_proj.*" + - ".*mlp\\.shared_expert\\.down_proj\\.weight$" + - ".*layers\\..*\\.experts\\.down_proj.*" + + moe_experts: + description: "MoE expert-path tensors, including routed experts and shared experts. These own *_exps and *_shexp forms so they do not collide with dense FFN groups." + patterns: + - ".*experts?\\..*" + # llama.cpp GGUF MoE routed expert tensors. + #- "^blk\\..*\\.ffn_.*expert.*$" + #- "^blk\\..*\\.ffn_.*exps.*$" + #- "^blk\\..*\\.ffn_up_exps\\.weight$" + #- "^blk\\..*\\.ffn_gate_exps\\.weight$" + #- "^blk\\..*\\.ffn_down_exps\\.weight$" + + # Qwen3.6 shared-expert GGUF tensors. + # These are material expert-path matrices and should not fall back through + # base_quant_exceptions. + #- "^blk\\..*\\.ffn_up_shexp\\.weight$" + #- "^blk\\..*\\.ffn_gate_shexp\\.weight$" + #- "^blk\\..*\\.ffn_down_shexp\\.weight$" + + # Generic expert container forms used by several HF architectures. + #- ".*experts?\\..*wi_0.*" + #- ".*experts?\\..*wi_1.*" + #- ".*experts?\\..*wo.*" + #- ".*experts?\\..*fc1.*" + #- ".*experts?\\..*fc2.*" + #- ".*experts?\\..*dense_h_to_4h.*" + #- ".*experts?\\..*dense_4h_to_h.*" + #- ".*experts?\\..*up_proj.*" + #- ".*experts?\\..*gate_proj.*" + #- ".*experts?\\..*down_proj.*" + + # Qwen3.5 / Qwen3.6 / modern HF MoE forms. + #- ".*mlp\\.experts\\.gate_up_proj.*" + #- ".*mlp\\.experts\\.gate_proj.*" + #- ".*mlp\\.experts\\.up_proj.*" + #- ".*mlp\\.experts\\.down_proj.*" + + # Shared experts are still MoE expert-path tensors, not normal dense FFN. + # Keeping them here prevents them from double-counting as generic FFN. + #- ".*mlp\\.shared_expert\\.gate_proj\\.weight$" + #- ".*mlp\\.shared_expert\\.up_proj\\.weight$" + #- ".*mlp\\.shared_expert\\.down_proj\\.weight$" + + # Gemma-style MoE forms. + #- ".*layers\\..*\\.experts\\.gate_up_proj.*" + #- ".*layers\\..*\\.experts\\.gate_proj.*" + #- ".*layers\\..*\\.experts\\.up_proj.*" + #- ".*layers\\..*\\.experts\\.down_proj.*" + + moe_router: + description: "MoE router/gating tensors. Keep this MoE-specific so dense FFN gates, attention gates, and Qwen3.6 hybrid/SSM gates do not masquerade as routers." + patterns: + # Top-level router/gating forms. + # These are intentionally anchored to names that begin with router/gating/routing. + - "^router.*" + - "^gating.*" + - "^routing.*" + + # Generic MoE-specific gate/router forms. + # Important: do NOT use a broad pattern like .*gate\.weight$ here. + # That catches dense FFN gates, attention gates, and hybrid SSM gates. + - ".*moe_gate\\.weight$" + - ".*moe\\.gate\\.weight$" + - ".*moe\\.router.*" + - ".*gating_network\\.weight$" + + # llama.cpp GGUF MoE router/gate forms. + # ffn_gate_inp is the router/gate input tensor, not the dense ffn_gate payload. + - "^blk\\..*\\.ffn_gate_inp\\.weight$" + - "^blk\\..*\\.gate_inp\\.weight$" + - "^blk\\..*\\.router.*" + - "^blk\\..*\\.router_fc.*" + + # Qwen3.6 shared-expert gate input. + # This appears alongside *_shexp expert tensors and is F32 in the failure + # log. Treat it as router/gating control, not as expert matrix payload. + - "^blk\\..*\\.ffn_gate_inp_shexp\\.weight$" + + # Qwen3.5 / Qwen3.6 native HF MoE router. + # Keep this scoped to mlp.gate, but be aware that some dense architectures + # may use gate_proj for ordinary gated MLP. gate_proj belongs to FFN/expert + # groups above, not this router group. + - ".*mlp\\.gate\\.weight$" + + # Gemma-style router tensors. + - ".*layers\\..*\\.router\\.proj\\.weight$" + - ".*layers\\..*\\.router\\.per_expert_scale$" + - ".*layers\\..*\\.router\\.scale$" + + # Do NOT add: + # - ".*gate\\.weight$" + # - ".*(? args) + { + if (args.Any(a => string.Equals(a.Name, "help", StringComparison.OrdinalIgnoreCase))) + { + ShowHelp(); + return; + } + + AnsiConsole.MarkupLine("[grey]build-hybrids now routes through the centralized learning/selection/export pipeline.[/]"); + await new QuantizationPipeline().Run(args); + } + + private static void ShowHelp() + { + AnsiConsole.MarkupLine("[bold yellow]Command: build-hybrids[/]"); + AnsiConsole.MarkupLine("Runs the centralized survival/export flow over the active MagicQuant pipeline."); + AnsiConsole.WriteLine("Usage: mq build-hybrids --model-dir \"\" [--config \"./config.default.yaml\"] [--output-dir \"\"] [--output-name-prefix \"Model\"] [--reuse-existing-final-artifacts] [--export-external-learned-baselines]"); + } +} diff --git a/src/MagicQuant/Commands/CloneRepositoryQuants.cs b/src/MagicQuant/Commands/CloneRepositoryQuants.cs new file mode 100644 index 0000000..18fdc18 --- /dev/null +++ b/src/MagicQuant/Commands/CloneRepositoryQuants.cs @@ -0,0 +1,1103 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using MagicQuant.Configuration; +using MagicQuant.Helpers; +using MagicQuant.Models; +using MagicQuant.Services; +using Microsoft.EntityFrameworkCore; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Commands; + +public sealed class CloneRepositoryQuants : ICommand +{ + private static readonly string[] ModelAdjacentFiles = + [ + "generation_config.json", + "config.json", + "chat_template.jinja", + "added_tokenizer.json", + "LICENSE", + "merges.txt", + "model.safetensors.index.json", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json" + ]; + + private static readonly string[] CloneBenchmarkDomains = ["general"]; + private const string CloneTensorPolicyPropertyName = "cloneTensorPolicy"; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + WriteIndented = true, + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true + }; + + public async Task Run(List args) + { + if (args.Any(a => string.Equals(a.Name, "help", StringComparison.OrdinalIgnoreCase))) + { + ShowHelp(); + return; + } + + bool allowMissingManifestTensors = args.Any(a => string.Equals(a.Name, CloneManifestTensorMapBuildService.AllowMissingManifestTensorsFlag, StringComparison.OrdinalIgnoreCase)); + string? missingManifestBaseQuantName = Get(args, CloneManifestTensorMapBuildService.MissingManifestBaseQuantFlag); + bool hasMissingManifestBaseQuantOverride = !string.IsNullOrWhiteSpace(missingManifestBaseQuantName); + + string? modelDirRaw = Get(args, "model-dir"); + if (string.IsNullOrWhiteSpace(modelDirRaw)) + modelDirRaw = Config.Current.Paths.ModelDir; + + if (string.IsNullOrWhiteSpace(modelDirRaw)) + throw new InvalidOperationException("Clone mode requires --model-dir or paths.model_dir in YAML."); + + string fullModelPath = Path.GetFullPath(modelDirRaw); + if (!Directory.Exists(fullModelPath)) + throw new DirectoryNotFoundException($"Model directory does not exist: {fullModelPath}"); + + var safeTensorFiles = Directory.GetFiles(fullModelPath, "*.safetensors", SearchOption.TopDirectoryOnly); + if (safeTensorFiles.Length == 0) + throw new InvalidOperationException($"No .safetensors files were found in model directory: {fullModelPath}"); + + Cache.ModelDirectory = fullModelPath; + Cache.ModelMagicQuantDirectory = Path.Combine(fullModelPath, "MagicQuant"); + ModelRuntimePathService.InitializeForCurrentModel(); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await new ExternalBaselineCacheCleanupService().CleanupStaleArtifactsAsync(); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await new ScratchStorageService(new ModelArtifactPathService()).CleanupStaleScratchArtifactsAsync(); + Cache.ForceRefreshHardwareProbe = Config.Current.Flags.ForceRefreshHardwareProbe; + Cache.UseImatrix = Config.Current.Flags.UseImatrix; + Cache.ForceImatrixRebuild = Config.Current.Flags.ForceImatrixRebuild; + Cache.SuppressBenchmarkPersistence = true; + + RuntimeSearchSpace.ResetForNewModel(); + RuntimeSearchSpace.SetImatrixAvailability(false); + RuntimeSearchSpace.AllowHighPrecisionHybrids = Config.Current.Flags.AllowHighPrecisionHybrids; + + JsonHelper.DetectAndSetTorchType(Cache.ModelDirectory); + Directory.CreateDirectory(Cache.ModelMagicQuantDirectory); + + Cache.OutputDirectory = ResolveAndValidateOutputDirectory(args); + + AnsiConsole.Write(new Rule("[yellow]Repository Quant Clone Mode[/]") { Justification = Justify.Left }); + AnsiConsole.MarkupLine($"Model Path: [blue]{Markup.Escape(Cache.ModelDirectory)}[/]"); + AnsiConsole.MarkupLine($"Work Path: [blue]{Markup.Escape(Cache.ModelMagicQuantDirectory)}[/]"); + AnsiConsole.MarkupLine($"Export Path: [blue]{Markup.Escape(Cache.OutputDirectory ?? "n/a")}[/]"); + AnsiConsole.MarkupLine($"Reuse final artifacts: {(Config.ReuseExistingFinalArtifacts ? "[green]yes[/]" : "[grey]no[/]")}"); + AnsiConsole.MarkupLine($"Allow missing manifest tensors: {(allowMissingManifestTensors || hasMissingManifestBaseQuantOverride ? "[yellow]yes[/]" : "[grey]no[/]")}"); + AnsiConsole.MarkupLine(hasMissingManifestBaseQuantOverride + ? $"Missing-manifest base quant override: [yellow]{Markup.Escape(missingManifestBaseQuantName!)}[/]" + : "Missing-manifest base quant override: [grey]none[/]"); + + AnsiConsole.MarkupLine("Getting safetensors hash. This may take a bit, please wait..."); + Cache.CurrentModelId = MagicQuantModelId.GetOrCreateModelId(Cache.ModelDirectory); + AnsiConsole.MarkupLine($"[green]Model ID Created/Found:[/] [cyan]{Markup.Escape(Cache.CurrentModelId)}[/]"); + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await EnsureSqliteReadyAsync(); + + var pyManager = new PythonManager(Cache.MagicQuantDirectory!); + var hf = new HuggingFaceBaselineService(pyManager); + var manifestService = new RepositoryCloneManifestService(hf); + + string? sourceRepo = Get(args, "source-repo") ?? Get(args, "clone-repo"); + string? sourceJson = Get(args, "source-json") ?? Get(args, "clone-json"); + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + var (manifest, manifestLocalPath, sourceDescription) = await manifestService.ResolveAsync( + sourceRepo, + sourceJson, + Cache.ModelMagicQuantDirectory!, + CancellationToken.None); + + var sourceClonePolicies = LoadCloneArtifactPolicies(manifestLocalPath); + + var benchmarkService = new BenchmarkService(pyManager); + var quantizationService = new QuantizationService(benchmarkService); + var imatrixService = new ImatrixService(); + var cloneBuildService = new CloneManifestTensorMapBuildService(quantizationService, imatrixService); + + string baseModelGgufPath = await quantizationService.EnsureBaseModelFileAsync(true); + + var sidecarService = new ModelSidecarArtifactService(pyManager); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await sidecarService.EnsureMmprojArtifactAvailableAsync(); + + var architectureFamilyService = new ArchitectureFamilyService(pyManager); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await architectureFamilyService.EnsureCurrentArchitectureFamilyAsync(baseModelGgufPath); + + var tensorGroupProfileService = new TensorGroupProfileService(); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await tensorGroupProfileService.EnsureCurrentProfileAsync(); + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + var resolvedCustomBaselines = await hf.PrecheckAndRegisterConfiguredBaselinesAsync(); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await new TargetedRelearnService().PlanConfirmAndExecuteAsync(resolvedCustomBaselines); + + var imatrixRequest = new ImatrixRequest + { + UseImatrix = Cache.UseImatrix, + ForceRebuild = Cache.ForceImatrixRebuild, + ImatrixUrl = Config.Current.Imatrix.ImatrixUrl, + DatasetRepo = Config.Current.Imatrix.DatasetRepo, + DatasetSplit = Config.Current.Imatrix.DatasetSplit, + DatasetConfig = Config.Current.Imatrix.DatasetConfig, + LocalDatasetFile = Config.Current.Imatrix.DatasetLocalFile, + ModelDirectory = Cache.ModelDirectory!, + MagicQuantDirectory = Cache.ModelMagicQuantDirectory! + }; + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + var imatrixEnsureResult = await imatrixService.EnsureImatrixAsync(imatrixRequest); + RuntimeSearchSpace.SetImatrixAvailability(imatrixEnsureResult.Available); + + var preCleanBenchmarkCache = Config.ReuseExistingFinalArtifacts + ? LoadReusableCloneBenchmarkRows(Cache.OutputDirectory!) + : new Dictionary(StringComparer.OrdinalIgnoreCase); + + bool canReuseEverything = TryLoadFullyReusableCloneRecords( + outputDirectory: Cache.OutputDirectory!, + manifest: manifest, + benchmarkCache: preCleanBenchmarkCache, + records: out var reusableRecords); + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await CleanOutputDirectoryAsync(Cache.OutputDirectory!, Config.ReuseExistingFinalArtifacts); + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + var archivedManifestFiles = await CopySourceManifestFilesAsync( + outputDirectory: Cache.OutputDirectory!, + sourceManifestLocalPath: manifestLocalPath, + sourceRepo: sourceRepo, + sourceJson: sourceJson, + huggingFace: hf, + ct: CancellationToken.None); + + // Always place the clone source manifest in the output manifest folder, stamped with this clone source. + manifest.SourceRepository = sourceRepo; + manifest.SourceJson = string.IsNullOrWhiteSpace(sourceRepo) ? sourceDescription : manifest.SourceJson; + string outputCloneManifestPath = MagicQuantManifestPathService.GetManifestFilePath(Cache.OutputDirectory!, MagicQuantManifestPathService.CloneConfigsFileName); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await File.WriteAllTextAsync(outputCloneManifestPath, JsonSerializer.Serialize(manifest, JsonOptions)); + archivedManifestFiles.Add(MagicQuantManifestPathService.CloneConfigsFileName); + + var cloneBuildResults = new Dictionary(StringComparer.OrdinalIgnoreCase); + + var records = canReuseEverything + ? reusableRecords + : new List(); + + if (records.Count == manifest.Artifacts.Count) + { + AnsiConsole.MarkupLine($"[green]Reused clone artifacts and benchmark summary:[/] all {records.Count:N0} artifact(s) matched existing GGUF byte sizes and {MagicQuantManifestPathService.CloneBenchmarksFileName}."); + } + else + { + records.Clear(); + var benchmarkCache = preCleanBenchmarkCache; + + string q8QuantizationKey = BaselineQuants.Q8_0.Names[0]; + string nativeQuantizationKey = (Cache.TorchType ?? Cache.MainTorchType.BF16).ToString(); + string cloneBenchmarkRootDir = Path.Combine(Cache.ModelMagicQuantDirectory!, "CloneBenchmarks"); + string nativeBenchDir = Path.Combine(cloneBenchmarkRootDir, nativeQuantizationKey); + string nativeLogitsDir = Path.Combine(nativeBenchDir, "logits"); + string pplCorporaDir = Path.Combine(cloneBenchmarkRootDir, "_ppl_corpora"); + + bool loadedPlanFromCache = !Cache.ForceRefreshHardwareProbe && + await benchmarkService.TryInitializeDynamicExecutionPlanFromCacheAsync( + q8QuantizationKey: q8QuantizationKey, + nativeModelPath: baseModelGgufPath, + nativeQuantizationKey: nativeQuantizationKey); + + if (loadedPlanFromCache) + { + AnsiConsole.MarkupLine("[grey]Clone mode reused the DB-backed hardware execution plan; no Q8 probe rebuild was needed.[/]"); + } + else + { + AnsiConsole.MarkupLine(Cache.ForceRefreshHardwareProbe + ? "[yellow]Hardware probe refresh requested; rebuilding Q8 probe and updating SQLite execution-plan cache.[/]" + : "[grey]No reusable hardware execution-plan cache row found; building one Q8 probe and saving it to SQLite.[/]"); + + await using var q8Lease = await quantizationService.BuildPureQ8ProbeLeaseAsync(); + await benchmarkService.EnsureDynamicExecutionPlanAsync( + q8ModelPath: q8Lease.GgufPath, + nativeModelPath: baseModelGgufPath, + q8QuantizationKey: q8QuantizationKey, + nativeQuantizationKey: nativeQuantizationKey, + discoveryTokenTarget: 8192, + forceRediscovery: Cache.ForceRefreshHardwareProbe); + } + + await EnsureCloneNativeBenchmarkArtifactsReadyAsync( + benchmarkService: benchmarkService, + nativeModelQuant: HybridQuant.CreatePureBaseline(BaselineQuants.GetBF16Quant()), + nativeModelPath: baseModelGgufPath, + nativeBenchDir: nativeBenchDir, + nativeLogitsDir: nativeLogitsDir, + pplCorporaDir: pplCorporaDir); + + foreach (var artifact in manifest.Artifacts) + { + string outputFile = Path.Combine(Cache.OutputDirectory!, artifact.FileName); + string baseQuantName = string.IsNullOrWhiteSpace(artifact.BaseQuant) + ? artifact.QuantFamily + : artifact.BaseQuant; + + var sourcePolicy = sourceClonePolicies.GetValueOrDefault(artifact.FileName); + string? artifactMissingManifestBaseQuantName = hasMissingManifestBaseQuantOverride + ? missingManifestBaseQuantName + : sourcePolicy?.MissingManifestBaseQuantName; + bool artifactAllowMissingManifestTensors = allowMissingManifestTensors || + hasMissingManifestBaseQuantOverride || + sourcePolicy?.AllowMissingManifestTensors == true || + !string.IsNullOrWhiteSpace(artifactMissingManifestBaseQuantName); + + AnsiConsole.Write(new Rule($"[yellow]Clone Artifact: {Markup.Escape(artifact.FileName)}[/]") { Justification = Justify.Left }); + + if (TryReuseExistingCloneArtifactAndBenchmark(outputFile, artifact, benchmarkCache, out var cachedRecord)) + { + AnsiConsole.MarkupLine($"[green]Reused existing clone artifact + benchmark:[/] {Markup.Escape(outputFile)}"); + records.Add(cachedRecord); + continue; + } + + bool artifactExists = Config.ReuseExistingFinalArtifacts && File.Exists(outputFile) && new FileInfo(outputFile).Length > 0; + if (artifactExists) + { + AnsiConsole.MarkupLine($"[green]Reused existing clone GGUF:[/] {Markup.Escape(outputFile)} [grey](benchmark cache missing/stale; rebenchmarking only)[/]"); + } + else + { + var cloneBuildResult = await cloneBuildService.BuildAsync( + tensorTypes: artifact.TensorTypes, + outputPath: outputFile, + baseQuantName: baseQuantName, + allowMissingManifestTensors: artifactAllowMissingManifestTensors, + missingManifestBaseQuantName: artifactMissingManifestBaseQuantName, + forceRebuild: true); + + cloneBuildResults[artifact.FileName] = cloneBuildResult; + } + + var benchmarkBaseline = ResolveCloneBenchmarkBaseline(baseQuantName, artifact.QuantFamily); + var quantForBenchmark = HybridQuant.CreatePureBaseline(benchmarkBaseline); + bool benchmarkRequiresKld = benchmarkBaseline.UniqueId != BaselineQuants.NativeSourceUniqueId; + + var bench = await benchmarkService.RunAllBenchmarksAsync( + quantConfig: quantForBenchmark, + modelPath: outputFile, + benchDir: Path.Combine(cloneBenchmarkRootDir, Path.GetFileNameWithoutExtension(artifact.FileName)), + klLogitsDir: benchmarkRequiresKld ? nativeLogitsDir : null, + domainsOverride: CloneBenchmarkDomains); + + var general = bench.Perplexity.TryGetValue("general", out var ppl) ? ppl : null; + + records.Add(new CloneArtifactBuildRecord + { + ManifestArtifact = artifact, + OutputPath = outputFile, + ActualSizeBytes = File.Exists(outputFile) ? (ulong)new FileInfo(outputFile).Length : 0UL, + Kld = general?.Kld, + Ppl = general?.Ppl, + PplDeltaPercent = null + }); + } + } + + ApplyCloneReferencePplDeltas(records); + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await CopyModelAdjacentFilesAsync(Cache.OutputDirectory!); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await CopyImatrixArtifactsAsync(Cache.OutputDirectory!); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await sidecarService.CopyMmprojArtifactsAsync(Cache.OutputDirectory!); + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await WriteCloneBenchmarkSummaryAsync(Cache.OutputDirectory!, records); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await WriteResolvedCloneConfigManifestAsync( + outputCloneManifestPath, + records, + sourceClonePolicies, + cloneBuildResults, + missingManifestBaseQuantName, + hasMissingManifestBaseQuantOverride); + archivedManifestFiles.Add(MagicQuantManifestPathService.CloneBenchmarksFileName); + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await new CloneReadmeGenerationService().GenerateAsync( + Cache.OutputDirectory!, + new DirectoryInfo(Cache.ModelDirectory!).Name, + sourceDescription, + !string.IsNullOrWhiteSpace(sourceRepo), + records, + archivedManifestFiles); + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await CleanCloneExportSidecarsAsync(Cache.OutputDirectory!); + + AnsiConsole.MarkupLine("[bold green]Repository quant clone complete.[/]"); + } + + private static async Task EnsureCloneNativeBenchmarkArtifactsReadyAsync( + BenchmarkService benchmarkService, + HybridQuant nativeModelQuant, + string nativeModelPath, + string nativeBenchDir, + string nativeLogitsDir, + string pplCorporaDir) + { + var status = ValidateCloneNativeBenchmarkEnvironment( + nativeBenchDir: nativeBenchDir, + nativeLogitsDir: nativeLogitsDir, + pplCorporaDir: pplCorporaDir); + + if (status.IsValid) + { + AnsiConsole.MarkupLine("[grey]Clone native benchmark/KLD artifacts already exist and passed validation.[/]"); + return; + } + + AnsiConsole.Write(new Rule("[yellow]Clone Native Benchmark/KLD Artifact Validation[/]") { Justification = Justify.Left }); + AnsiConsole.MarkupLine("[yellow]Clone native benchmark/KLD artifacts are missing or incomplete.[/] Regenerating required artifacts."); + PrintCloneNativeBenchmarkEnvironmentIssues(status); + + await ForceRegenerateCloneNativeBenchmarkArtifactsAsync( + benchmarkService: benchmarkService, + nativeModelQuant: nativeModelQuant, + nativeModelPath: nativeModelPath, + nativeBenchDir: nativeBenchDir, + nativeLogitsDir: nativeLogitsDir); + + status = ValidateCloneNativeBenchmarkEnvironment( + nativeBenchDir: nativeBenchDir, + nativeLogitsDir: nativeLogitsDir, + pplCorporaDir: pplCorporaDir); + + if (!status.IsValid) + { + var details = string.Join( + Environment.NewLine, + status.MissingOrInvalidArtifacts.Select(x => $"- {x}")); + + throw new InvalidOperationException( + "Clone native benchmark/logit generation completed, but required native benchmark artifacts are still missing or invalid. " + + "This is fatal because every cloned non-native benchmark requires complete native KLD logits." + + Environment.NewLine + + details); + } + + AnsiConsole.MarkupLine("[green]Clone native benchmark/KLD artifacts validated.[/]"); + } + + private static async Task ForceRegenerateCloneNativeBenchmarkArtifactsAsync( + BenchmarkService benchmarkService, + HybridQuant nativeModelQuant, + string nativeModelPath, + string nativeBenchDir, + string nativeLogitsDir) + { + if (Directory.Exists(nativeBenchDir)) + { + AnsiConsole.MarkupLine( + $"[grey]Clearing incomplete/stale clone native benchmark directory:[/] {Markup.Escape(nativeBenchDir)}"); + + Directory.Delete(nativeBenchDir, recursive: true); + } + + Directory.CreateDirectory(nativeBenchDir); + Directory.CreateDirectory(nativeLogitsDir); + + bool previousSuppressBenchmarkPersistence = Cache.SuppressBenchmarkPersistence; + + try + { + Cache.SuppressBenchmarkPersistence = true; + + await benchmarkService.RunAllBenchmarksAsync( + quantConfig: nativeModelQuant, + modelPath: nativeModelPath, + benchDir: nativeBenchDir, + klLogitsDir: nativeLogitsDir, + saveLogits: true, + domainsOverride: CloneBenchmarkDomains); + } + finally + { + Cache.SuppressBenchmarkPersistence = previousSuppressBenchmarkPersistence; + } + } + + private static CloneNativeBenchmarkEnvironmentStatus ValidateCloneNativeBenchmarkEnvironment( + string nativeBenchDir, + string nativeLogitsDir, + string pplCorporaDir) + { + var issues = new List(); + + if (string.IsNullOrWhiteSpace(nativeBenchDir)) + issues.Add("Clone native benchmark directory path is null/empty."); + else if (!Directory.Exists(nativeBenchDir)) + issues.Add($"Clone native benchmark directory does not exist: {nativeBenchDir}"); + + if (string.IsNullOrWhiteSpace(nativeLogitsDir)) + issues.Add("Clone native KLD logits directory path is null/empty."); + else if (!Directory.Exists(nativeLogitsDir)) + issues.Add($"Clone native KLD logits directory does not exist: {nativeLogitsDir}"); + + if (string.IsNullOrWhiteSpace(pplCorporaDir)) + issues.Add("Clone _ppl_corpora directory path is null/empty."); + else if (!Directory.Exists(pplCorporaDir)) + issues.Add($"Clone _ppl_corpora directory does not exist: {pplCorporaDir}"); + else if (!Directory.EnumerateFiles(pplCorporaDir, "*", SearchOption.AllDirectories).Any()) + issues.Add($"Clone _ppl_corpora directory exists but contains no files: {pplCorporaDir}"); + + foreach (var domain in CloneBenchmarkDomains.OrderBy(x => x, StringComparer.Ordinal)) + { + if (!string.IsNullOrWhiteSpace(nativeBenchDir) && Directory.Exists(nativeBenchDir)) + { + var pplLog = Path.Combine(nativeBenchDir, $"perplexity_{domain}.log"); + + if (!File.Exists(pplLog)) + issues.Add($"Missing clone native perplexity log for domain '{domain}': {pplLog}"); + else if (new FileInfo(pplLog).Length <= 0) + issues.Add($"Clone native perplexity log is empty for domain '{domain}': {pplLog}"); + } + + if (!string.IsNullOrWhiteSpace(nativeLogitsDir) && Directory.Exists(nativeLogitsDir)) + { + var logitsFile = Path.Combine(nativeLogitsDir, $"kld_logits_{domain}.bin"); + + if (!File.Exists(logitsFile)) + issues.Add($"Missing clone native KLD logits for domain '{domain}': {logitsFile}"); + else if (new FileInfo(logitsFile).Length <= 0) + issues.Add($"Clone native KLD logits file is empty for domain '{domain}': {logitsFile}"); + } + } + + return new CloneNativeBenchmarkEnvironmentStatus( + IsValid: issues.Count == 0, + MissingOrInvalidArtifacts: issues); + } + + private static void PrintCloneNativeBenchmarkEnvironmentIssues(CloneNativeBenchmarkEnvironmentStatus status) + { + if (status.IsValid) + return; + + foreach (var issue in status.MissingOrInvalidArtifacts.Take(20)) + AnsiConsole.MarkupLine($"[grey]- {Markup.Escape(issue)}[/]"); + + if (status.MissingOrInvalidArtifacts.Count > 20) + { + AnsiConsole.MarkupLine( + $"[grey]- ...and {status.MissingOrInvalidArtifacts.Count - 20:N0} more issue(s).[/]"); + } + } + + private static BaselineQuants ResolveCloneBenchmarkBaseline(string? baseQuantName, string? quantFamily) + { + var native = BaselineQuants.GetBF16Quant(); + + foreach (var raw in new[] { baseQuantName, quantFamily }) + { + if (string.IsNullOrWhiteSpace(raw)) + continue; + + string name = raw.Trim(); + + if (IsNativeBaselineName(name, native)) + return native; + + var standard = BaselineQuants.ResolveBuiltInStandardBaseline(name); + if (standard != null) + return standard; + + var recognized = BaselineQuants.GetAllRecognizedBaselines() + .FirstOrDefault(x => BaselineNameMatches(x, name)); + + if (recognized != null) + return recognized; + } + + return BaselineQuants.Q8_0; + } + + private static bool IsNativeBaselineName(string name, BaselineQuants native) + { + if (string.IsNullOrWhiteSpace(name)) + return false; + + string normalized = name.Trim(); + + return string.Equals(normalized, "native", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, "native_source", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, "source", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, (Cache.TorchType ?? Cache.MainTorchType.BF16).ToString(), StringComparison.OrdinalIgnoreCase) || + BaselineNameMatches(native, normalized); + } + + private static bool BaselineNameMatches(BaselineQuants baseline, string name) + { + if (string.IsNullOrWhiteSpace(name)) + return false; + + string normalized = name.Trim(); + + if (baseline.Names.Any(x => string.Equals(x, normalized, StringComparison.OrdinalIgnoreCase))) + return true; + + if (string.Equals(baseline.QuantizeBaseArgumentName, normalized, StringComparison.OrdinalIgnoreCase)) + return true; + + if (string.Equals(baseline.CanonicalKey, normalized, StringComparison.OrdinalIgnoreCase)) + return true; + + if (!string.IsNullOrWhiteSpace(baseline.ShortSourceName) && + string.Equals(baseline.ShortSourceName, normalized, StringComparison.OrdinalIgnoreCase)) + return true; + + if (!string.IsNullOrWhiteSpace(baseline.SourceFileName) && + string.Equals(Path.GetFileNameWithoutExtension(baseline.SourceFileName), normalized, StringComparison.OrdinalIgnoreCase)) + return true; + + if (baseline.PrimaryTensorWeightScheme.Names.Any(x => string.Equals(x, normalized, StringComparison.OrdinalIgnoreCase))) + return true; + + return false; + } + + private static void ApplyCloneReferencePplDeltas(IReadOnlyList records) + { + if (records.Count == 0) + return; + + var reference = records.FirstOrDefault(IsCloneQ8ReferenceRecord); + if (reference == null || !reference.Ppl.HasValue || reference.Ppl.Value <= 0d) + { + AnsiConsole.MarkupLine("[grey]Clone PPL delta reference unavailable; keeping any cached/source PPL delta values as-is.[/]"); + return; + } + + double referencePpl = reference.Ppl.Value; + foreach (var record in records) + { + if (record.Ppl.HasValue && record.Ppl.Value > 0d) + record.PplDeltaPercent = FinalReleaseMetadataService.CalculatePplDeltaPercent(record.Ppl.Value, referencePpl); + } + + AnsiConsole.MarkupLine( + $"[grey]Clone PPL deltas calculated from final Q8 artifact:[/] {Markup.Escape(reference.ManifestArtifact.FileName)}"); + } + + private static bool IsCloneQ8ReferenceRecord(CloneArtifactBuildRecord record) + { + var artifact = record.ManifestArtifact; + + foreach (var raw in new[] + { + artifact.BaseQuant, + artifact.QuantFamily, + artifact.DisplayName, + Path.GetFileNameWithoutExtension(artifact.FileName) + }) + { + if (string.IsNullOrWhiteSpace(raw)) + continue; + + if (BaselineNameMatches(BaselineQuants.Q8_0, raw.Trim())) + return true; + } + + return false; + } + + private static bool TryLoadFullyReusableCloneRecords( + string outputDirectory, + MagicQuantCloneManifest manifest, + IReadOnlyDictionary benchmarkCache, + out List records) + { + records = new List(); + + if (!Config.ReuseExistingFinalArtifacts || benchmarkCache.Count == 0) + return false; + + foreach (var artifact in manifest.Artifacts) + { + string outputFile = Path.Combine(outputDirectory, artifact.FileName); + if (!TryReuseExistingCloneArtifactAndBenchmark(outputFile, artifact, benchmarkCache, out var record)) + { + records.Clear(); + return false; + } + + records.Add(record); + } + + return records.Count == manifest.Artifacts.Count; + } + + private static bool TryReuseExistingCloneArtifactAndBenchmark( + string outputFile, + MagicQuantCloneArtifact artifact, + IReadOnlyDictionary benchmarkCache, + out CloneArtifactBuildRecord record) + { + record = default!; + + if (!Config.ReuseExistingFinalArtifacts) + return false; + + if (!File.Exists(outputFile)) + return false; + + var info = new FileInfo(outputFile); + if (info.Length <= 0) + return false; + + if (!benchmarkCache.TryGetValue(artifact.FileName, out var cached)) + return false; + + if (cached.SizeBytes != (ulong)info.Length) + return false; + + record = new CloneArtifactBuildRecord + { + ManifestArtifact = artifact, + OutputPath = outputFile, + ActualSizeBytes = cached.SizeBytes, + Kld = cached.Kld, + Ppl = cached.Ppl, + PplDeltaPercent = cached.PplDeltaPercent + }; + + return true; + } + + private static Dictionary LoadReusableCloneBenchmarkRows(string outputDirectory) + { + string path = MagicQuantManifestPathService.GetManifestFilePath(outputDirectory, MagicQuantManifestPathService.CloneBenchmarksFileName); + if (!File.Exists(path)) + return new Dictionary(StringComparer.OrdinalIgnoreCase); + + try + { + var rows = JsonSerializer.Deserialize>(File.ReadAllText(path), JsonOptions) + ?? new List(); + + return rows + .Where(x => !string.IsNullOrWhiteSpace(x.FileName) && x.SizeBytes > 0) + .GroupBy(x => x.FileName, StringComparer.OrdinalIgnoreCase) + .ToDictionary(g => g.Key, g => g.First(), StringComparer.OrdinalIgnoreCase); + } + catch (Exception ex) + { + AnsiConsole.MarkupLine($"[yellow]Existing clone benchmark summary could not be reused:[/] {Markup.Escape(ex.Message)}"); + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + } + + private static Dictionary LoadCloneArtifactPolicies(string manifestPath) + { + var policies = new Dictionary(StringComparer.OrdinalIgnoreCase); + + if (string.IsNullOrWhiteSpace(manifestPath) || !File.Exists(manifestPath)) + return policies; + + try + { + var root = JsonNode.Parse(File.ReadAllText(manifestPath)) as JsonObject; + var artifacts = TryGetProperty(root, "artifacts") as JsonArray; + if (artifacts == null) + return policies; + + foreach (var node in artifacts.OfType()) + { + string? fileName = TryGetString(node, "fileName"); + if (string.IsNullOrWhiteSpace(fileName)) + continue; + + var policyNode = TryGetProperty(node, CloneTensorPolicyPropertyName) as JsonObject; + if (policyNode == null) + continue; + + policies[fileName] = new CloneArtifactPolicy( + AllowMissingManifestTensors: TryGetBool(policyNode, "allowMissingManifestTensors"), + MissingManifestBaseQuantName: TryGetString(policyNode, "missingManifestBaseQuant")); + } + } + catch (Exception ex) + { + AnsiConsole.MarkupLine($"[yellow]Clone tensor policy metadata could not be read from source manifest:[/] {Markup.Escape(ex.Message)}"); + } + + return policies; + } + + private static async Task WriteResolvedCloneConfigManifestAsync( + string outputCloneManifestPath, + IReadOnlyCollection records, + IReadOnlyDictionary sourcePolicies, + IReadOnlyDictionary cloneBuildResults, + string? cliMissingManifestBaseQuantName, + bool hasCliMissingManifestBaseQuantOverride) + { + if (!File.Exists(outputCloneManifestPath)) + return; + + var root = JsonNode.Parse(await File.ReadAllTextAsync(outputCloneManifestPath)) as JsonObject; + var artifacts = TryGetProperty(root, "artifacts") as JsonArray; + if (root == null || artifacts == null) + return; + + int policiesWritten = 0; + var recordByFileName = records + .Where(x => !string.IsNullOrWhiteSpace(x.ManifestArtifact.FileName)) + .GroupBy(x => x.ManifestArtifact.FileName, StringComparer.OrdinalIgnoreCase) + .ToDictionary(g => g.Key, g => g.First(), StringComparer.OrdinalIgnoreCase); + + foreach (var artifactNode in artifacts.OfType()) + { + string? fileName = TryGetString(artifactNode, "fileName"); + if (string.IsNullOrWhiteSpace(fileName)) + continue; + + recordByFileName.TryGetValue(fileName, out var record); + cloneBuildResults.TryGetValue(fileName, out var buildResult); + sourcePolicies.TryGetValue(fileName, out var sourcePolicy); + + bool allowMissing = buildResult?.UsedManifestSubset == true || + sourcePolicy?.AllowMissingManifestTensors == true || + hasCliMissingManifestBaseQuantOverride; + string? missingBaseQuant = buildResult?.UsedManifestSubset == true + ? buildResult.EffectiveBaseQuantName + : hasCliMissingManifestBaseQuantOverride + ? cliMissingManifestBaseQuantName + : sourcePolicy?.MissingManifestBaseQuantName; + + if (!allowMissing && string.IsNullOrWhiteSpace(missingBaseQuant)) + { + artifactNode.Remove(CloneTensorPolicyPropertyName); + continue; + } + + var policyNode = new JsonObject + { + ["allowMissingManifestTensors"] = allowMissing, + ["missingManifestBaseQuant"] = string.IsNullOrWhiteSpace(missingBaseQuant) ? null : missingBaseQuant, + ["generatedAtUtc"] = DateTimeOffset.UtcNow.ToString("O") + }; + + if (record?.ActualSizeBytes > 0) + policyNode["actualSizeBytes"] = (long)Math.Min(record.ActualSizeBytes, long.MaxValue); + + if (buildResult?.UsedManifestSubset == true) + { + policyNode["missingManifestTensorCount"] = buildResult.MissingInManifest.Count; + policyNode["missingManifestTensors"] = new JsonArray(buildResult.MissingInManifest.Select(x => JsonValue.Create(x)).ToArray()); + } + else if (sourcePolicy?.AllowMissingManifestTensors == true || !string.IsNullOrWhiteSpace(sourcePolicy?.MissingManifestBaseQuantName)) + { + policyNode["inheritedFromSourceManifest"] = true; + } + else if (hasCliMissingManifestBaseQuantOverride) + { + policyNode["createdFromCliOverride"] = true; + } + + artifactNode[CloneTensorPolicyPropertyName] = policyNode; + policiesWritten++; + } + + await File.WriteAllTextAsync(outputCloneManifestPath, root.ToJsonString(JsonOptions)); + AnsiConsole.MarkupLine($"[green]Resolved clone config manifest updated:[/] {Markup.Escape(outputCloneManifestPath)} [grey](policies={policiesWritten:N0})[/]"); + } + + private static JsonNode? TryGetProperty(JsonObject? obj, string name) + { + if (obj == null) + return null; + + foreach (var kv in obj) + { + if (string.Equals(kv.Key, name, StringComparison.OrdinalIgnoreCase)) + return kv.Value; + } + + return null; + } + + private static string? TryGetString(JsonObject obj, string name) + => TryGetProperty(obj, name)?.GetValue(); + + private static bool TryGetBool(JsonObject obj, string name) + { + var node = TryGetProperty(obj, name); + if (node == null) + return false; + + try + { + return node.GetValue(); + } + catch + { + return bool.TryParse(node.ToString(), out var value) && value; + } + } + + private static async Task> CopySourceManifestFilesAsync( + string outputDirectory, + string sourceManifestLocalPath, + string? sourceRepo, + string? sourceJson, + HuggingFaceBaselineService huggingFace, + CancellationToken ct) + { + string targetManifestDir = MagicQuantManifestPathService.EnsureManifestDirectory(outputDirectory); + var copied = new HashSet(StringComparer.OrdinalIgnoreCase); + + if (!string.IsNullOrWhiteSpace(sourceRepo)) + { + foreach (var fileName in MagicQuantManifestPathService.KnownManifestFileNames) + { + if (string.Equals(fileName, MagicQuantManifestPathService.CloneBenchmarksFileName, StringComparison.OrdinalIgnoreCase)) + continue; + + if (await TryDownloadOptionalSourceManifestFileAsync(sourceRepo.Trim(), fileName, Path.Combine(targetManifestDir, fileName), huggingFace, ct)) + copied.Add(fileName); + } + + return copied; + } + + string sourceDir = Path.GetDirectoryName(sourceManifestLocalPath) ?? string.Empty; + if (Directory.Exists(sourceDir)) + { + foreach (var file in Directory.EnumerateFiles(sourceDir, "magicquant*.json", SearchOption.TopDirectoryOnly)) + { + string fileName = Path.GetFileName(file); + if (string.Equals(fileName, MagicQuantManifestPathService.CloneBenchmarksFileName, StringComparison.OrdinalIgnoreCase)) + continue; + + File.Copy(file, Path.Combine(targetManifestDir, fileName), overwrite: true); + copied.Add(fileName); + } + } + + return copied; + } + + private static async Task TryDownloadOptionalSourceManifestFileAsync( + string repoId, + string fileName, + string destinationPath, + HuggingFaceBaselineService huggingFace, + CancellationToken ct) + { + var candidates = new[] + { + MagicQuantManifestPathService.RelativeManifestPath(fileName), + fileName + }; + + foreach (var candidate in candidates) + { + try + { + await huggingFace.DownloadRepositoryFileAsync( + repoId: repoId, + fileName: candidate, + destinationPath: destinationPath, + forceRedownload: true, + ct: ct); + + AnsiConsole.MarkupLine($"[green]Archived source manifest file:[/] {Markup.Escape(candidate)}"); + return true; + } + catch + { + // Optional source manifest sidecars may not exist, especially in older repos. + } + } + + AnsiConsole.MarkupLine($"[grey]Optional source manifest file unavailable:[/] {Markup.Escape(fileName)}"); + return false; + } + + private static async Task WriteCloneBenchmarkSummaryAsync(string outputDirectory, IReadOnlyCollection records) + { + string path = MagicQuantManifestPathService.GetManifestFilePath(outputDirectory, MagicQuantManifestPathService.CloneBenchmarksFileName); + + var payload = records + .OrderBy(x => x.Kld ?? double.MaxValue) + .ThenBy(x => x.ActualSizeBytes) + .Select(x => new CloneBenchmarkCacheRow + { + FileName = x.ManifestArtifact.FileName, + DisplayName = x.ManifestArtifact.DisplayName, + Provider = x.ManifestArtifact.Provider, + QuantFamily = x.ManifestArtifact.QuantFamily, + BaseQuant = x.ManifestArtifact.BaseQuant, + Kld = x.Kld, + Ppl = x.Ppl, + PplDeltaPercent = x.PplDeltaPercent, + SizeBytes = x.ActualSizeBytes, + SizeGB = x.ActualSizeBytes / 1000d / 1000d / 1000d, + SizeGiB = x.ActualSizeBytes / 1024d / 1024d / 1024d, + SourceKld = x.ManifestArtifact.SourceKld, + SourcePpl = x.ManifestArtifact.SourcePpl, + SourceSizeBytes = x.ManifestArtifact.SourceSizeBytes + }) + .ToList(); + + await File.WriteAllTextAsync(path, JsonSerializer.Serialize(payload, JsonOptions)); + AnsiConsole.MarkupLine($"[green]Clone benchmark summary generated:[/] {Markup.Escape(path)}"); + } + + private static async Task CopyModelAdjacentFilesAsync(string outputDirectory) + { + foreach (var fileName in ModelAdjacentFiles) + { + string source = Path.Combine(Cache.ModelDirectory!, fileName); + string target = Path.Combine(outputDirectory, fileName); + + if (!File.Exists(source)) + continue; + + File.Copy(source, target, overwrite: true); + await Task.Yield(); + AnsiConsole.MarkupLine($"[green]Copied model-adjacent file:[/] {Markup.Escape(fileName)}"); + } + } + + private static Task CopyImatrixArtifactsAsync(string outputDirectory) + { + if (!Cache.IsImatrixAvailable || string.IsNullOrWhiteSpace(Cache.ActiveImatrixPath)) + return Task.CompletedTask; + + string target = Path.Combine(outputDirectory, "imatrix.dat"); + File.Copy(Cache.ActiveImatrixPath!, target, overwrite: true); + AnsiConsole.MarkupLine($"[green]Copied imatrix artifact:[/] {Markup.Escape(target)}"); + return Task.CompletedTask; + } + + private static async Task CleanOutputDirectoryAsync(string outputDirectory, bool preserveReusableGgufs) + { + Directory.CreateDirectory(outputDirectory); + + foreach (var file in Directory.EnumerateFiles(outputDirectory, "*", SearchOption.TopDirectoryOnly)) + { + if (preserveReusableGgufs && + string.Equals(Path.GetExtension(file), ".gguf", StringComparison.OrdinalIgnoreCase) && + new FileInfo(file).Length > 0) + { + continue; + } + + await HardDeleteHelper.DeleteFileIfExistsAsync(file); + } + + foreach (var directory in Directory.EnumerateDirectories(outputDirectory, "*", SearchOption.TopDirectoryOnly)) + await HardDeleteHelper.DeleteDirectoryIfExistsAsync(directory, CancellationToken.None); + + AnsiConsole.MarkupLine(preserveReusableGgufs + ? $"[grey]Cleaned clone export metadata/non-GGUF files; preserved existing non-empty GGUFs for reuse validation:[/] {Markup.Escape(outputDirectory)}" + : $"[grey]Cleaned clone export directory:[/] {Markup.Escape(outputDirectory)}"); + } + + private static async Task CleanCloneExportSidecarsAsync(string outputDirectory) + { + string[] patterns = + [ + "*.success.json", + "*.quantize.log", + "*.convert.log", + "imatrix.success.json", + "imatrix.metadata.json", + "imatrix.build.log" + ]; + + foreach (var pattern in patterns) + { + foreach (var file in Directory.EnumerateFiles(outputDirectory, pattern, SearchOption.TopDirectoryOnly)) + await HardDeleteHelper.DeleteFileIfExistsAsync(file); + } + + AnsiConsole.MarkupLine($"[grey]Cleaned clone export sidecar success/log files:[/] {Markup.Escape(outputDirectory)}"); + } + + private static async Task EnsureSqliteReadyAsync() + { + await using var db = new MagicQuantContext(); + await db.Database.MigrateAsync(); + } + + private static string ResolveAndValidateOutputDirectory(IReadOnlyCollection args) + { + string? explicitOutput = Get(args, "output-dir"); + string outputDir = OutputPathService.Clone( + Cache.ModelMagicQuantDirectory!, explicitOutput, Config.OutputDirectory); + Directory.CreateDirectory(outputDir); + return outputDir; + } + + private static string? Get(IReadOnlyCollection args, string name) + => args.FirstOrDefault(a => string.Equals(a.Name, name, StringComparison.OrdinalIgnoreCase))?.Value; + + private static void ShowHelp() + { + AnsiConsole.MarkupLine("[bold yellow]Command: clone-repository-quants[/]"); + AnsiConsole.MarkupLine("Rebuilds the final GGUF list from a MagicQuant-compatible tensor config manifest without running the discovery pipeline."); + AnsiConsole.MarkupLine("Usage:"); + AnsiConsole.WriteLine(" mq clone-repository-quants --model-dir \"\" --architecture-family \"\" --source-repo \"owner/repo\" [--output-dir \"\"] [--reuse-existing-final-artifacts]"); + AnsiConsole.WriteLine(" mq clone-repository-quants --model-dir \"\" --architecture-family \"\" --source-json \"\" [--output-dir \"\"] [--reuse-existing-final-artifacts]"); + AnsiConsole.MarkupLine("Options:"); + AnsiConsole.MarkupLine($" --source-repo Hugging Face repo containing {MagicQuantManifestPathService.RelativeManifestPath(MagicQuantManifestPathService.CloneConfigsFileName)} or legacy root {MagicQuantManifestPathService.CloneConfigsFileName}"); + AnsiConsole.MarkupLine(" --source-json Local or http(s) path to magicquant.clone-configs.json"); + AnsiConsole.MarkupLine(" --use-imatrix Use configured/provided imatrix for the cloned model"); + AnsiConsole.MarkupLine(" --reuse-existing-final-artifacts Reuse matching existing GGUFs and matching clone benchmark JSON rows"); + AnsiConsole.MarkupLine(" --allow-missing-manifest-tensors Allow clone manifests that are strict subsets of the current model tensor list; extra source tensors receive no explicit --tensor-type override and fall through to base quantization"); + AnsiConsole.MarkupLine(" --missing-manifest-base-quant Allow strict-subset clone manifests and use this llama.cpp base quant for tensors absent from the manifest, e.g. Q8_0"); + AnsiConsole.MarkupLine(" --recheck-hardware-probe / --force-refresh-hardware-probe Force Q8/native hardware probe and refresh the SQLite execution-plan cache"); + } + + private sealed record CloneNativeBenchmarkEnvironmentStatus( + bool IsValid, + IReadOnlyList MissingOrInvalidArtifacts); + + private sealed record CloneArtifactPolicy( + bool AllowMissingManifestTensors, + string? MissingManifestBaseQuantName); + + private sealed class CloneBenchmarkCacheRow + { + public string FileName { get; set; } = string.Empty; + public string DisplayName { get; set; } = string.Empty; + public string Provider { get; set; } = string.Empty; + public string QuantFamily { get; set; } = string.Empty; + public string BaseQuant { get; set; } = string.Empty; + public double? Kld { get; set; } + public double? Ppl { get; set; } + public double? PplDeltaPercent { get; set; } + public ulong SizeBytes { get; set; } + public double SizeGB { get; set; } + public double SizeGiB { get; set; } + public double? SourceKld { get; set; } + public double? SourcePpl { get; set; } + public ulong? SourceSizeBytes { get; set; } + } +} diff --git a/src/MagicQuant/Commands/CommandCatalog.cs b/src/MagicQuant/Commands/CommandCatalog.cs new file mode 100644 index 0000000..24abee9 --- /dev/null +++ b/src/MagicQuant/Commands/CommandCatalog.cs @@ -0,0 +1,20 @@ +namespace MagicQuant.Commands; + +/// One registry for dispatch and top-level help, including script compatibility aliases. +public static class CommandCatalog +{ + public static bool IsHelp(string argument) => + argument.Equals("help", StringComparison.OrdinalIgnoreCase) || argument is "--help" or "-h"; + + public static Dictionary Factory)> Create() => + new(StringComparer.OrdinalIgnoreCase) + { + ["init-config"] = ("Create an editable config from the packaged profile", () => new InitConfig()), + ["pipeline"] = ("Learn baselines, discover hybrids, validate and export survivors", () => new QuantizationPipeline()), + ["evolution"] = ("Compatibility alias for pipeline", () => new QuantizationPipeline()), + ["validate-predictions"] = ("Compare KLD predictions with existing SQLite benchmarks", () => new ValidatePredictions()), + ["build-hybrids"] = ("Compatibility entry point for the full pipeline and export", () => new BuildHybrids()), + ["clone-repository-quants"] = ("Rebuild final tensor configurations from a compatible repository/manifest", () => new CloneRepositoryQuants()), + ["initialize-llama-cpp"] = ("Initialize or update llama.cpp and Python dependencies", () => new InitializeLlamaCpp()) + }; +} diff --git a/src/MagicQuant/Commands/Evolution.cs b/src/MagicQuant/Commands/Evolution.cs new file mode 100644 index 0000000..d1a224b --- /dev/null +++ b/src/MagicQuant/Commands/Evolution.cs @@ -0,0 +1,7 @@ +namespace MagicQuant.Commands; + +/// +/// Compatibility entry point for callers using the historical command class. +/// MagicQuant now performs benchmark-driven discovery rather than evolutionary search. +/// +public class Evolution : QuantizationPipeline; diff --git a/src/MagicQuant/Commands/InitConfig.cs b/src/MagicQuant/Commands/InitConfig.cs new file mode 100644 index 0000000..a3e4e65 --- /dev/null +++ b/src/MagicQuant/Commands/InitConfig.cs @@ -0,0 +1,36 @@ +using MagicQuant.Models; + +namespace MagicQuant.Commands; + +/// Copies the packaged profile to a user-owned file without initializing runtime state. +public sealed class InitConfig : ICommand +{ + public static void ValidateTokens(string[] tokens) + { + bool valid = tokens.Length == 0 + || (tokens.Length == 2 && tokens[0].Equals("--output", StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrWhiteSpace(tokens[1]) && !tokens[1].StartsWith("--", StringComparison.Ordinal)) + || (tokens.Length == 1 && tokens[0].StartsWith("--output=", StringComparison.OrdinalIgnoreCase) + && !string.IsNullOrWhiteSpace(tokens[0][9..])); + if (!valid) throw new ArgumentException("Usage: magicquant init-config [--output config.yaml]"); + } + + public async Task Run(List args) + { + if (args.Any(a => string.Equals(a.Name, "help", StringComparison.OrdinalIgnoreCase))) + { + Console.WriteLine("Usage: magicquant init-config [--output config.yaml]"); + Console.WriteLine("Copy the complete bundled profile. Existing files are never overwritten."); + return; + } + if (args.Any(a => !string.Equals(a.Name, "output", StringComparison.OrdinalIgnoreCase)) || args.Count > 1) + throw new ArgumentException("init-config accepts only one --output path."); + if (args.Count == 1 && string.IsNullOrWhiteSpace(args[0].Value)) + throw new ArgumentException("--output requires a file path."); + string destination = Path.GetFullPath(args.Count == 0 ? "config.yaml" : args[0].Value!); + using var source = File.OpenRead(Path.Combine(AppContext.BaseDirectory, "config.default.yaml")); + using var target = new FileStream(destination, FileMode.CreateNew, FileAccess.Write); + await source.CopyToAsync(target); + Console.WriteLine($"Created {destination}. Edit model, architecture, output and scratch settings before running."); + } +} diff --git a/src/MagicQuant/Commands/InitializeLlamaCpp.cs b/src/MagicQuant/Commands/InitializeLlamaCpp.cs new file mode 100644 index 0000000..d8602d2 --- /dev/null +++ b/src/MagicQuant/Commands/InitializeLlamaCpp.cs @@ -0,0 +1,296 @@ +using MagicQuant.Models; +using MagicQuant.Helpers; +using Spectre.Console; +using System.Runtime.InteropServices; +using System.Diagnostics; +using MQ.DB; +using MQ.DB.Models; + +namespace MagicQuant.Commands; + +public class InitializeLlamaCpp : ICommand +{ + public async Task Run(List args) + { + if (args.Any(a => string.Equals(a.Name, "help", StringComparison.OrdinalIgnoreCase))) + { + AnsiConsole.MarkupLine("[bold yellow]Command: initialize-llama-cpp[/]"); + AnsiConsole.WriteLine("Initialize llama.cpp and Python dependencies in the shared user MagicQuant directory."); + AnsiConsole.WriteLine(" --update Update dependencies and rebuild llama.cpp"); + AnsiConsole.WriteLine(" --llama-root Existing llama.cpp checkout (requires both paths below)"); + AnsiConsole.WriteLine(" --llama-bin Existing compiled binaries directory"); + AnsiConsole.WriteLine(" --convert-script Existing convert_hf_to_gguf.py file"); + AnsiConsole.WriteLine("Without custom paths, setup can download dependencies and request sudo on Linux."); + AnsiConsole.WriteLine("--validate / --verify retain setup behavior; they are not a read-only check."); + return; + } + + // --------------------------------------------------------- + // 1. Argument Parsing & Path Validation + // --------------------------------------------------------- + bool update = args.Any(a => a.Name?.ToLower() == "update"); + + string? convertScript = args.FirstOrDefault(a => a.Name?.ToLower() == "convert-script")?.Value; + string? llamaBin = args.FirstOrDefault(a => a.Name?.ToLower() == "llama-bin")?.Value; + string? llamaRoot = args.FirstOrDefault(a => a.Name?.ToLower() == "llama-root")?.Value; + + convertScript ??= Config.Current.Paths.ConvertScript; + llamaBin ??= Config.Current.Paths.LlamaBin; + llamaRoot ??= Config.Current.Paths.LlamaRoot; + + // Custom Path Validation + if (!string.IsNullOrEmpty(llamaRoot)) + { + if (string.IsNullOrEmpty(convertScript) || string.IsNullOrEmpty(llamaBin)) + { + throw new ArgumentException("Custom paths require --llama-root, --llama-bin, AND --convert-script (or their YAML equivalents)."); + } + + // Normalize and Check + llamaRoot = Path.GetFullPath(llamaRoot); + llamaBin = Path.GetFullPath(llamaBin); + convertScript = Path.GetFullPath(convertScript); + + if (!Directory.Exists(llamaRoot) || !Directory.Exists(llamaBin) || !File.Exists(convertScript)) + { + throw new DirectoryNotFoundException("One or more custom llama.cpp paths do not exist."); + } + + Cache.LlamaRoot = llamaRoot; + Cache.LlamaBin = llamaBin; + Cache.ConvertScript = convertScript; + AnsiConsole.MarkupLine("[green]✔ Custom Environment Validated.[/]"); + _ = DetectAndCacheSystemInfo(); + return; + } + else if (!string.IsNullOrEmpty(convertScript) || !string.IsNullOrEmpty(llamaBin)) + { + throw new ArgumentException("Partial llama.cpp paths provided. Provide all three custom paths or none."); + } + + // --------------------------------------------------------- + // 2. Setup Default Paths + // --------------------------------------------------------- + string userHome = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); + string magicQuantPath = Path.Combine(userHome, MagicConstants.MagicQuantFolder); + if (!Directory.Exists(magicQuantPath)) Directory.CreateDirectory(magicQuantPath); + + // --------------------------------------------------------- + // 3. Hardware Detection + // --------------------------------------------------------- + var sysInfo = DetectAndCacheSystemInfo(); + + // --------------------------------------------------------- + // 4. Linux System Deps (Sudo Handling) + // --------------------------------------------------------- + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + var requiredPackages = new List + { + "build-essential", "cmake", "ninja-build", "git", + "python3", "python3-venv", "python3-pip", "libcurl4-openssl-dev" + }; + + if (sysInfo.GpuInfo.FirstOrDefault()?.GpuVendor == GpuVendor.Nvidia) requiredPackages.Add("nvidia-cuda-toolkit"); + + // Check if updates are needed + if (update || !AreLinuxPackagesInstalled(requiredPackages)) + { + AnsiConsole.MarkupLine("[yellow]System dependencies are missing or update requested.[/]"); + AnsiConsole.MarkupLine("[grey]Sudo permissions are required to install system packages via apt.[/]"); + + // A. Ask for Sudo permission upfront + try + { + await RefreshSudoCredentialsAsync(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + throw new InvalidOperationException("Sudo access denied or cancelled. Cannot install system dependencies.", ex); + } + + // B. Run Install WITH sudo + AnsiConsole.MarkupLine("[cyan]Installing/Updating System Dependencies (sudo apt)...[/]"); + string aptArgs = "install -y " + string.Join(" ", requiredPackages); + + // We run 'sudo' directly here + await RunSimpleProcess("sudo", "apt " + aptArgs); + } + else + { + AnsiConsole.MarkupLine("[green]✔ System dependencies already installed.[/]"); + } + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + throw new PlatformNotSupportedException("Automatic macOS setup is not implemented. Provide an existing llama.cpp environment."); + } + + // --------------------------------------------------------- + // 5. Python Environment Setup (Runs as Normal User) + // --------------------------------------------------------- + var pyManager = new PythonManager(magicQuantPath); + await pyManager.SetupEnvironmentAsync(); + + // --------------------------------------------------------- + // 6. Build Llama.cpp (Runs as Normal User) + // --------------------------------------------------------- + // The installer always lives in the user's shared MagicQuant directory, but + // dependency validation is also invoked inside commands that may use an + // isolated --magic-quant-root. Do not overwrite that configured runtime root: + // doing so silently redirects SQLite and other campaign state back to the + // user's shared installation directory. + var builder = new LlamaBuilder(magicQuantPath, sysInfo); + await builder.PrepareAndBuildAsync(update); + + // --------------------------------------------------------- + // 7. Install Python Libraries (Runs as Normal User) + // --------------------------------------------------------- + AnsiConsole.Write(new Rule("[yellow]Installing Python Libraries[/]") { Justification = Justify.Left }); + + // Helper to decide if we need to install + async Task EnsurePackage(string name, string installCmd, Dictionary? env = null) + { + if (!update) + { + string? version = await pyManager.GetInstalledVersionAsync(name); + if (version != null) + { + AnsiConsole.MarkupLine($"[green]✔ {name} is already installed (v{version}).[/]"); + return; + } + } + + AnsiConsole.MarkupLine($"[cyan]Installing {name}...[/]"); + await pyManager.RunPipInstallAsync(installCmd, env); + } + + // A. Purge Cache (Only on update) + if (update) + { + AnsiConsole.MarkupLine("[grey]Purging pip cache...[/]"); + await pyManager.RunPipInstallAsync("cache purge"); + } + + // B. Install PyTorch (Hardware Specific & Dynamic) + string torchCmd = "torch torchvision torchaudio"; + bool isNvidia = sysInfo.GpuInfo.FirstOrDefault()?.GpuVendor == GpuVendor.Nvidia; + + if (isNvidia) + { + double cudaVer = HardwareHelper.GetCudaVersion(); + AnsiConsole.MarkupLine($"[grey]Detected CUDA Version: {cudaVer}[/]"); + + if (cudaVer >= 12.0) + { + torchCmd += " --index-url https://download.pytorch.org/whl/cu124"; + AnsiConsole.MarkupLine($"[cyan]Targeting PyTorch for CUDA 12.x...[/]"); + } + else if (cudaVer >= 11.0) + { + torchCmd += " --index-url https://download.pytorch.org/whl/cu118"; + AnsiConsole.MarkupLine($"[cyan]Targeting PyTorch for CUDA 11.x...[/]"); + } + else + { + AnsiConsole.MarkupLine("[yellow]Warning: Could not detect CUDA version or version is < 11. Installing default PyTorch.[/]"); + } + } + else + { + AnsiConsole.MarkupLine($"[cyan]Installing Standard PyTorch (CPU/AMD/Intel)...[/]"); + } + + await EnsurePackage("torch", torchCmd); + + // C. Install Core Utilities + string coreDeps = "gguf tokenizers transformers mistral-common sentencepiece datasets huggingface_hub"; + + if (!update && await pyManager.GetInstalledVersionAsync("transformers") != null) + { + AnsiConsole.MarkupLine("[green]✔ Core utilities (transformers, etc.) are installed.[/]"); + } + else + { + AnsiConsole.MarkupLine("[cyan]Installing Core Utilities...[/]"); + await pyManager.RunPipInstallAsync($"--upgrade --no-cache-dir {coreDeps}"); + } + + // D. Install llama-cpp-python + var llamaEnv = new Dictionary(); + if (isNvidia) + { + llamaEnv["CMAKE_ARGS"] = "-DGGML_CUDA=on"; + llamaEnv["FORCE_CMAKE"] = "1"; + } + else if (sysInfo.GpuInfo.FirstOrDefault()?.GpuVendor == GpuVendor.Amd) + { + llamaEnv["CMAKE_ARGS"] = "-DGGML_HIPBLAS=on"; + llamaEnv["FORCE_CMAKE"] = "1"; + } + + await EnsurePackage("llama-cpp-python", + "--upgrade --force-reinstall --no-cache-dir llama-cpp-python", + llamaEnv); + + AnsiConsole.MarkupLine("[bold green]Initialization Complete![/]"); + AnsiConsole.MarkupLine($"Llama Binaries: [grey]{builder.GetLlamaBinPath()}[/]"); + } + + private static SystemInfo DetectAndCacheSystemInfo() + { + var sysInfo = HardwareHelper.GetSystemInfo(); + Cache.SysInfo = sysInfo; + + AnsiConsole.Write(new Rule("[yellow]System Detection[/]") { Justification = Justify.Left }); + AnsiConsole.MarkupLine( + $"Detected GPU: [green]{sysInfo.GpuInfo.FirstOrDefault()?.GpuVendor}[/] " + + $"([blue]{sysInfo.GpuInfo.FirstOrDefault()?.GpuName}[/] - {sysInfo.GpuInfo.Sum(x => x.VramGb):F1} GB)"); + AnsiConsole.MarkupLine($"Detected RAM: [blue]{sysInfo.RamGb:F1} GB[/]"); + + return sysInfo; + } + + // --- Helpers --- + + private static async Task RunSimpleProcess(string exe, string args) + { + var result = await new MagicQuant.Runtime.ProcessRunner().RunAsync(new ProcessStartInfo(exe, args), + onLine: (line, _) => AnsiConsole.WriteLine(line)); + if (!result.Success) throw new InvalidOperationException($"{exe} failed with exit code {result.ExitCode}."); + } + + private static Task RefreshSudoCredentialsAsync() => LinuxHelper.RefreshSudoCredentialsAsync(); + + private bool AreLinuxPackagesInstalled(List packages) + { + // dpkg-query check + foreach (var pkg in packages) + { + try + { + var psi = new ProcessStartInfo + { + FileName = "dpkg-query", + Arguments = $"-W -f='${{Status}}' {pkg}", + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true + }; + using var p = Process.Start(psi); + string output = p?.StandardOutput.ReadToEnd() ?? ""; + p?.WaitForExit(); + + if (!output.Contains("install ok installed")) + { + return false; // Found a missing package + } + } + catch + { + return false; // Command failed, assume missing + } + } + return true; + } +} diff --git a/src/MagicQuant/Commands/QuantizationPipeline.cs b/src/MagicQuant/Commands/QuantizationPipeline.cs new file mode 100644 index 0000000..8f97f6e --- /dev/null +++ b/src/MagicQuant/Commands/QuantizationPipeline.cs @@ -0,0 +1,823 @@ +using MagicQuant.Configuration; +using MagicQuant.Helpers; +using MagicQuant.Models; +using MagicQuant.Services; +using MagicQuant.Services.Progress; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; +using Microsoft.EntityFrameworkCore; +using Spectre.Console; + +namespace MagicQuant.Commands; + +/// +/// Coordinates baseline learning, isolation measurements, prediction-guided selection, +/// real benchmark validation, and final export. Numerical policy lives in services. +/// +public class QuantizationPipeline : ICommand +{ + private static readonly string[] RequiredNativeKldDomains = ["general", "code", "math"]; + + public async Task Run(List args) + { + if (args.Any(a => string.Equals(a.Name, "help", StringComparison.OrdinalIgnoreCase))) + { + ShowPipelineHelp(); + return; + } + + string? modelDirRaw = args.FirstOrDefault(a => + string.Equals(a.Name, "model-dir", StringComparison.OrdinalIgnoreCase))?.Value; + + if (string.IsNullOrWhiteSpace(modelDirRaw)) + modelDirRaw = Config.Current.Paths.ModelDir; + + if (string.IsNullOrWhiteSpace(modelDirRaw)) + { + const string msg = "[red]Error:[/] Missing required model directory. Provide [yellow]--model-dir[/] or set [yellow]paths.model_dir[/] in YAML."; + AnsiConsole.MarkupLine(msg); + ShowPipelineHelp(); + throw new InvalidOperationException("Missing required model directory."); + } + + string fullModelPath = Path.GetFullPath(modelDirRaw); + + if (!Directory.Exists(fullModelPath)) + { + string msg = + $"[red]Error:[/] The directory [yellow]{Markup.Escape(fullModelPath)}[/] does not exist."; + AnsiConsole.MarkupLine(msg); + ShowPipelineHelp(); + throw new DirectoryNotFoundException($"The directory '{fullModelPath}' does not exist."); + } + + var safeTensorFiles = Directory.GetFiles(fullModelPath, "*.safetensors", SearchOption.TopDirectoryOnly); + + if (safeTensorFiles.Length == 0) + { + AnsiConsole.MarkupLine( + $"[red]Error:[/] No [yellow].safetensors[/] files found in [blue]{Markup.Escape(fullModelPath)}[/]."); + AnsiConsole.MarkupLine("[grey]Please ensure this is a valid HuggingFace model directory.[/]"); + throw new InvalidOperationException("No .safetensors files were found in the provided model directory."); + } + + Cache.ModelDirectory = fullModelPath; + Cache.ModelMagicQuantDirectory = Path.Combine(fullModelPath, "MagicQuant"); + ModelRuntimePathService.InitializeForCurrentModel(); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await new ExternalBaselineCacheCleanupService().CleanupStaleArtifactsAsync(); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await new ScratchStorageService(new ModelArtifactPathService()).CleanupStaleScratchArtifactsAsync(); + Cache.ForceRefreshHardwareProbe = Config.Current.Flags.ForceRefreshHardwareProbe; + Cache.UseImatrix = Config.Current.Flags.UseImatrix; + Cache.ForceImatrixRebuild = Config.Current.Flags.ForceImatrixRebuild; + + RuntimeSearchSpace.ResetForNewModel(); + RuntimeSearchSpace.SetImatrixAvailability(false); + RuntimeSearchSpace.AllowHighPrecisionHybrids = Config.Current.Flags.AllowHighPrecisionHybrids; + + JsonHelper.DetectAndSetTorchType(Cache.ModelDirectory); + + if (!Directory.Exists(Cache.ModelMagicQuantDirectory)) + Directory.CreateDirectory(Cache.ModelMagicQuantDirectory); + + Cache.OutputDirectory = ResolveAndValidateOutputDirectory(); + + AnsiConsole.MarkupLine("[green]✔ Model Directory Validated[/]"); + AnsiConsole.Write(new Rule("[yellow]Pipeline Configuration[/]") { Justification = Justify.Left }); + AnsiConsole.MarkupLine($"Model Path: [blue]{Markup.Escape(Cache.ModelDirectory)}[/]"); + AnsiConsole.MarkupLine($"Work Path: [blue]{Markup.Escape(Cache.ModelMagicQuantDirectory)}[/]"); + AnsiConsole.MarkupLine($"Export Path: [blue]{Markup.Escape(Cache.OutputDirectory ?? "n/a")}[/]"); + AnsiConsole.MarkupLine($"Files Found: [green]{safeTensorFiles.Length:N0}[/] safe tensors"); + AnsiConsole.MarkupLine($"Tensor Review: [cyan]{(Cache.ConfirmTensorGroupProfile ? "prompt" : "skip prompt")}[/]"); + AnsiConsole.MarkupLine($"Regex Rebucket: [cyan]{(Cache.RebucketLearnedTensorGroupsFromExistingTruth ? "enabled" : "disabled")}[/]"); + + if (string.IsNullOrEmpty(Cache.LlamaBin)) + AnsiConsole.MarkupLine("[yellow]Warning:[/] Llama binaries path not set in Cache. (Did Initialization run?)"); + + AnsiConsole.MarkupLine("[grey]Acquiring unique model ID...[/]"); + Cache.CurrentModelId = MagicQuantModelId.GetOrCreateModelId(Cache.ModelDirectory); + AnsiConsole.MarkupLine($"[green]Model ID Created/Found:[/] [cyan]{Markup.Escape(Cache.CurrentModelId)}[/]"); + + var pyManager = new PythonManager(Cache.MagicQuantDirectory!); + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await EnsureSqliteReadyAsync(); + + var benchmarkService = new BenchmarkService(pyManager); + var quantizationService = new QuantizationService(benchmarkService); + var imatrixService = new ImatrixService(); + + string q8QuantizationKey = BaselineQuants.Q8_0.Names[0]; + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + var bf16ModelGgufPath = await quantizationService.EnsureBaseModelFileAsync(true); + + var sidecarService = new ModelSidecarArtifactService(pyManager); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await sidecarService.EnsureMmprojArtifactAvailableAsync(); + + // Review the active regex profile against the native/BF16 tensor list before + // architecture/profile-scoped learning truth is persisted or reused. This is + // the early "do these groups look sane?" gate for catching YAML regex mistakes. + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await new TensorGroupReviewService().ReviewNativeTensorGroupingAsync( + quantizationService: quantizationService, + nativeGgufPath: bf16ModelGgufPath, + requireConfirmation: Cache.ConfirmTensorGroupProfile); + + var architectureFamilyService = new ArchitectureFamilyService(pyManager); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await architectureFamilyService.EnsureCurrentArchitectureFamilyAsync(bf16ModelGgufPath); + + var tensorGroupProfileService = new TensorGroupProfileService(); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await tensorGroupProfileService.EnsureCurrentProfileAsync(); + + var customBaselineService = new HuggingFaceBaselineService(pyManager); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + var resolvedCustomBaselines = await customBaselineService.PrecheckAndRegisterConfiguredBaselinesAsync(); + + if (Config.Current.Baselines.CustomRepositories.Any(x => x.Enabled) && resolvedCustomBaselines.Count == 0) + { + throw new InvalidOperationException( + "Custom baseline repositories were enabled, but no custom baselines resolved into the runtime registry."); + } + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await new TargetedRelearnService().PlanConfirmAndExecuteAsync(resolvedCustomBaselines); + + var imatrixRequest = new ImatrixRequest + { + UseImatrix = Cache.UseImatrix, + ForceRebuild = Cache.ForceImatrixRebuild, + ImatrixUrl = Config.Current.Imatrix.ImatrixUrl, + DatasetRepo = Config.Current.Imatrix.DatasetRepo, + DatasetSplit = Config.Current.Imatrix.DatasetSplit, + DatasetConfig = Config.Current.Imatrix.DatasetConfig, + LocalDatasetFile = Config.Current.Imatrix.DatasetLocalFile, + ModelDirectory = Cache.ModelDirectory!, + MagicQuantDirectory = Cache.ModelMagicQuantDirectory! + }; + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + var imatrixEnsureResult = await imatrixService.EnsureImatrixAsync(imatrixRequest, ct: MagicQuant.Runtime.RunCancellation.Token); + + if (imatrixEnsureResult.Enabled) + { + string canonicalPath = imatrixEnsureResult.CanonicalImatrixPath ?? "n/a"; + string rebuiltText = imatrixEnsureResult.Rebuilt ? "yes" : "no"; + AnsiConsole.MarkupLine( + $"[green]Imatrix active:[/] {Markup.Escape(canonicalPath)} (rebuilt={rebuiltText})"); + } + else + { + AnsiConsole.MarkupLine("[grey]Imatrix disabled for this run.[/]"); + } + + // Re-assert the live runtime flag from the imatrix resolution result so later phases + // cannot accidentally inherit a stale default. + RuntimeSearchSpace.SetImatrixAvailability(imatrixEnsureResult.Enabled); + + if (Cache.RebucketLearnedTensorGroupsFromExistingTruth) + { + await new TensorGroupRebucketService().RebucketFromExistingProfileTruthAsync(); + } + + string baseTypeName = (Cache.TorchType ?? Cache.MainTorchType.BF16).ToString(); + bool loadedPlanFromCache = !Cache.ForceRefreshHardwareProbe && + await benchmarkService.TryInitializeDynamicExecutionPlanFromCacheAsync( + q8QuantizationKey: q8QuantizationKey, + nativeModelPath: bf16ModelGgufPath, + nativeQuantizationKey: baseTypeName); + + if (!loadedPlanFromCache) + { + AnsiConsole.MarkupLine("[grey]Dynamic execution-plan cache not usable; probing Q8 + native anchors...[/]"); + await using var q8Lease = await quantizationService.BuildPureQ8ProbeLeaseAsync(); + await benchmarkService.EnsureDynamicExecutionPlanAsync( + q8ModelPath: q8Lease.GgufPath, + nativeModelPath: bf16ModelGgufPath, + q8QuantizationKey: q8QuantizationKey, + nativeQuantizationKey: baseTypeName, + forceRediscovery: Cache.ForceRefreshHardwareProbe); + } + + bool nativeTruthAlreadyLearned = + await quantizationService.HasNativeSourceLearnedTruthAsync(); + if (nativeTruthAlreadyLearned && loadedPlanFromCache) + { + AnsiConsole.MarkupLine( + "[grey]Native-source truth already exists and dynamic plan loaded from cache.[/]"); + } + var benchmarkRootDir = Path.Combine(Cache.ModelMagicQuantDirectory!, "Benchmarks"); + var baseBenchDir = Path.Combine(benchmarkRootDir, baseTypeName); + var baseLogitsDir = Path.Combine(baseBenchDir, "logits"); + var pplCorporaDir = Path.Combine(benchmarkRootDir, "_ppl_corpora"); + + var baseModelQuant = HybridQuant.CreatePureBaseline(BaselineQuants.GetBF16Quant()); + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await EnsureNativeBenchmarkEnvironmentReadyAsync( + benchmarkService: benchmarkService, + quantizationService: quantizationService, + baseModelQuant: baseModelQuant, + bf16ModelGgufPath: bf16ModelGgufPath, + baseBenchDir: baseBenchDir, + baseLogitsDir: baseLogitsDir, + pplCorporaDir: pplCorporaDir, + nativeTruthAlreadyLearned: nativeTruthAlreadyLearned); + + var compatibilityService = new ModelCompatibilityService(pyManager); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await compatibilityService.RunCompatibilityCheckAsync(bf16ModelGgufPath); + + // Compatibility must not be allowed to silently downgrade the live policy flags for the + // remainder of the pipeline run. Re-assert them here as a final safeguard. + RuntimeSearchSpace.SetImatrixAvailability(imatrixEnsureResult.Enabled); + RuntimeSearchSpace.AllowHighPrecisionHybrids = Config.Current.Flags.AllowHighPrecisionHybrids; + + PrintCustomBaselineRuntimeSummary(resolvedCustomBaselines, imatrixEnsureResult.Enabled); + + var comboCountBefore = ComboCounter.CountAll(); + var totalLearnedPruningResult = new LearnedBaselinePruningResult(); + + AnsiConsole.MarkupLine("[grey]Learned-baseline early pruning is disabled for this build. Startup sampling will proceed without learned-scheme candidate elimination.[/]"); + + AnsiConsole.Write(new Rule("[yellow]Initial Isolation Startup Samples[/]") { Justification = Justify.Left }); + + var isolationPlanner = new IsolationPlanningService(); + var initialPlan = isolationPlanner.BuildInitialPlan(Cache.UnusedTensorGroups); + + AnsiConsole.MarkupLine($"[grey]Queued initial startup samples:[/] [cyan]{initialPlan.TotalCount:N0}[/]"); + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + var initialSummary = await quantizationService.ProcessHybridBatchAsync( + initialPlan.Plans, + new StageProgressOptions + { + StageName = "Initial isolation startup samples", + Total = initialPlan.TotalCount, + MinimumNonSkippedSamplesBeforeEta = 2, + ShowEta = true, + CountSkippedForEta = false + }, + default); + + AnsiConsole.MarkupLine("[bold green]Initial startup sampling complete.[/]"); + AnsiConsole.MarkupLine($" [green]Completed:[/] {initialSummary.Completed:N0}"); + AnsiConsole.MarkupLine($" [yellow]Skipped existing:[/] {initialSummary.Skipped:N0}"); + AnsiConsole.MarkupLine($" [red]Failed:[/] {initialSummary.Failed:N0}"); + + AnsiConsole.MarkupLine("[bold magenta]Pipeline progress:[/] startup sampling finished. Learned-baseline early pruning remains disabled for subsequent phases."); + + var isolationOptimizer = new IsolationOptimizationService(); + + AnsiConsole.Write(new Rule("[yellow]Initial Probe Analysis[/]") { Justification = Justify.Left }); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + var initialAnalysis = await isolationOptimizer.AnalyzeInitialIsolationProbesAsync(initialPlan); + + AnsiConsole.Write(new Rule("[yellow]Initial Probe Group Decisions[/]") { Justification = Justify.Left }); + PrintIsolationGroupDecisions(initialAnalysis.GroupDetails); + + foreach (var note in initialAnalysis.Notes) + AnsiConsole.MarkupLine($" [grey]- {Markup.Escape(note)}[/]"); + + SearchSpaceDebugPrinter.PrintCurrentSearchSpace("Search Space After Initial Probe Analysis"); + + AnsiConsole.Write(new Rule("[yellow]Continuation Isolation Samples[/]") { Justification = Justify.Left }); + + AnsiConsole.MarkupLine($"[grey]Groups continuing after early probe:[/] [cyan]{initialAnalysis.GroupsToContinue.Count:N0}[/]"); + + var continuationPlan = isolationPlanner.BuildContinuationPlan( + initialAnalysis.GroupsToContinue, + Cache.UnusedTensorGroups); + + if (continuationPlan.TotalCount > 0) + { + AnsiConsole.MarkupLine($"[grey]Queued continuation samples:[/] [cyan]{continuationPlan.TotalCount:N0}[/]"); + + var continuationSummary = await quantizationService.ProcessHybridBatchAsync( + continuationPlan.Plans, + new StageProgressOptions + { + StageName = "Continuation isolation samples", + Total = continuationPlan.TotalCount, + MinimumNonSkippedSamplesBeforeEta = 2, + ShowEta = true, + CountSkippedForEta = false + }, + default); + + AnsiConsole.MarkupLine("[bold green]Continuation sampling complete.[/]"); + AnsiConsole.MarkupLine($" [green]Completed:[/] {continuationSummary.Completed:N0}"); + AnsiConsole.MarkupLine($" [yellow]Skipped existing:[/] {continuationSummary.Skipped:N0}"); + AnsiConsole.MarkupLine($" [red]Failed:[/] {continuationSummary.Failed:N0}"); + } + else + { + AnsiConsole.MarkupLine("[grey]No continuation samples were required after smallest-first gating.[/]"); + } + + var mergedPlan = initialPlan.MergeWith(continuationPlan); + + var archivalGroupIds = TReg.All + .Where(x => !Cache.UnusedTensorGroups.Any(u => u.UniqueId == x.UniqueId)) + .Select(x => x.UniqueId) + .Except(initialAnalysis.GroupsToContinue) + .OrderBy(x => x) + .ToList(); + + SearchSpaceDebugPrinter.PrintCurrentSearchSpace("Search Space Before Final Isolation Optimization"); + + AnsiConsole.Write(new Rule("[yellow]Final Isolation Optimization[/]") { Justification = Justify.Left }); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + var isolationResult = await isolationOptimizer.AnalyzeAndApplyFinalAsync(mergedPlan); + + SearchSpaceDebugPrinter.PrintCurrentSearchSpace("Search Space After Final Isolation Optimization"); + + foreach (var gd in isolationResult.GroupDetails.OrderBy(x => x.GroupName)) + { + AnsiConsole.Write( + new Rule($"[yellow]Isolation Group: {Markup.Escape(gd.GroupName)}[/]") + { + Justification = Justify.Left + }); + + AnsiConsole.MarkupLine($"[green]Best savings:[/] {gd.BestReductionRatio:P2}"); + AnsiConsole.MarkupLine($"[green]Winning candidate:[/] {Markup.Escape(gd.WinningCandidate ?? "n/a")}"); + AnsiConsole.MarkupLine($"[green]Explicit quant banned:[/] {(gd.ExplicitQuantBanned ? "[red]yes[/]" : "[green]no[/]")}"); + AnsiConsole.MarkupLine($"[green]BF16 suppressed:[/] {(gd.Bf16Suppressed ? "[yellow]yes[/]" : "[green]no[/]")}"); + + foreach (var line in gd.Candidates) + AnsiConsole.MarkupLine($" [grey]- {Markup.Escape(line)}[/]"); + } + + var comboCountAfterRulePruning = ComboCounter.CountAll(); + + var dbService = new QuantDatabaseService(); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await dbService.InitializeAsync(forceRebuild: true); + + // The old MDA/predicted-size ceiling pass is intentionally removed. + // DuckDB now stays as the allowed candidate universe, and the rank-safe + // isolation predictor chooses which candidates deserve real validation. + long predictedSizePruned = 0; + long highPrecisionPruned = await dbService.PruneHighPrecisionHybridCandidatesAsync(); + + AnsiConsole.MarkupLine($"[green]Learned-baseline eliminations:[/] {totalLearnedPruningResult.GroupCandidateEliminations:N0} [grey](early pruning disabled)[/]"); + AnsiConsole.MarkupLine($"[green]Baselines skipped without learned rows:[/] {totalLearnedPruningResult.BaselinesSkippedWithoutLearnedRows:N0} [grey](early pruning disabled)[/]"); + AnsiConsole.MarkupLine($"[green]Groups reduced to explicit-banned->Q8-fallback:[/] {isolationResult.ExplicitQuantBannedGroups:N0}"); + AnsiConsole.MarkupLine($"[green]BF16-suppressed groups:[/] {isolationResult.Bf16SuppressedGroups:N0}"); + AnsiConsole.MarkupLine($"[green]Hard damage eliminations:[/] {isolationResult.HardDamageEliminations:N0}"); + AnsiConsole.MarkupLine($"[green]Dominance eliminations:[/] {isolationResult.DominatedGroupCandidatesBanned:N0}"); + AnsiConsole.MarkupLine($"[green]Bad trade eliminations:[/] {isolationResult.BadTradeEliminations:N0}"); + AnsiConsole.MarkupLine($"[green]Synergy second-chance reinstatements:[/] {isolationResult.SynergySecondChanceReinstatements:N0}"); + AnsiConsole.MarkupLine($"[green]Final KLD cleanup eliminations:[/] {isolationResult.FinalKldCleanupEliminations:N0}"); + AnsiConsole.MarkupLine($"[green]Disabled combination baselines:[/] {isolationResult.DisabledBaselines:N0}"); + AnsiConsole.MarkupLine($"[green]Combination count before pruning:[/] {comboCountBefore:N0}"); + AnsiConsole.MarkupLine($"[green]Combination count after rule pruning:[/] {comboCountAfterRulePruning:N0}"); + AnsiConsole.MarkupLine($"[green]Predicted-size combo removals:[/] {predictedSizePruned:N0} [grey](obsolete MDA ceiling pruning removed)[/]"); + AnsiConsole.MarkupLine($"[green]Late-stage high-precision combo removals:[/] {highPrecisionPruned:N0}"); + + foreach (var note in isolationResult.Notes) + AnsiConsole.MarkupLine($" [grey]- {Markup.Escape(note)}[/]"); + + long finalRemainingCombinationCount = await dbService.GetRemainingCombinationCountAsync(); + + AnsiConsole.MarkupLine($"[green]Final surviving combinations after stage-1 pruning:[/] {finalRemainingCombinationCount:N0}"); + + AnsiConsole.Write(new Rule("[yellow]Archival Isolation Coverage[/]") { Justification = Justify.Left }); + + var archivalCoveragePlan = isolationPlanner.BuildArchivalCoveragePlan( + groupIdsToArchive: archivalGroupIds, + existingPlanKeys: mergedPlan.Plans.Select(x => x.Key), + missingTensorGroups: Cache.UnusedTensorGroups); + + var archivalCoverageGroups = archivalCoveragePlan.Plans + .Where(x => x.TargetGroupId.HasValue) + .Select(x => x.TargetGroupId!.Value) + .Distinct() + .Count(); + + AnsiConsole.MarkupLine($"[grey]Groups queued for archival coverage:[/] [cyan]{archivalCoverageGroups:N0}[/]"); + AnsiConsole.MarkupLine($"[grey]Non-continuing groups targeted for archival fill:[/] [cyan]{archivalGroupIds.Count:N0}[/]"); + AnsiConsole.MarkupLine("[grey]This pass does not feed current-run pruning; it only fills missing isolated-sample coverage in the database for groups that were fixed/collapsed out of combo exploration.[/]"); + + if (archivalCoveragePlan.TotalCount > 0) + { + AnsiConsole.MarkupLine($"[grey]Queued archival isolation samples:[/] [cyan]{archivalCoveragePlan.TotalCount:N0}[/]"); + + var archivalCoverageSummary = await quantizationService.ProcessHybridBatchAsync( + archivalCoveragePlan.Plans, + new StageProgressOptions + { + StageName = "Archival isolation coverage samples", + Total = archivalCoveragePlan.TotalCount, + MinimumNonSkippedSamplesBeforeEta = 2, + ShowEta = true, + CountSkippedForEta = false + }, + default); + + AnsiConsole.MarkupLine("[bold green]Archival isolation coverage complete.[/]"); + AnsiConsole.MarkupLine($" [green]Completed:[/] {archivalCoverageSummary.Completed:N0}"); + AnsiConsole.MarkupLine($" [yellow]Skipped existing:[/] {archivalCoverageSummary.Skipped:N0}"); + AnsiConsole.MarkupLine($" [red]Failed:[/] {archivalCoverageSummary.Failed:N0}"); + } + else + { + AnsiConsole.MarkupLine("[grey]No archival isolation coverage samples were required.[/]"); + } + + var finalIsolationManifestPlan = mergedPlan.MergeWith(archivalCoveragePlan); + + var survivalPipeline = new CombinationSurvivalPipelineService(quantizationService); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + var finalizationResult = await survivalPipeline.RunAsync( + isolationSamplePlan: finalIsolationManifestPlan, + isolationOptimizationResult: isolationResult, + ct: MagicQuant.Runtime.RunCancellation.Token); + + AnsiConsole.Write(new Rule("[yellow]Export Summary[/]") { Justification = Justify.Left }); + AnsiConsole.MarkupLine($"[green]Export directory:[/] [blue]{Markup.Escape(Cache.OutputDirectory ?? "n/a")}[/]"); + AnsiConsole.MarkupLine($"[green]Final brutal survivors:[/] [cyan]{finalizationResult.BrutalSurvivors.Count:N0}[/]"); + AnsiConsole.MarkupLine($"[green]Selected survivors:[/] [cyan]{finalizationResult.SelectedRows.Count(x => x.Enabled):N0}[/]"); + AnsiConsole.MarkupLine($"[green]Exported/linkable artifacts:[/] [cyan]{finalizationResult.ExportedArtifacts.Count:N0}[/]"); + } + + private static async Task EnsureNativeBenchmarkEnvironmentReadyAsync( + BenchmarkService benchmarkService, + QuantizationService quantizationService, + HybridQuant baseModelQuant, + string bf16ModelGgufPath, + string baseBenchDir, + string baseLogitsDir, + string pplCorporaDir, + bool nativeTruthAlreadyLearned) + { + var status = ValidateNativeBenchmarkEnvironment( + baseBenchDir: baseBenchDir, + baseLogitsDir: baseLogitsDir, + pplCorporaDir: pplCorporaDir, + requiredDomains: RequiredNativeKldDomains); + + bool mustRegenerateNativeBenchmarkArtifacts = + !status.IsValid; + + if (mustRegenerateNativeBenchmarkArtifacts) + { + AnsiConsole.Write(new Rule("[yellow]Native BF16 Benchmark/KLD Artifact Validation[/]") { Justification = Justify.Left }); + + AnsiConsole.MarkupLine("[yellow]Native BF16 benchmark/KLD artifacts are missing or incomplete.[/] Regenerating required artifacts."); + + PrintNativeBenchmarkEnvironmentIssues(status); + + await ForceRegenerateNativeBenchmarkArtifactsAsync( + benchmarkService: benchmarkService, + baseModelQuant: baseModelQuant, + bf16ModelGgufPath: bf16ModelGgufPath, + baseBenchDir: baseBenchDir, + baseLogitsDir: baseLogitsDir); + + status = ValidateNativeBenchmarkEnvironment( + baseBenchDir: baseBenchDir, + baseLogitsDir: baseLogitsDir, + pplCorporaDir: pplCorporaDir, + requiredDomains: RequiredNativeKldDomains); + + if (!status.IsValid) + { + var details = string.Join( + Environment.NewLine, + status.MissingOrInvalidArtifacts.Select(x => $"- {x}")); + + throw new InvalidOperationException( + "Native BF16 benchmark/logit generation completed, but required native benchmark artifacts are still missing or invalid. " + + "This is fatal because every non-base benchmark requires complete native KLD logits." + + Environment.NewLine + + details); + } + + AnsiConsole.MarkupLine("[green]Native BF16 benchmark/KLD artifacts validated.[/]"); + } + else + { + AnsiConsole.MarkupLine("[grey]Native BF16 benchmark/KLD artifacts already exist and passed validation.[/]"); + } + + // Disk artifact validation is not enough. Native tensor learning is tied to the + // persisted TensorCombo/AiBenchmark identity. The repair path above may run in + // transient mode so it can regenerate logits even when stale DB truth exists; after + // the artifacts are valid, explicitly hydrate/validate the SQLite benchmark row + // from those artifacts before native-source learning tries to attach to it. + await EnsureNativeBenchmarkDbTruthAsync( + benchmarkService: benchmarkService, + baseModelQuant: baseModelQuant, + bf16ModelGgufPath: bf16ModelGgufPath, + baseBenchDir: baseBenchDir, + baseLogitsDir: baseLogitsDir); + + if (!nativeTruthAlreadyLearned) + { + await quantizationService.LearnNativeSourceTruthAsync(bf16ModelGgufPath); + } + else + { + AnsiConsole.MarkupLine( + "[grey]Skipping native-source tensor relearn because learned native-source truth already exists.[/]"); + } + } + + private static async Task EnsureNativeBenchmarkDbTruthAsync( + BenchmarkService benchmarkService, + HybridQuant baseModelQuant, + string bf16ModelGgufPath, + string baseBenchDir, + string baseLogitsDir) + { + bool previousSuppressBenchmarkPersistence = Cache.SuppressBenchmarkPersistence; + + try + { + Cache.SuppressBenchmarkPersistence = false; + + await benchmarkService.RunAllBenchmarksAsync( + quantConfig: baseModelQuant, + modelPath: bf16ModelGgufPath, + benchDir: baseBenchDir, + klLogitsDir: baseLogitsDir, + saveLogits: true, + domainsOverride: RequiredNativeKldDomains); + + AnsiConsole.MarkupLine("[grey]Native BF16 benchmark DB truth hydrated/validated.[/]"); + } + finally + { + Cache.SuppressBenchmarkPersistence = previousSuppressBenchmarkPersistence; + } + } + + private static async Task ForceRegenerateNativeBenchmarkArtifactsAsync( + BenchmarkService benchmarkService, + HybridQuant baseModelQuant, + string bf16ModelGgufPath, + string baseBenchDir, + string baseLogitsDir) + { + if (Directory.Exists(baseBenchDir)) + { + AnsiConsole.MarkupLine( + $"[grey]Clearing incomplete/stale native benchmark directory:[/] {Markup.Escape(baseBenchDir)}"); + + Directory.Delete(baseBenchDir, recursive: true); + } + + Directory.CreateDirectory(baseBenchDir); + Directory.CreateDirectory(baseLogitsDir); + + bool previousSuppressBenchmarkPersistence = Cache.SuppressBenchmarkPersistence; + + try + { + // This is intentional. + // + // If persisted native BF16 benchmark rows already exist in SQLite, the normal + // BenchmarkService path may return DB truth without actually running llama-perplexity, + // which means missing KLD logits would stay missing forever. + // + // Transient mode forces this artifact-repair pass to rely on disk execution instead + // of DB benchmark truth. The native tensor truth is learned separately below. + Cache.SuppressBenchmarkPersistence = true; + + await benchmarkService.RunAllBenchmarksAsync( + quantConfig: baseModelQuant, + modelPath: bf16ModelGgufPath, + benchDir: baseBenchDir, + klLogitsDir: baseLogitsDir, + saveLogits: true, + domainsOverride: RequiredNativeKldDomains); + } + finally + { + Cache.SuppressBenchmarkPersistence = previousSuppressBenchmarkPersistence; + } + } + + private static NativeBenchmarkEnvironmentStatus ValidateNativeBenchmarkEnvironment( + string baseBenchDir, + string baseLogitsDir, + string pplCorporaDir, + IReadOnlyCollection requiredDomains) + { + var issues = new List(); + + if (string.IsNullOrWhiteSpace(baseBenchDir)) + { + issues.Add("Native benchmark directory path is null/empty."); + } + else if (!Directory.Exists(baseBenchDir)) + { + issues.Add($"Native benchmark directory does not exist: {baseBenchDir}"); + } + + if (string.IsNullOrWhiteSpace(baseLogitsDir)) + { + issues.Add("Native KLD logits directory path is null/empty."); + } + else if (!Directory.Exists(baseLogitsDir)) + { + issues.Add($"Native KLD logits directory does not exist: {baseLogitsDir}"); + } + + if (string.IsNullOrWhiteSpace(pplCorporaDir)) + { + issues.Add("_ppl_corpora directory path is null/empty."); + } + else if (!Directory.Exists(pplCorporaDir)) + { + issues.Add($"_ppl_corpora directory does not exist: {pplCorporaDir}"); + } + else if (!Directory.EnumerateFiles(pplCorporaDir, "*", SearchOption.AllDirectories).Any()) + { + issues.Add($"_ppl_corpora directory exists but contains no files: {pplCorporaDir}"); + } + + foreach (var domain in requiredDomains.OrderBy(x => x, StringComparer.Ordinal)) + { + if (!string.IsNullOrWhiteSpace(baseBenchDir) && Directory.Exists(baseBenchDir)) + { + var pplLog = Path.Combine(baseBenchDir, $"perplexity_{domain}.log"); + + if (!File.Exists(pplLog)) + { + issues.Add($"Missing native BF16 perplexity log for domain '{domain}': {pplLog}"); + } + else if (new FileInfo(pplLog).Length <= 0) + { + issues.Add($"Native BF16 perplexity log is empty for domain '{domain}': {pplLog}"); + } + } + + if (!string.IsNullOrWhiteSpace(baseLogitsDir) && Directory.Exists(baseLogitsDir)) + { + var logitsFile = Path.Combine(baseLogitsDir, $"kld_logits_{domain}.bin"); + + if (!File.Exists(logitsFile)) + { + issues.Add($"Missing native KLD logits for domain '{domain}': {logitsFile}"); + } + else if (new FileInfo(logitsFile).Length <= 0) + { + issues.Add($"Native KLD logits file is empty for domain '{domain}': {logitsFile}"); + } + } + } + + return new NativeBenchmarkEnvironmentStatus( + IsValid: issues.Count == 0, + MissingOrInvalidArtifacts: issues); + } + + private static void PrintNativeBenchmarkEnvironmentIssues(NativeBenchmarkEnvironmentStatus status) + { + if (status.IsValid) + return; + + foreach (var issue in status.MissingOrInvalidArtifacts.Take(20)) + AnsiConsole.MarkupLine($"[grey]- {Markup.Escape(issue)}[/]"); + + if (status.MissingOrInvalidArtifacts.Count > 20) + { + AnsiConsole.MarkupLine( + $"[grey]- ...and {status.MissingOrInvalidArtifacts.Count - 20:N0} more issue(s).[/]"); + } + } + + private sealed record NativeBenchmarkEnvironmentStatus( + bool IsValid, + IReadOnlyList MissingOrInvalidArtifacts); + + private static void PrintIsolationGroupDecisions(IEnumerable decisions) + { + foreach (var gd in decisions.OrderBy(x => x.GroupName)) + { + AnsiConsole.Write( + new Rule($"[yellow]Isolation Group: {Markup.Escape(gd.GroupName)}[/]") + { + Justification = Justify.Left + }); + + AnsiConsole.MarkupLine($"[green]Best savings:[/] {gd.BestReductionRatio:P2}"); + AnsiConsole.MarkupLine($"[green]Winning candidate:[/] {Markup.Escape(gd.WinningCandidate ?? "n/a")}"); + AnsiConsole.MarkupLine($"[green]Explicit quant banned:[/] {(gd.ExplicitQuantBanned ? "[red]yes[/]" : "[green]no[/]")}"); + AnsiConsole.MarkupLine($"[green]BF16 suppressed:[/] {(gd.Bf16Suppressed ? "[yellow]yes[/]" : "[green]no[/]")}"); + + foreach (var line in gd.Candidates) + AnsiConsole.MarkupLine($" [grey]- {Markup.Escape(line)}[/]"); + } + } + + private static void PrintCustomBaselineRuntimeSummary( + IReadOnlyCollection resolvedCustomBaselines, + bool hasUsableImatrix) + { + AnsiConsole.Write(new Rule("[yellow]Custom Baseline Runtime Summary[/]") { Justification = Justify.Left }); + + var learning = BaselineQuants.GetLearningBaselines(hasUsableImatrix); + var carriers = BaselineQuants.GetCombinationCarrierBaselines(hasUsableImatrix); + var explicitCandidates = BaselineQuants.GetGroupCombinationCandidates(hasUsableImatrix, Config.Current.Flags.AllowHighPrecisionHybrids); + + AnsiConsole.MarkupLine($"[grey]Learning baselines in runtime registry:[/] [cyan]{learning.Count:N0}[/]"); + AnsiConsole.MarkupLine($"[grey]Combination carriers in runtime registry:[/] [cyan]{carriers.Count:N0}[/]"); + AnsiConsole.MarkupLine($"[grey]Explicit group candidates in runtime registry:[/] [cyan]{explicitCandidates.Count:N0}[/]"); + + if (resolvedCustomBaselines.Count == 0) + { + AnsiConsole.MarkupLine("[grey]No custom baselines were resolved for this run.[/]"); + return; + } + + AnsiConsole.MarkupLine($"[green]Custom baselines registered:[/] [cyan]{resolvedCustomBaselines.Count:N0}[/]"); + + foreach (var custom in resolvedCustomBaselines.OrderBy(x => x.DynamicBaselineId)) + { + bool inLearning = learning.Any(x => x.UniqueId == custom.DynamicBaselineId); + bool inCarriers = carriers.Any(x => x.UniqueId == custom.DynamicBaselineId); + bool inExplicit = explicitCandidates.Any(x => x.UniqueId == custom.DynamicBaselineId); + + string revision = string.IsNullOrWhiteSpace(custom.Revision) ? "main" : custom.Revision; + AnsiConsole.MarkupLine( + $" [cyan]{custom.DynamicBaselineId}[/] [yellow]{Markup.Escape(custom.DisplayName)}[/] family={Markup.Escape(custom.BaselineFamily)} file={Markup.Escape(custom.SourceFileName)} revision={Markup.Escape(revision)} learning={inLearning} carrier={inCarriers} explicit={inExplicit}"); + } + } + + private void ShowPipelineHelp() + { + AnsiConsole.MarkupLine("[bold yellow]Command: pipeline (legacy alias: evolution)[/]"); + AnsiConsole.WriteLine("Runs the full quantization search on a target model, then uses rank-safe isolation prediction to choose validated final hybrids."); + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[bold]Usage:[/]"); + AnsiConsole.WriteLine(" mq pipeline --model-dir \"\" [options]"); + AnsiConsole.WriteLine(" mq pipeline --config \"./config.default.yaml\""); + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[bold]Arguments:[/]"); + AnsiConsole.MarkupLine(" [green]--model-dir[/] Path to the model directory containing .safetensors files (Optional if set in YAML)"); + AnsiConsole.MarkupLine(" [green]--magic-quant-root[/] Isolated runtime root containing MagicQuant_SQLite.db and shared runtime assets (Optional)"); + AnsiConsole.MarkupLine(" [green]--recheck-hardware-probe[/] Force hardware/Q8 probe and update cached plan in SQLite (Optional)"); + AnsiConsole.MarkupLine(" [green]--use-imatrix[/] Enable imatrix acquisition/build and allow imatrix-required search candidates (Optional)"); + AnsiConsole.MarkupLine(" [green]--allow-high-precision-hybrids[/] Keep BF16/F16 explicit group candidates in final surviving combos (Optional, default false)"); + AnsiConsole.MarkupLine(" [green]--imatrix-force-rebuild[/] Delete/rebuild canonical imatrix artifacts before run (Optional)"); + AnsiConsole.MarkupLine(" [green]--imatrix-url[/] HTTPS URL for direct imatrix artifact download (Optional)"); + AnsiConsole.MarkupLine(" [green]--imatrix-dataset-repo[/] Hugging Face dataset repo ID for imatrix generation (Optional)"); + AnsiConsole.MarkupLine(" [green]--imatrix-dataset-split[/] Dataset split for HF/local dataset source metadata/build (Optional)"); + AnsiConsole.MarkupLine(" [green]--imatrix-dataset-config[/] Optional dataset config name for HF datasets (Optional)"); + AnsiConsole.MarkupLine(" [green]--imatrix-dataset-local-file[/] Full path to local .json/.jsonl dataset source (Optional)"); + AnsiConsole.MarkupLine(" [green]--selection-near-baseline-max-size-growth-percent[/] Phase-2 size premium for replacing a smaller/higher-damage anchor (Optional; default = 1.0)"); + AnsiConsole.MarkupLine(" [green]--selection-interior-window-fractions[/] Comma-separated phase-3 interior windows, e.g. 0.35,0.35 (Optional)"); + AnsiConsole.MarkupLine(" [green]--prediction-bit-stress-threshold-candidates[/] Comma-separated interaction-fit thresholds, e.g. 4,5,6,7,8,9,10,11,12 (Optional)"); + AnsiConsole.MarkupLine(" [green]--output-dir[/] Final export/output directory for selected survivor artifacts (Optional; default = /MagicQuant/Final_Outputs)"); + AnsiConsole.MarkupLine(" [green]--output-name-prefix[/] Output filename prefix for exported GGUF files (Optional; default = Model)"); + AnsiConsole.MarkupLine(" [green]--reuse-existing-final-artifacts[/] Reuse valid final GGUFs only when exact file name + benchmark byte size match (Optional; default false)"); + AnsiConsole.MarkupLine(" [green]--allow-eight-bit-anchor-replacements[/] Permit final prediction to try replacing 8-bit anchors like Q8_0 (see YAML policy)"); + AnsiConsole.MarkupLine(" [green]--export-external-learned-baselines[/] Also locally rebuild/export pure learned external baselines such as Unsloth (Optional; default false)"); + AnsiConsole.MarkupLine(" [green]--rebucket-learned-tensor-groups[/] Compatibility alias; regex rebucketing from DB is enabled by default"); + AnsiConsole.MarkupLine(" [green]--no-rebucket-learned-tensor-groups[/] Disable safe DB rebucketing and force the slower/full learned-group path instead"); + AnsiConsole.MarkupLine(" [green]--skip-tensor-group-confirm[/] Skip the native BF16 tensor-group review confirmation prompt for unattended runs (Optional; YAML default true asks)"); + AnsiConsole.MarkupLine(" [green]--selection-max-candidates-per-interior-window[/] Candidate count retained per interior window (Optional; default = 1)"); + AnsiConsole.MarkupLine(" [green]--config[/] Path to YAML runtime config. CLI flags override YAML values."); + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[bold]Example:[/]"); + AnsiConsole.WriteLine(" mq pipeline --model-dir \"C:\\Models\\Mistral-7B\""); + } + + private static string ResolveAndValidateOutputDirectory() + { + string resolved = OutputPathService.Pipeline( + Cache.ModelMagicQuantDirectory!, Config.Current.Output.OutputDir); + + Directory.CreateDirectory(resolved); + + string probe = Path.Combine(resolved, $".write_test_{Guid.NewGuid():N}.tmp"); + File.WriteAllText(probe, "ok"); + File.Delete(probe); + + return resolved; + } + + private static async Task EnsureSqliteReadyAsync(CancellationToken ct = default) + { + await using var db = new MagicQuantContext(); + await db.Database.MigrateAsync(ct); + + var model = await db.AiModelHashes + .FirstOrDefaultAsync(x => x.UniqueHash == Cache.CurrentModelId, ct); + + if (model != null) + return; + + db.AiModelHashes.Add(new AiModelHash { UniqueHash = Cache.CurrentModelId }); + await db.SaveChangesAsync(ct); + } +} diff --git a/src/MagicQuant/Commands/ValidatePredictions.cs b/src/MagicQuant/Commands/ValidatePredictions.cs new file mode 100644 index 0000000..eead297 --- /dev/null +++ b/src/MagicQuant/Commands/ValidatePredictions.cs @@ -0,0 +1,149 @@ +using MagicQuant.Helpers; +using MagicQuant.Models; +using MagicQuant.Services; +using Microsoft.EntityFrameworkCore; +using MQ.DB; +using MQ.DB.Data; +using Spectre.Console; + +namespace MagicQuant.Commands; + +public sealed class ValidatePredictions : ICommand +{ + public async Task Run(List args) + { + if (args.Any(a => string.Equals(a.Name, "help", StringComparison.OrdinalIgnoreCase))) + { + ShowHelp(); + return; + } + + string? modelDirRaw = args.FirstOrDefault(a => + string.Equals(a.Name, "model-dir", StringComparison.OrdinalIgnoreCase))?.Value; + + modelDirRaw = string.IsNullOrWhiteSpace(modelDirRaw) + ? Config.Current.Paths.ModelDir + : modelDirRaw; + + if (string.IsNullOrWhiteSpace(modelDirRaw)) + throw new InvalidOperationException("Missing model directory. Provide --model-dir or set paths.model_dir in YAML."); + + string modelDir = Path.GetFullPath(modelDirRaw); + if (!Directory.Exists(modelDir)) + throw new DirectoryNotFoundException($"Model directory does not exist: {modelDir}"); + + Cache.ModelDirectory = modelDir; + Cache.ModelMagicQuantDirectory = Path.Combine(modelDir, "MagicQuant"); + ModelRuntimePathService.InitializeForCurrentModel(); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await new ExternalBaselineCacheCleanupService().CleanupStaleArtifactsAsync(); + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await new ScratchStorageService(new ModelArtifactPathService()).CleanupStaleScratchArtifactsAsync(); + Directory.CreateDirectory(Cache.ModelMagicQuantDirectory); + + Cache.CurrentModelId = MagicQuantModelId.GetOrCreateModelId(modelDir); + JsonHelper.DetectAndSetTorchType(Cache.ModelDirectory); + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await ResolveArchitectureFamilyFromConfigAsync(); + + ApplyOptionalImatrixContext(args); + + string outputDir = ResolveOutputDirectory(args); + Directory.CreateDirectory(outputDir); + + var repository = new HybridBenchmarkRepository(); + var effectiveResolver = new EffectiveCandidateStateResolverService(repository); + var prediction = new RankSafeKldPredictionService(repository, effectiveResolver); + var validator = new PredictionValidationService(repository, prediction); + + MagicQuant.Runtime.RunCancellation.Token.ThrowIfCancellationRequested(); + await validator.ExportAsync(outputDir); + } + + private static async Task ResolveArchitectureFamilyFromConfigAsync() + { + Cache.CurrentArchitectureFamilyId = null; + + if (string.IsNullOrWhiteSpace(Cache.CurrentArchitectureFamilyName)) + return; + + string normalized = Cache.CurrentArchitectureFamilyNormalizedName; + + await using var db = new MagicQuantContext(); + var family = await db.ArchitectureFamilies + .AsNoTracking() + .FirstOrDefaultAsync(x => x.NormalizedName == normalized); + + if (family == null) + { + AnsiConsole.MarkupLine($"[yellow]Warning:[/] Architecture family '{Markup.Escape(Cache.CurrentArchitectureFamilyName)}' was configured but not found in SQLite. Validation will use the raw model hash scope."); + return; + } + + Cache.CurrentArchitectureFamilyId = family.Id; + AnsiConsole.MarkupLine($"[green]Architecture family scope:[/] {Markup.Escape(family.DisplayName)} (Id={family.Id})"); + } + + private static void ApplyOptionalImatrixContext(IReadOnlyList args) + { + string? imatrixPath = args.FirstOrDefault(a => string.Equals(a.Name, "imatrix-path", StringComparison.OrdinalIgnoreCase))?.Value; + string? imatrixHash = args.FirstOrDefault(a => string.Equals(a.Name, "imatrix-identity-hash", StringComparison.OrdinalIgnoreCase))?.Value; + + if (!string.IsNullOrWhiteSpace(imatrixPath)) + { + string fullPath = Path.GetFullPath(imatrixPath); + if (!File.Exists(fullPath)) + throw new FileNotFoundException($"Imatrix path does not exist: {fullPath}"); + + Cache.IsImatrixAvailable = true; + Cache.ActiveImatrixPath = fullPath; + Cache.ActiveImatrixIdentityHash = null; + ImatrixIdentityService.EnsureActiveImatrixIdentityHashAsync().GetAwaiter().GetResult(); + AnsiConsole.MarkupLine($"[green]Validation imatrix path:[/] {Markup.Escape(fullPath)}"); + return; + } + + if (!string.IsNullOrWhiteSpace(imatrixHash)) + { + Cache.IsImatrixAvailable = true; + Cache.ActiveImatrixPath = null; + Cache.ActiveImatrixIdentityHash = imatrixHash.Trim().ToLowerInvariant(); + AnsiConsole.MarkupLine($"[green]Validation imatrix identity:[/] {Markup.Escape(Cache.ActiveImatrixIdentityHash)}"); + return; + } + + Cache.IsImatrixAvailable = false; + Cache.ActiveImatrixPath = null; + Cache.ActiveImatrixIdentityHash = null; + + if (Config.Current.Flags.UseImatrix) + { + AnsiConsole.MarkupLine("[yellow]Warning:[/] flags.use_imatrix is true, but validate-predictions was not given --imatrix-path or --imatrix-identity-hash. Strict validation will use the no-imatrix bucket."); + } + } + + private static string ResolveOutputDirectory(IReadOnlyList args) + { + string? explicitOutput = args.FirstOrDefault(a => string.Equals(a.Name, "output-dir", StringComparison.OrdinalIgnoreCase))?.Value; + return OutputPathService.PredictionValidation( + Cache.ModelMagicQuantDirectory!, explicitOutput, Config.OutputDirectory); + } + + private static void ShowHelp() + { + AnsiConsole.MarkupLine("[bold]validate-predictions[/]"); + AnsiConsole.MarkupLine("Validates rank-safe isolation KLD predictions against existing SQLite category=General benchmark truth."); + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[yellow]Required:[/]"); + AnsiConsole.MarkupLine(" --model-dir HuggingFace source model directory, or set paths.model_dir in YAML"); + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[yellow]Optional:[/]"); + AnsiConsole.MarkupLine(" --architecture-family Uses configured family scope if present in SQLite"); + AnsiConsole.MarkupLine(" --imatrix-path Hash this imatrix and validate that exact bucket"); + AnsiConsole.MarkupLine(" --imatrix-identity-hash Validate an already-known imatrix bucket"); + AnsiConsole.MarkupLine(" --output-dir Report output directory"); + AnsiConsole.WriteLine(); + AnsiConsole.MarkupLine("[grey]Outputs prediction_validation_general.csv and prediction_validation_general.md.[/]"); + } +} diff --git a/src/MagicQuant/Config.cs b/src/MagicQuant/Config.cs new file mode 100644 index 0000000..b815286 --- /dev/null +++ b/src/MagicQuant/Config.cs @@ -0,0 +1,111 @@ +using MagicQuant.Configuration; + +namespace MagicQuant; + +/// +/// Process-wide normalized settings for one CLI run. Load through MagicQuantYamlLoader; +/// tests changing this state must restore the previous configuration. +/// +public static class Config +{ + public static MagicQuantYamlConfig Current { get; private set; } = MagicQuantYamlConfig.CreateDefault(); + + public static void Load(MagicQuantYamlConfig config) + { + Current = config ?? throw new ArgumentNullException(nameof(config)); + } + + public static void SetResolvedCustomBaselines(IEnumerable baselines) + { + Current.Baselines.ResolvedCustomBaselines = baselines?.ToList() ?? new List(); + } + + public static ResolvedCustomBaselineSpec? GetResolvedCustomBaseline(string canonicalKey) + { + return Current.Baselines.ResolvedCustomBaselines.FirstOrDefault(x => + string.Equals(x.CanonicalKey, canonicalKey, StringComparison.Ordinal)); + } + + public static ulong ManualMaxPredictedSizeBytes => Current.Prediction.ManualMaxPredictedSizeBytes; + + public static IReadOnlyList PredictionBitStressThresholdCandidates => + Current.Prediction.BitStressThresholdCandidates.Count == 0 + ? new[] { Current.Prediction.DefaultBitStressThreshold } + : Current.Prediction.BitStressThresholdCandidates; + + public static double PredictionDefaultBitStressThreshold => Current.Prediction.DefaultBitStressThreshold; + public static int PredictionMinimumFitRows => Math.Max(2, Current.Prediction.MinimumFitRows); + public static long MaxInMemoryCombinationLoadRows => Math.Max(1L, Current.Prediction.MaxInMemoryCombinationLoadRows); + + public static double SelectionNearBaselineMaxSizeGrowthPercent => + Math.Max(0d, Current.CandidateSelection.NearBaselineMaxSizeGrowthPercent); + + public static IReadOnlyList SelectionInteriorWindowFractions => + Current.CandidateSelection.InteriorWindowFractions.Count == 0 + ? new[] { 0.35d, 0.35d } + : Current.CandidateSelection.InteriorWindowFractions; + + public static int SelectionMaxCandidatesPerInteriorWindow => + Math.Max(1, Current.CandidateSelection.MaxCandidatesPerInteriorWindow); + + public static int SelectionMaxFallbackAttemptsPerAnchor => + Math.Max(1, Current.CandidateSelection.MaxFallbackAttemptsPerAnchor); + + public static bool SelectionSmartFallbackEnabled => + Current.CandidateSelection.SmartFallbackEnabled && SelectionSmartFallbackAttemptsPerFailure > 0; + + public static int SelectionSmartFallbackAttemptsPerFailure => + Math.Max(0, Current.CandidateSelection.SmartFallbackAttemptsPerFailure); + + public static int SelectionSmartFallbackMaxHigherFidelitySteps => + Math.Max(0, Current.CandidateSelection.SmartFallbackMaxHigherFidelitySteps); + + public static double SelectionMinimumKldImprovementEpsilon => + Math.Max(0d, Current.CandidateSelection.MinimumKldImprovementEpsilon); + + public static double SelectionMinimumNeighborGapFractionOfGlobalSpan => + Math.Clamp(Current.CandidateSelection.MinimumNeighborGapFractionOfGlobalSpan, 0d, 1d); + + public static double SelectionNearLowerAnchorBrutalZoneFractionOfPairSpan => + Math.Clamp(Current.CandidateSelection.NearLowerAnchorBrutalZoneFractionOfPairSpan, 0d, 1d); + + public static double SelectionNearAnchorRequiredKldGainFractionOfPairGap => + Math.Max(0d, Current.CandidateSelection.NearAnchorRequiredKldGainFractionOfPairGap); + + public static bool SelectionAllowEightBitAnchorReplacements => + Current.CandidateSelection.AllowEightBitAnchorReplacements; + + public static bool SelectionValidateAllAnomalyStrictCandidatesAfterSuccess => + Current.CandidateSelection.ValidateAllAnomalyStrictCandidatesAfterSuccess; + + public static bool SelectionDiversifyValidationCandidates => + Current.CandidateSelection.DiversifyValidationCandidates; + + public static int SelectionDiversityScanMultiplier => + Math.Max(1, Current.CandidateSelection.DiversityScanMultiplier); + + public static int SelectionDiversityScanMinCandidates => + Math.Max(1, Current.CandidateSelection.DiversityScanMinCandidates); + + public static int SelectionDiversityScanMaxCandidates => + Math.Max(SelectionDiversityScanMinCandidates, Current.CandidateSelection.DiversityScanMaxCandidates); + + public static bool SelectionDiversityLowBitOnly => + Current.CandidateSelection.DiversityLowBitOnly; + + public static RuntimeAnomalyDetectionConfig AnomalyDetection => Current.AnomalyDetection; + public static RuntimeSynergyDetectionConfig SynergyDetection => Current.SynergyDetection; + public static bool AnomalyDetectionEnabled => Current.AnomalyDetection.Enabled; + public static bool SynergyDetectionEnabled => Current.SynergyDetection.Enabled; + + public static string? OutputDirectory => Current.Output.OutputDir; + public static string OutputNamePrefix => string.IsNullOrWhiteSpace(Current.Output.OutputNamePrefix) + ? "Model" + : Current.Output.OutputNamePrefix.Trim(); + + public static bool ExportExternalLearnedBaselines => Current.Output.ExportExternalLearnedBaselines; + public static bool AttemptMmprojBuild => Current.Output.AttemptMmprojBuild; + public static bool RequireMmprojForVisionModels => Current.Output.RequireMmprojForVisionModels; + public static bool ReuseExistingFinalArtifacts => Current.Output.ReuseExistingFinalArtifacts; + +} diff --git a/src/MagicQuant/Configuration/CliOptionValidator.cs b/src/MagicQuant/Configuration/CliOptionValidator.cs new file mode 100644 index 0000000..9a3fbd3 --- /dev/null +++ b/src/MagicQuant/Configuration/CliOptionValidator.cs @@ -0,0 +1,125 @@ +using System.Globalization; +using MagicQuant.Models; + +namespace MagicQuant.Configuration; + +/// Rejects typos and ambiguous CLI values before any configuration or runtime mutation. +public static class CliOptionValidator +{ + private static readonly HashSet Flags = new(StringComparer.OrdinalIgnoreCase) + { + "allow-architecture-family-alias-override", + "allow-eight-bit-anchor-replacements", + "allow-high-precision-hybrids", + "allow-missing-manifest-tensors", + "check-config", + "disable-tensor-group-rebucket", + "export-external-learned-baselines", + "force-refresh-hardware-probe", + "force-relearn-baseline-tensor-mappings", + "force_refresh_hardware_probe", + "full-relearn-tensor-groups", + "help", + "imatrix-force-rebuild", + "no-rebucket-learned-tensor-groups", + "rebucket-learned-tensor-groups", + "rebucket-tensor-groups-from-db", + "recheck-hardware-probe", + "relearn-baseline-mappings", + "relearn-tensor-groups-from-db", + "reuse-existing-final-artifacts", + "skip-tensor-group-confirm", + "strict-config", + "update", + "use-imatrix", + "validate", + "validate-all-anomaly-strict-candidates-after-success", + "verify", + "yes-tensor-groups", + }; + private static readonly HashSet Values = new(StringComparer.OrdinalIgnoreCase) + { + "architecture-family", + "clone-json", + "clone-repo", + "config", + "convert-script", + "imatrix-dataset-config", + "imatrix-dataset-local-file", + "imatrix-dataset-repo", + "imatrix-dataset-split", + "imatrix-identity-hash", + "imatrix-path", + "imatrix-url", + "llama-bin", + "llama-root", + "magic-quant-root", + "manual-max-predicted-size-bytes", + "missing-manifest-base-quant", + "model-dir", + "output-dir", + "output-name-prefix", + "prediction-bit-stress-threshold-candidates", + "prediction-default-bit-stress-threshold", + "prediction-minimum-fit-rows", + "selection-diversify-validation-candidates", + "selection-diversity-low-bit-only", + "selection-diversity-scan-max-candidates", + "selection-diversity-scan-min-candidates", + "selection-diversity-scan-multiplier", + "selection-interior-window-fractions", + "selection-max-candidates-per-interior-window", + "selection-max-fallback-attempts-per-anchor", + "selection-minimum-kld-improvement-epsilon", + "selection-minimum-neighbor-gap-fraction", + "selection-near-anchor-required-kld-gain-fraction", + "selection-near-baseline-max-size-growth-percent", + "selection-near-lower-anchor-brutal-zone-fraction", + "source-json", + "source-repo", + }; + + public static void Validate(IReadOnlyList args) + { + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var arg in args) + { + string name = arg.Name ?? ""; + if (!seen.Add(name)) throw new ArgumentException($"Duplicate option --{name}. Supply it once."); + if (Flags.Contains(name)) + { + if (!string.IsNullOrEmpty(arg.Value)) throw new ArgumentException($"--{name} is a flag and takes no value. Set boolean policy in YAML when disabling it."); + } + else if (Values.Contains(name)) + { + if (string.IsNullOrWhiteSpace(arg.Value)) throw new ArgumentException($"--{name} requires a value."); + ValidateTypedValue(name.ToLowerInvariant(), arg.Value); + } + else throw new ArgumentException($"Unknown or removed option --{name}. See command --help and docs/configuration.md."); + } + } + private static void ValidateTypedValue(string name, string value) + { + string[] integers = ["prediction-minimum-fit-rows", "selection-max-candidates-per-interior-window", + "selection-max-fallback-attempts-per-anchor", "selection-diversity-scan-multiplier", + "selection-diversity-scan-min-candidates", "selection-diversity-scan-max-candidates"]; + string[] numbers = ["prediction-default-bit-stress-threshold", "selection-near-baseline-max-size-growth-percent", + "selection-minimum-kld-improvement-epsilon", "selection-minimum-neighbor-gap-fraction", + "selection-near-lower-anchor-brutal-zone-fraction", "selection-near-anchor-required-kld-gain-fraction"]; + bool valid = true; + if (integers.Contains(name)) + valid = int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out int n) && n >= (name == "prediction-minimum-fit-rows" ? 2 : 1); + else if (numbers.Contains(name)) + valid = double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out double n) && double.IsFinite(n) && + (name == "prediction-default-bit-stress-threshold" ? n > 0 : n >= 0); + else if (name == "manual-max-predicted-size-bytes") + valid = ulong.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out _); + else if (name is "prediction-bit-stress-threshold-candidates" or "selection-interior-window-fractions") + valid = value.Split(',').All(v => double.TryParse(v, NumberStyles.Float, CultureInfo.InvariantCulture, out double n) && + double.IsFinite(n) && n > 0 && (name != "selection-interior-window-fractions" || n <= 1)); + else if (name is "selection-diversify-validation-candidates" or "selection-diversity-low-bit-only") + valid = new[] { "true", "false", "1", "0", "yes", "no", "y", "n", "on", "off" }.Contains(value, StringComparer.OrdinalIgnoreCase); + if (!valid) throw new ArgumentException($"Invalid value '{value}' for --{name}. Check the option's type/range; decimals use a dot."); + } + +} diff --git a/src/MagicQuant/Configuration/CommandPreflight.cs b/src/MagicQuant/Configuration/CommandPreflight.cs new file mode 100644 index 0000000..9dace48 --- /dev/null +++ b/src/MagicQuant/Configuration/CommandPreflight.cs @@ -0,0 +1,77 @@ +using System.Text.Json; +using MagicQuant.Models; +using MagicQuant.Services; +using MQ.DB.Models; + +namespace MagicQuant.Configuration; + +/// Read-only checks before cleanup, dependency setup, model hashing, or database initialization. +public static class CommandPreflight +{ + public static void Validate(string command, MagicQuantYamlConfig config, IReadOnlyList args) + { + string? Get(string name) => args.FirstOrDefault(a => string.Equals(a.Name, name, StringComparison.OrdinalIgnoreCase))?.Value; + if (!new[] { "all", "none", "selected" }.Contains(config.Baselines.StandardBaselinesMode.Trim(), StringComparer.OrdinalIgnoreCase)) + throw new InvalidOperationException("baselines.standard_baselines_mode must be all, selected, or none."); + PathSafety.ValidateFolderName(config.Paths.ExternalBaselineCacheDirName, "paths.external_baseline_cache_dir_name"); + PathSafety.ValidateFolderName(config.Output.OutputNamePrefix, "output.output_name_prefix"); + var custom = new[] { config.Paths.LlamaRoot, config.Paths.LlamaBin, config.Paths.ConvertScript }; + if (custom.Any(p => !string.IsNullOrWhiteSpace(p))) + { + if (custom.Any(string.IsNullOrWhiteSpace)) + throw new InvalidOperationException("Provide all three custom llama.cpp paths: llama_root, llama_bin, convert_script."); + if (!Directory.Exists(custom[0]) || !Directory.Exists(custom[1]) || !File.Exists(custom[2])) + throw new InvalidOperationException("One or more custom llama.cpp paths do not exist."); + } + if (command.Equals("initialize-llama-cpp", StringComparison.OrdinalIgnoreCase)) return; + string model = config.Paths.ModelDir ?? ""; + if (string.IsNullOrWhiteSpace(model) || !Directory.Exists(model)) + throw new InvalidOperationException("A valid model directory is required. Set paths.model_dir or --model-dir."); + model = Path.GetFullPath(model); + string work = Path.Combine(model, "MagicQuant"); + string runtime = string.IsNullOrWhiteSpace(config.Paths.MagicQuantRoot) + ? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), MagicConstants.MagicQuantFolder) + : Path.GetFullPath(config.Paths.MagicQuantRoot); + bool validation = command.Equals("validate-predictions", StringComparison.OrdinalIgnoreCase); + bool clone = command.Equals("clone-repository-quants", StringComparison.OrdinalIgnoreCase); + if (!validation) + { + if (!Directory.EnumerateFiles(model, "*.safetensors").Any()) + throw new InvalidOperationException("The model directory has no top-level .safetensors files."); + string modelConfig = Path.Combine(model, "config.json"); + if (!File.Exists(modelConfig)) throw new InvalidOperationException("The source model is missing config.json."); + using var parsed = JsonDocument.Parse(File.ReadAllText(modelConfig)); + if (parsed.RootElement.ValueKind != JsonValueKind.Object) throw new InvalidOperationException("Model config.json must contain a JSON object."); + if (string.IsNullOrWhiteSpace(config.Identity.ArchitectureFamilyName)) + throw new InvalidOperationException("Set identity.architecture_family_name or --architecture-family explicitly."); + } + string output = validation ? OutputPathService.PredictionValidation(work, Get("output-dir"), config.Output.OutputDir) + : clone ? OutputPathService.Clone(work, Get("output-dir"), config.Output.OutputDir) + : OutputPathService.Pipeline(work, config.Output.OutputDir); + var managed = new List + { + Path.Combine(work, "GGUF"), Path.Combine(work, "Benchmarks"), Path.Combine(work, "Logs"), Path.Combine(work, "Runs"), + Path.Combine(work, config.Paths.ExternalBaselineCacheDirName), Path.Combine(work, ".MagicQuant_tmp"), + Path.Combine(runtime, MagicConstants.LlamaRepoName), Path.Combine(runtime, MagicConstants.EnvName), Path.Combine(runtime, "Runs") + }; + if (!string.IsNullOrWhiteSpace(config.Paths.LlamaRoot)) managed.Add(config.Paths.LlamaRoot); + if (!string.IsNullOrWhiteSpace(config.Paths.LlamaBin)) managed.Add(config.Paths.LlamaBin); + managed.AddRange(config.Paths.ScratchRoots.Select(s => Path.Combine(s, ".MagicQuant_tmp"))); + PathSafety.ValidateExportDirectory(output, model, runtime, managed.ToArray()); + if (clone) + { + string? repo = Get("source-repo") ?? Get("clone-repo"); + string? source = Get("source-json") ?? Get("clone-json"); + if (string.IsNullOrWhiteSpace(repo) == string.IsNullOrWhiteSpace(source)) + throw new InvalidOperationException("Clone requires exactly one of --source-repo or --source-json."); + if (source != null && !IsHttpUrl(source) && !File.Exists(source)) + throw new InvalidOperationException("The local --source-json manifest does not exist."); + } + if (validation && Get("imatrix-path") is { } matrix && !File.Exists(matrix)) + throw new InvalidOperationException("The validation --imatrix-path does not exist."); + if (config.Flags.UseImatrix && config.Imatrix.DatasetLocalFile is { Length: > 0 } dataset && !File.Exists(dataset)) + throw new InvalidOperationException("imatrix.dataset_local_file does not exist."); + } + + private static bool IsHttpUrl(string source) => Uri.TryCreate(source, UriKind.Absolute, out var uri) && uri.Scheme is "http" or "https"; +} diff --git a/src/MagicQuant/Configuration/ConfigurationShapeValidator.cs b/src/MagicQuant/Configuration/ConfigurationShapeValidator.cs new file mode 100644 index 0000000..5bb607f --- /dev/null +++ b/src/MagicQuant/Configuration/ConfigurationShapeValidator.cs @@ -0,0 +1,47 @@ +using System.Collections; +using System.Reflection; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace MagicQuant.Configuration; + +/// Rejects malformed state before normalization or global caches are changed. +public static class ConfigurationShapeValidator +{ + public static void Validate(MagicQuantYamlConfig config) => ValidateObject(config, ""); + + private static void ValidateObject(object value, string path) + { + if (value is double number && !double.IsFinite(number)) + throw new InvalidOperationException($"Configuration '{path}' must be finite."); + if (value is string || value.GetType().IsValueType) + return; + if (value is IDictionary dictionary) + { + foreach (DictionaryEntry entry in dictionary) + if (entry.Value != null) ValidateObject(entry.Value, $"{path}[{entry.Key}]"); + return; + } + if (value is IEnumerable sequence) + { + foreach (var item in sequence) + { + if (item == null) throw new InvalidOperationException($"Configuration '{path}' cannot contain null items."); + ValidateObject(item, path); + } + return; + } + var nullability = new NullabilityInfoContext(); + foreach (var property in value.GetType().GetProperties().Where(p => p.GetCustomAttribute() == null)) + { + string key = UnderscoredNamingConvention.Instance.Apply(property.Name); + string fullKey = path.Length == 0 ? key : $"{path}.{key}"; + // Model-card metadata is intentionally free-form and supports null values. + if (fullKey == "readme.frontmatter") continue; + var member = property.GetValue(value); + if (member == null && nullability.Create(property).ReadState == NullabilityState.NotNull) + throw new InvalidOperationException($"Configuration '{fullKey}' cannot be null. Omit it to use its default."); + if (member != null) ValidateObject(member, fullKey); + } + } +} diff --git a/src/MagicQuant/Configuration/MagicQuantYamlConfig.cs b/src/MagicQuant/Configuration/MagicQuantYamlConfig.cs new file mode 100644 index 0000000..a8c93fd --- /dev/null +++ b/src/MagicQuant/Configuration/MagicQuantYamlConfig.cs @@ -0,0 +1,403 @@ +using YamlDotNet.Serialization; + +namespace MagicQuant.Configuration; + +public sealed class MagicQuantYamlConfig +{ + public RuntimePathConfig Paths { get; set; } = new(); + public RuntimeFlagConfig Flags { get; set; } = new(); + public RuntimeReadmeConfig Readme { get; set; } = new(); + public RuntimeImatrixConfig Imatrix { get; set; } = new(); + public RuntimeIsolationPruningConfig IsolationPruning { get; set; } = new(); + public RuntimePredictionConfig Prediction { get; set; } = new(); + public RuntimeIdentityConfig Identity { get; set; } = new(); + public RuntimeBaselineConfig Baselines { get; set; } = new(); + public RuntimeLearningConfig Learning { get; set; } = new(); + public RuntimeOutputConfig Output { get; set; } = new(); + public RuntimeCandidateSelectionConfig CandidateSelection { get; set; } = new(); + public RuntimeAnomalyDetectionConfig AnomalyDetection { get; set; } = new(); + public RuntimeSynergyDetectionConfig SynergyDetection { get; set; } = new(); + public RuntimeHardwareConfig Hardware { get; set; } = new(); + + public static MagicQuantYamlConfig CreateDefault() => new(); +} + +public sealed class RuntimePathConfig +{ + public string? MagicQuantRoot { get; set; } + public string? ModelDir { get; set; } + public string? LlamaRoot { get; set; } + public string? LlamaBin { get; set; } + public string? ConvertScript { get; set; } + public List ScratchRoots { get; set; } = new(); + public string ExternalBaselineCacheDirName { get; set; } = "ExternalBaselines"; +} + +public sealed class RuntimeFlagConfig +{ + public bool UseImatrix { get; set; } + public bool ForceImatrixRebuild { get; set; } + public bool ForceRefreshHardwareProbe { get; set; } + public bool AllowHighPrecisionHybrids { get; set; } +} + +public sealed class RuntimeReadmeConfig +{ + public string? TitleModelNameOverride { get; set; } + + // Flexible by design: Hugging Face frontmatter can grow without requiring + // new strongly typed C# properties for every key. + public Dictionary Frontmatter { get; set; } = new(StringComparer.OrdinalIgnoreCase); +} + +public sealed class RuntimeImatrixConfig +{ + public string? ImatrixUrl { get; set; } + public string? DatasetRepo { get; set; } + public string? DatasetSplit { get; set; } + public string? DatasetConfig { get; set; } + public string? DatasetLocalFile { get; set; } +} + +public sealed class RuntimeIsolationPruningConfig +{ + public double MinimumIsolationReductionToContinueRatio { get; set; } = 0.04d; + public double MinimumIsolationReductionToSuppressBf16Ratio { get; set; } = 0.10d; + public double MaximumIsolationPplDeltaPercent { get; set; } = 5.0d; + public double MaximumIsolationKld { get; set; } = 0.1d; + public double BadTradeMaxSizeDeltaPercent { get; set; } = 4.0d; + public double BadTradeKldMultiplier { get; set; } = 2.5d; + public double BadTradePplMultiplier { get; set; } = 3.5d; + public double FloatingPointEpsilon { get; set; } = 1e-8d; + public double MinimumMeaningfulBaseOnlyReductionRatio { get; set; } = 0.01d; +} + +public sealed class RuntimePredictionConfig +{ + /// + /// Legacy emergency ceiling. Keep at 0 for the rank-safe isolation predictor. + /// + public ulong ManualMaxPredictedSizeBytes { get; set; } = 0; + + /// + /// Candidate thresholds used while fitting the low-bit interaction correction. + /// The best threshold is selected by lowest MAE against existing general-category truth. + /// + public List BitStressThresholdCandidates { get; set; } = + [ + 4.0d, + 5.0d, + 6.0d, + 7.0d, + 8.0d, + 9.0d, + 10.0d, + 11.0d, + 12.0d + ]; + + public double DefaultBitStressThreshold { get; set; } = 8.0d; + public int MinimumFitRows { get; set; } = 12; + public long MaxInMemoryCombinationLoadRows { get; set; } = 5_000_000; +} + +public sealed class RuntimeIdentityConfig +{ + public string? ArchitectureFamilyName { get; set; } + public bool AllowArchitectureFamilyAliasOverride { get; set; } +} + +public sealed class RuntimeOutputConfig +{ + public string? OutputDir { get; set; } + public string OutputNamePrefix { get; set; } = "Model"; + public bool ExportExternalLearnedBaselines { get; set; } = false; + public bool AttemptMmprojBuild { get; set; } = true; + public bool RequireMmprojForVisionModels { get; set; } = false; + public bool ReuseExistingFinalArtifacts { get; set; } = false; +} + +public sealed class RuntimeCandidateSelectionConfig +{ + /// + /// Phase 2 window. 1.0 means "up to one percent larger than the smaller/higher-damage anchor". + /// + public double NearBaselineMaxSizeGrowthPercent { get; set; } = 1.0d; + + /// + /// Phase 3 windows as fractions of each adjacent anchor-pair size span. + /// Example [0.35, 0.35] tests the first 35% and next 35% of the span. + /// + public List InteriorWindowFractions { get; set; } = + [ + 0.35d, + 0.35d + ]; + + public int MaxCandidatesPerInteriorWindow { get; set; } = 1; + public int MaxFallbackAttemptsPerAnchor { get; set; } = 5; + + /// + /// Enables the conservative SQLite/isolation-truth baseline tuning fallback. + /// This does not query DuckDB and only runs after a normal phase fails to + /// validate a candidate for its anchor/window. + /// + public bool SmartFallbackEnabled { get; set; } = true; + + /// + /// Extra build/benchmark attempts permitted after the normal prediction-guided + /// attempts fail for a strict, near-baseline, or interior window. + /// + public int SmartFallbackAttemptsPerFailure { get; set; } = 3; + + /// + /// Maximum number of higher-fidelity anchor steps the smart fallback may climb + /// for a single tensor group. Lower-fidelity swaps are still only allowed when + /// their isolated KLD is measurably better than the baseline group state. + /// + public int SmartFallbackMaxHigherFidelitySteps { get; set; } = 2; + + /// + /// Strict epsilon for "lower KLD" claims. This is intentionally tiny because + /// the validator verifies the final relationship against real benchmark truth. + /// + public double MinimumKldImprovementEpsilon { get; set; } = 1e-9d; + + /// + /// Final spacing pass: candidates closer than this fraction of the global survivor + /// size span are collapsed to a single winner. + /// + public double MinimumNeighborGapFractionOfGlobalSpan { get; set; } = 0.03d; + + /// + /// Extra-brutal near-small-anchor zone. A candidate extremely close to the smaller + /// anchor must earn a larger KLD gain to survive. + /// + public double NearLowerAnchorBrutalZoneFractionOfPairSpan { get; set; } = 0.02d; + + /// + /// Required gain fraction of the adjacent-anchor KLD gap when a candidate sits in + /// the near-small-anchor brutal zone. + /// + public double NearAnchorRequiredKldGainFractionOfPairGap { get; set; } = 0.05d; + + /// + /// When false, the prediction selector does not spend build/benchmark attempts trying + /// to replace 8-bit anchors such as Q8_0 during strict dominance or near-anchor checks. + /// Q8 remains the highest-fidelity practical anchor unless this is explicitly enabled. + /// + public bool AllowEightBitAnchorReplacements { get; set; } = false; + + /// + /// Legacy/diagnostic mode for strict Q8/anomaly discovery. When false, once a strict + /// candidate validates for an anchor, MagicQuant accepts it and stops spending more + /// build/benchmark attempts on the rest of the fetched top-N list. + /// + public bool ValidateAllAnomalyStrictCandidatesAfterSuccess { get; set; } = false; + + /// + /// When true, windows with more predicted candidates than validation attempts fetch a + /// bounded scan pool and round-robin across candidate theory families before validation. + /// + public bool DiversifyValidationCandidates { get; set; } = true; + + /// + /// Scan roughly attemptLimit * multiplier predicted rows before selecting final attempts. + /// + public int DiversityScanMultiplier { get; set; } = 25; + + /// + /// Lower bound for the prediction-only scan pool when diversity is active. + /// + public int DiversityScanMinCandidates { get; set; } = 100; + + /// + /// Upper bound for the prediction-only scan pool when diversity is active. + /// + public int DiversityScanMaxCandidates { get; set; } = 500; + + /// + /// Optional escape hatch: if true, diversify only windows whose anchor band is Q4-ish or below. + /// Defaults false because diversity is cheap and does not increase validation attempts. + /// + public bool DiversityLowBitOnly { get; set; } = false; +} + + +public sealed class RuntimeAnomalyDetectionConfig +{ + public bool Enabled { get; set; } = true; + public int MaxAnomalyRefinementRounds { get; set; } = 1; + public double MinActualGainVsTwinKld { get; set; } = 0.00025d; + public double MinPredictedSizeSavingsVsTwinPercent { get; set; } = 1.0d; + public int MaxProbeGroupCount { get; set; } = 4; + public int MaxProbesPerSeed { get; set; } = 16; + public int MaxTotalProbesPerRun { get; set; } = 32; + public double MaxPredictionSpaceGapVsTwinKld { get; set; } = 0.00050d; + public double MaxRelativePredictionPenaltyVsTwin { get; set; } = 0.35d; + public double PredictionSpaceViolationMargin { get; set; } = 0.00005d; + public double AnomalyAdjustmentShrinkFactor { get; set; } = 0.50d; + public double MinRuleConfidenceToApply { get; set; } = 0.50d; + public double MaxNegativeAdjustmentKld { get; set; } = 0.00075d; + public double MaxPositiveAdjustmentKld { get; set; } = 0.00075d; + public double MaxAdjustmentFractionOfBaseKld { get; set; } = 0.75d; + + /// + /// Advisory diagnostics cap for confirmed pairwise ordering corrections. Beneficial + /// pairwise rules are allowed to cross their own measured twin even when the required + /// adjustment exceeds this value; the cap is reported, not used to resurrect the old + /// broad-boost poison. + /// + public double MaxConfirmedPairwiseOrderingAdjustmentKld { get; set; } = 0.006d; + + public int MaxSmokeCandidatesPerReferenceZone { get; set; } = 12; + public bool PersistSuppressionResults { get; set; } = true; + public bool VerboseAnomalyLogging { get; set; } = true; + public RuntimeConfirmedAnomalyExpansionConfig ConfirmedAnomalyExpansion { get; set; } = new(); +} + +public sealed class RuntimeConfirmedAnomalyExpansionConfig +{ + public bool Enabled { get; set; } = true; + public int MaxNeighborsPerConfirmedRule { get; set; } = 6; + public int MaxTotalExpansionProbes { get; set; } = 12; + public List AllowedReferenceQuants { get; set; } = ["Q8_0"]; + // External display names are campaign-specific; opt in after configuring that source. + public List AllowedCandidateQuants { get; set; } = ["Q6_K", "Q5_K"]; +} + + +public sealed class RuntimeSynergyDetectionConfig +{ + public bool Enabled { get; set; } = true; + public int MaxRefinementRounds { get; set; } = 1; + public double ExactContextConfidenceMultiplier { get; set; } = 1.00d; + public double SameSelectedGroupsConfidenceMultiplier { get; set; } = 0.55d; + public double EquivalentQuantFamilyConfidenceMultiplier { get; set; } = 0.30d; + public double GroupFamilySuspicionConfidenceMultiplier { get; set; } = 0.15d; + public double MinConfidenceToApplyAdjustment { get; set; } = 0.35d; + public double MinConfidenceToScheduleTransferProbe { get; set; } = 0.25d; + public double MaxNegativeAdjustmentKld { get; set; } = 0.002d; + public double MaxNegativeAdjustmentFractionOfBaseKld { get; set; } = 0.75d; + public bool TransferProbeEnabled { get; set; } = true; + public int MaxTransferProbesPerTemplate { get; set; } = 6; + public int MaxTotalTransferProbesPerRun { get; set; } = 24; + public RuntimeSynergyTransferProbeContextStrataConfig TransferProbeContextStrata { get; set; } = new(); + public bool ExploratoryContextPairEnabled { get; set; } = true; + public int MaxExploratoryContextPairsPerRun { get; set; } = 14; + public List ExploratoryPairBitRanges { get; set; } = [4]; + public List ExploratoryPairContextStrata { get; set; } = ["mid-fidelity", "low-fidelity"]; + public bool ContextScopedRuleApplicationEnabled { get; set; } = true; + public int MaxNonRuleGroupContextMismatches { get; set; } = 1; + public bool VerboseSynergyLogging { get; set; } = true; + public double MinSmokeScore { get; set; } = 0.55d; + public double MaxSmokeGapKld { get; set; } = 0.004d; + public int TopRejectedSmokePreview { get; set; } = 25; + public bool CompositionProbeEnabled { get; set; } = true; + public int MaxTemplateCompositionGroupCount { get; set; } = 4; + public int MaxCompositionProbesPerRun { get; set; } = 8; + public int MaxTemplatesToCompose { get; set; } = 4; + public double MinTemplateConfidenceForComposition { get; set; } = 0.50d; + public double MinCombinedExpectedSizeSavingsPercent { get; set; } = 1.0d; + public bool ContaminatingPassengerDetectionEnabled { get; set; } = true; + public double MinFailureMarginForContaminationKld { get; set; } = 0.00050d; + public double ContaminationPenaltyConfidenceMultiplier { get; set; } = 0.45d; + public bool SuppressRepeatedContaminatedAttempts { get; set; } = true; +} + +public sealed class RuntimeSynergyTransferProbeContextStrataConfig +{ + public List HighFidelityReferenceQuants { get; set; } = ["Q6_K", "Q5_K"]; + public List MidFidelityReferenceQuants { get; set; } = ["Q4_K_M"]; + public List LowFidelityReferenceQuants { get; set; } = ["IQ3_S"]; + public bool LowFidelityEnabled { get; set; } = false; +} + +public sealed class RuntimeLearningConfig +{ + public bool ForceRelearnArchitectureFamily { get; set; } + public List ForceRelearnStandardBaselines { get; set; } = new(); + + /// + /// Safety gate for regex/profile mistakes. When true, the pipeline run prints + /// native BF16 tensor-group counts and asks before continuing. + /// + public bool ConfirmTensorGroupProfile { get; set; } = true; + + /// + /// Transient repair command for regex changes. Rebuilds learned tensor/group + /// rows for the active profile from existing DB truth where possible, avoiding + /// needless re-download/re-quantization of pure baseline learning artifacts. + /// + public bool RebucketLearnedTensorGroupsFromExistingTruth { get; set; } = true; +} + +public sealed class RuntimeBaselineConfig +{ + public string StandardBaselinesMode { get; set; } = "all"; + public List EnabledStandardLearningBaselines { get; set; } = new(); + public List EnabledStandardCombinationCarriers { get; set; } = new(); + public List EnabledStandardExplicitGroupCandidates { get; set; } = new(); + public List CustomRepositories { get; set; } = new(); + + [YamlIgnore] + public List ResolvedCustomBaselines { get; set; } = new(); +} + +public sealed class CustomBaselineRepositoryConfig +{ + public string RepoId { get; set; } = string.Empty; + public string? Revision { get; set; } + public string? ShortSourceName { get; set; } + public string SourceKind { get; set; } = "huggingface_gguf_repository"; + public bool Enabled { get; set; } = true; + public bool RequireAllIncludesToResolve { get; set; } = true; + public bool ValidateTensorNamesAgainstSourceModel { get; set; } = true; + public bool DeletePartialOrDirtyDownloads { get; set; } = true; + public bool ResumeOrRetryDownloads { get; set; } = true; + public bool AllowAsCombinationCarrier { get; set; } + public bool AllowAsExplicitGroupCandidate { get; set; } = true; + public bool AllowAsLearningBaseline { get; set; } = true; + public List Includes { get; set; } = new(); +} + +public sealed class CustomBaselineIncludeConfig +{ + public string BaselineFamily { get; set; } = string.Empty; + public string? FileName { get; set; } + public string? DisplayName { get; set; } + public string? QuantizeBaseName { get; set; } + public bool? RequiresImatrix { get; set; } + public bool? AllowAsCombinationCarrier { get; set; } + public bool? AllowAsExplicitGroupCandidate { get; set; } + public bool? AllowAsLearningBaseline { get; set; } + public bool ForceRelearn { get; set; } + public List BannedGroupIds { get; set; } = new(); +} + +public sealed class ResolvedCustomBaselineSpec +{ + public byte DynamicBaselineId { get; set; } + public string CanonicalKey { get; set; } = string.Empty; + public string DisplayName { get; set; } = string.Empty; + public string RepoId { get; set; } = string.Empty; + public string? Revision { get; set; } + public string SourceOwner { get; set; } = string.Empty; + public string SourceFileName { get; set; } = string.Empty; + public string ShortSourceName { get; set; } = string.Empty; + public string BaselineFamily { get; set; } = string.Empty; + public string QuantizeBaseName { get; set; } = string.Empty; + public bool RequiresImatrix { get; set; } + public bool AllowAsLearningBaseline { get; set; } + public bool AllowAsCombinationCarrier { get; set; } + public bool AllowAsExplicitGroupCandidate { get; set; } + public bool ForceRelearn { get; set; } + public int? BaselineQuantDefinitionId { get; set; } + public bool IsActiveInCurrentConfig { get; set; } = true; + public IReadOnlyList BannedGroupIds { get; set; } = Array.Empty(); +} + +public sealed class RuntimeHardwareConfig +{ + public Dictionary GpuMemoryLimitsGb { get; set; } = new(); +} diff --git a/src/MagicQuant/Configuration/MagicQuantYamlLoader.cs b/src/MagicQuant/Configuration/MagicQuantYamlLoader.cs new file mode 100644 index 0000000..83e96c9 --- /dev/null +++ b/src/MagicQuant/Configuration/MagicQuantYamlLoader.cs @@ -0,0 +1,486 @@ +using System.Globalization; +using MagicQuant.Helpers; +using MagicQuant.Models; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace MagicQuant.Configuration; + +/// +/// Loads one YAML document, applies supported CLI overrides, and initializes runtime state. +/// Custom files inherit typed defaults, not values from the distributed YAML profile. +/// +public static class MagicQuantYamlLoader +{ + public sealed record LoadedConfiguration(string Path, MagicQuantYamlConfig Settings, IReadOnlyList Warnings); + + public static MagicQuantYamlConfig LoadAndApply(string commandName, IReadOnlyList args) + { + var loaded = Read(args); + Apply(loaded); + return loaded.Settings; + } + + /// Reads and validates configuration without changing globals or creating directories. + public static LoadedConfiguration Read(IReadOnlyList args) + { + string configPath = ResolveConfigPath(args); + if (!File.Exists(configPath)) + throw new FileNotFoundException($"MagicQuant config file was not found at '{configPath}'. Pass --config or copy config.default.yaml next to the executable."); + string yaml = File.ReadAllText(configPath); + var warnings = YamlConfigurationDiagnostics.Inspect(yaml); + if (warnings.Count > 0 && args.Any(a => string.Equals(a.Name, "strict-config", StringComparison.OrdinalIgnoreCase))) + throw new InvalidOperationException(string.Join(Environment.NewLine, warnings)); + var deserializer = new DeserializerBuilder() + .IgnoreUnmatchedProperties() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .Build(); + var config = deserializer.Deserialize(yaml) ?? MagicQuantYamlConfig.CreateDefault(); + ConfigurationShapeValidator.Validate(config); + ApplyCliOverrides(config, args); + ConfigurationShapeValidator.Validate(config); + return new LoadedConfiguration(configPath, config, warnings); + } + + public static void Apply(LoadedConfiguration loaded) + { + Cache.ActiveConfigPath = loaded.Path; + NormalizeAndApply(loaded.Settings); + Config.Load(loaded.Settings); + AnsiConsole.MarkupLine($"[grey]Using config:[/] {Markup.Escape(loaded.Path)}"); + foreach (string warning in loaded.Warnings) + AnsiConsole.MarkupLine($"[yellow]{Markup.Escape(warning)}[/]"); + } + + public static string ResolveConfigPath(IReadOnlyList args) + { + string? explicitPath = args.FirstOrDefault(a => string.Equals(a.Name, "config", StringComparison.OrdinalIgnoreCase))?.Value; + if (!string.IsNullOrWhiteSpace(explicitPath)) + return Path.GetFullPath(explicitPath); + + return Path.Combine(AppContext.BaseDirectory, "config.default.yaml"); + } + + private static void NormalizeAndApply(MagicQuantYamlConfig config) + { + config.Paths.MagicQuantRoot = ResolveMagicQuantRoot(config.Paths.MagicQuantRoot); + Cache.MagicQuantDirectory = config.Paths.MagicQuantRoot; + Cache.LlamaRoot = NormalizeNullOrFullPath(config.Paths.LlamaRoot); + Cache.LlamaBin = NormalizeNullOrFullPath(config.Paths.LlamaBin); + Cache.ConvertScript = NormalizeNullOrFullPath(config.Paths.ConvertScript); + Cache.ScratchRoots = NormalizeScratchRoots(config.Paths.ScratchRoots); + + Directory.CreateDirectory(Cache.MagicQuantDirectory!); + + config.Learning ??= new RuntimeLearningConfig(); + + Cache.UseImatrix = config.Flags.UseImatrix; + Cache.ForceImatrixRebuild = config.Flags.ForceImatrixRebuild; + Cache.ForceRefreshHardwareProbe = config.Flags.ForceRefreshHardwareProbe; + Cache.ConfirmTensorGroupProfile = config.Learning.ConfirmTensorGroupProfile; + Cache.RebucketLearnedTensorGroupsFromExistingTruth = config.Learning.RebucketLearnedTensorGroupsFromExistingTruth; + + config.Hardware.GpuMemoryLimitsGb ??= new Dictionary(); + config.Hardware.GpuMemoryLimitsGb = config.Hardware.GpuMemoryLimitsGb + .Where(x => x.Key >= 0 && x.Value > 0d) + .ToDictionary(x => x.Key, x => x.Value); + + Cache.GpuMemoryLimitsGb = config.Hardware.GpuMemoryLimitsGb + .Where(x => x.Key >= 0 && x.Value > 0d) + .ToDictionary(x => x.Key, x => x.Value); + + RuntimeSearchSpace.AllowHighPrecisionHybrids = config.Flags.AllowHighPrecisionHybrids; + Cache.CurrentArchitectureFamilyName = + config.Identity.ArchitectureFamilyName?.Trim() ?? string.Empty; + + Cache.AllowArchitectureFamilyAliasOverride = + config.Identity.AllowArchitectureFamilyAliasOverride; + + Cache.CurrentArchitectureFamilyId = null; + Cache.CurrentTensorGroupProfileId = null; + Cache.CurrentTensorGroupProfileFingerprintHash = null; + + config.Output.OutputDir = string.IsNullOrWhiteSpace(config.Output.OutputDir) + ? null + : config.Output.OutputDir.Trim(); + + config.Output.OutputNamePrefix = string.IsNullOrWhiteSpace(config.Output.OutputNamePrefix) + ? "Model" + : config.Output.OutputNamePrefix.Trim(); + + config.Readme ??= new RuntimeReadmeConfig(); + + config.Readme.TitleModelNameOverride = string.IsNullOrWhiteSpace(config.Readme.TitleModelNameOverride) + ? null + : config.Readme.TitleModelNameOverride.Trim(); + + config.Readme.Frontmatter ??= new Dictionary(StringComparer.OrdinalIgnoreCase); + config.Readme.Frontmatter = config.Readme.Frontmatter + .Where(x => !string.IsNullOrWhiteSpace(x.Key) && !IsEmptyFrontmatterValue(x.Value)) + .ToDictionary(x => x.Key.Trim(), x => x.Value, StringComparer.OrdinalIgnoreCase); + + if (config.Prediction.BitStressThresholdCandidates.Count == 0) + config.Prediction.BitStressThresholdCandidates.Add(config.Prediction.DefaultBitStressThreshold); + + config.Prediction.BitStressThresholdCandidates = config.Prediction.BitStressThresholdCandidates + .Where(x => x > 0d) + .Distinct() + .OrderBy(x => x) + .ToList(); + + if (config.Prediction.MinimumFitRows < 2) + config.Prediction.MinimumFitRows = 2; + + if (config.CandidateSelection.InteriorWindowFractions.Count == 0) + { + config.CandidateSelection.InteriorWindowFractions.Add(0.35d); + config.CandidateSelection.InteriorWindowFractions.Add(0.35d); + } + + config.CandidateSelection.InteriorWindowFractions = config.CandidateSelection.InteriorWindowFractions + .Select(x => Math.Clamp(x, 0d, 1d)) + .Where(x => x > 0d) + .ToList(); + + config.CandidateSelection.MaxCandidatesPerInteriorWindow = Math.Max(1, config.CandidateSelection.MaxCandidatesPerInteriorWindow); + config.CandidateSelection.MaxFallbackAttemptsPerAnchor = Math.Max(1, config.CandidateSelection.MaxFallbackAttemptsPerAnchor); + config.CandidateSelection.NearBaselineMaxSizeGrowthPercent = Math.Max(0d, config.CandidateSelection.NearBaselineMaxSizeGrowthPercent); + config.CandidateSelection.MinimumKldImprovementEpsilon = Math.Max(0d, config.CandidateSelection.MinimumKldImprovementEpsilon); + config.CandidateSelection.DiversityScanMultiplier = Math.Max(1, config.CandidateSelection.DiversityScanMultiplier); + config.CandidateSelection.DiversityScanMinCandidates = Math.Max(1, config.CandidateSelection.DiversityScanMinCandidates); + config.CandidateSelection.DiversityScanMaxCandidates = Math.Max(config.CandidateSelection.DiversityScanMinCandidates, config.CandidateSelection.DiversityScanMaxCandidates); + + config.AnomalyDetection ??= new RuntimeAnomalyDetectionConfig(); + config.SynergyDetection ??= new RuntimeSynergyDetectionConfig(); + NormalizeSynergyDetection(config); + config.AnomalyDetection.MaxAnomalyRefinementRounds = Math.Clamp(config.AnomalyDetection.MaxAnomalyRefinementRounds, 0, 1); + config.AnomalyDetection.MinActualGainVsTwinKld = Math.Max(0d, config.AnomalyDetection.MinActualGainVsTwinKld); + config.AnomalyDetection.MinPredictedSizeSavingsVsTwinPercent = Math.Max(0d, config.AnomalyDetection.MinPredictedSizeSavingsVsTwinPercent); + config.AnomalyDetection.MaxProbeGroupCount = Math.Clamp(config.AnomalyDetection.MaxProbeGroupCount, 1, 9); + config.AnomalyDetection.MaxProbesPerSeed = Math.Max(1, config.AnomalyDetection.MaxProbesPerSeed); + config.AnomalyDetection.MaxTotalProbesPerRun = Math.Max(0, config.AnomalyDetection.MaxTotalProbesPerRun); + config.AnomalyDetection.MaxPredictionSpaceGapVsTwinKld = Math.Max(0d, config.AnomalyDetection.MaxPredictionSpaceGapVsTwinKld); + config.AnomalyDetection.MaxRelativePredictionPenaltyVsTwin = Math.Clamp(config.AnomalyDetection.MaxRelativePredictionPenaltyVsTwin, 0d, 1d); + config.AnomalyDetection.PredictionSpaceViolationMargin = Math.Max(0d, config.AnomalyDetection.PredictionSpaceViolationMargin); + config.AnomalyDetection.AnomalyAdjustmentShrinkFactor = Math.Clamp(config.AnomalyDetection.AnomalyAdjustmentShrinkFactor, 0d, 1d); + config.AnomalyDetection.MinRuleConfidenceToApply = Math.Clamp(config.AnomalyDetection.MinRuleConfidenceToApply, 0d, 1d); + config.AnomalyDetection.MaxNegativeAdjustmentKld = Math.Max(0d, config.AnomalyDetection.MaxNegativeAdjustmentKld); + config.AnomalyDetection.MaxPositiveAdjustmentKld = Math.Max(0d, config.AnomalyDetection.MaxPositiveAdjustmentKld); + config.AnomalyDetection.MaxAdjustmentFractionOfBaseKld = Math.Clamp(config.AnomalyDetection.MaxAdjustmentFractionOfBaseKld, 0d, 1d); + config.AnomalyDetection.MaxConfirmedPairwiseOrderingAdjustmentKld = Math.Max(0d, config.AnomalyDetection.MaxConfirmedPairwiseOrderingAdjustmentKld); + config.AnomalyDetection.MaxSmokeCandidatesPerReferenceZone = Math.Max(1, config.AnomalyDetection.MaxSmokeCandidatesPerReferenceZone); + + foreach (var repository in config.Baselines.CustomRepositories) + { + repository.RepoId = repository.RepoId.Trim(); + repository.Revision = string.IsNullOrWhiteSpace(repository.Revision) + ? null + : repository.Revision.Trim(); + } + + ApplyStandardBaselineFilters(config.Baselines); + BaselineQuants.ResetDynamicCustomBaselines(); + } + + + private static void NormalizeSynergyDetection(MagicQuantYamlConfig config) + { + var s = config.SynergyDetection; + s.MaxRefinementRounds = Math.Clamp(s.MaxRefinementRounds, 0, 1); + s.ExactContextConfidenceMultiplier = Math.Clamp(s.ExactContextConfidenceMultiplier, 0d, 1d); + s.SameSelectedGroupsConfidenceMultiplier = Math.Clamp(s.SameSelectedGroupsConfidenceMultiplier, 0d, 1d); + s.EquivalentQuantFamilyConfidenceMultiplier = Math.Clamp(s.EquivalentQuantFamilyConfidenceMultiplier, 0d, 1d); + s.GroupFamilySuspicionConfidenceMultiplier = Math.Clamp(s.GroupFamilySuspicionConfidenceMultiplier, 0d, 1d); + s.MinConfidenceToApplyAdjustment = Math.Clamp(s.MinConfidenceToApplyAdjustment, 0d, 1d); + s.MinConfidenceToScheduleTransferProbe = Math.Clamp(s.MinConfidenceToScheduleTransferProbe, 0d, 1d); + s.MaxNegativeAdjustmentKld = Math.Max(0d, s.MaxNegativeAdjustmentKld); + s.MaxNegativeAdjustmentFractionOfBaseKld = Math.Clamp(s.MaxNegativeAdjustmentFractionOfBaseKld, 0d, 1d); + s.MaxTransferProbesPerTemplate = Math.Max(0, s.MaxTransferProbesPerTemplate); + s.MaxTotalTransferProbesPerRun = Math.Max(0, s.MaxTotalTransferProbesPerRun); + s.TransferProbeContextStrata ??= new RuntimeSynergyTransferProbeContextStrataConfig(); + s.TransferProbeContextStrata.HighFidelityReferenceQuants ??= new List(); + s.TransferProbeContextStrata.MidFidelityReferenceQuants ??= new List(); + s.TransferProbeContextStrata.LowFidelityReferenceQuants ??= new List(); + s.MaxExploratoryContextPairsPerRun = Math.Max(0, s.MaxExploratoryContextPairsPerRun); + s.ExploratoryPairBitRanges ??= new List(); + s.ExploratoryPairBitRanges = s.ExploratoryPairBitRanges.Where(x => x is >= 1 and <= 16).Distinct().OrderBy(x => x).ToList(); + s.ExploratoryPairContextStrata ??= new List(); + s.ExploratoryPairContextStrata = s.ExploratoryPairContextStrata + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Select(x => x.Trim().ToLowerInvariant()) + .Distinct(StringComparer.Ordinal) + .ToList(); + s.MaxNonRuleGroupContextMismatches = Math.Clamp(s.MaxNonRuleGroupContextMismatches, 0, 9); + s.MinSmokeScore = Math.Clamp(s.MinSmokeScore, 0d, 1d); + s.MaxSmokeGapKld = Math.Max(0d, s.MaxSmokeGapKld); + s.TopRejectedSmokePreview = Math.Max(1, s.TopRejectedSmokePreview); + s.MaxTemplateCompositionGroupCount = Math.Clamp(s.MaxTemplateCompositionGroupCount, 1, 9); + s.MaxCompositionProbesPerRun = Math.Max(0, s.MaxCompositionProbesPerRun); + s.MaxTemplatesToCompose = Math.Max(0, s.MaxTemplatesToCompose); + s.MinTemplateConfidenceForComposition = Math.Clamp(s.MinTemplateConfidenceForComposition, 0d, 1d); + s.MinCombinedExpectedSizeSavingsPercent = Math.Max(0d, s.MinCombinedExpectedSizeSavingsPercent); + s.MinFailureMarginForContaminationKld = Math.Max(0d, s.MinFailureMarginForContaminationKld); + s.ContaminationPenaltyConfidenceMultiplier = Math.Clamp(s.ContaminationPenaltyConfidenceMultiplier, 0d, 1d); + } + + private static void ApplyStandardBaselineFilters(RuntimeBaselineConfig baselineConfig) + { + string mode = (baselineConfig.StandardBaselinesMode ?? "all").Trim().ToLowerInvariant(); + + HashSet? learning = null; + HashSet? carriers = null; + HashSet? explicitCandidates = null; + + if (mode == "none") + { + learning = new HashSet(); + carriers = new HashSet(); + explicitCandidates = new HashSet(); + } + else if (mode == "selected") + { + learning = ResolveStandardBaselineIds(baselineConfig.EnabledStandardLearningBaselines); + carriers = ResolveStandardBaselineIds(baselineConfig.EnabledStandardCombinationCarriers); + explicitCandidates = ResolveStandardBaselineIds(baselineConfig.EnabledStandardExplicitGroupCandidates); + } + + BaselineQuants.ConfigureStandardRoleFilters(learning, carriers, explicitCandidates); + } + + private static HashSet ResolveStandardBaselineIds(IEnumerable names) + { + var result = new HashSet(); + + foreach (var raw in names ?? Array.Empty()) + { + if (string.IsNullOrWhiteSpace(raw)) + continue; + + var baseline = BaselineQuants.ResolveBuiltInStandardRoleBaseline(raw.Trim()); + if (baseline == null) + { + throw new InvalidOperationException( + $"Unknown built-in baseline '{raw}'. " + + $"Known values: {string.Join(", ", BaselineQuants.GetBuiltInStandardBaselines().Select(x => x.Names[0]))}"); + } + + result.Add(baseline.UniqueId); + } + + return result; + } + + + private static void ApplyCliOverrides(MagicQuantYamlConfig config, IReadOnlyList args) + { + config.Learning ??= new RuntimeLearningConfig(); + + string? Get(string name) => args.FirstOrDefault(a => string.Equals(a.Name, name, StringComparison.OrdinalIgnoreCase))?.Value; + bool Has(string name) => args.Any(a => string.Equals(a.Name, name, StringComparison.OrdinalIgnoreCase)); + + config.Paths.MagicQuantRoot = Prefer(Get("magic-quant-root"), config.Paths.MagicQuantRoot); + config.Paths.ModelDir = Prefer(Get("model-dir"), config.Paths.ModelDir); + config.Paths.LlamaRoot = Prefer(Get("llama-root"), config.Paths.LlamaRoot); + config.Paths.LlamaBin = Prefer(Get("llama-bin"), config.Paths.LlamaBin); + config.Paths.ConvertScript = Prefer(Get("convert-script"), config.Paths.ConvertScript); + + if (Has("use-imatrix")) config.Flags.UseImatrix = true; + if (Has("imatrix-force-rebuild")) config.Flags.ForceImatrixRebuild = true; + if (Has("relearn-baseline-mappings") || Has("force-relearn-baseline-tensor-mappings")) + throw new InvalidOperationException("--relearn-baseline-mappings was removed because it globally wiped learned tensor truth. Use YAML learning.force_relearn_architecture_family, learning.force_relearn_standard_baselines, or custom_repositories/includes/force_relearn instead."); + if (Has("recheck-hardware-probe") || Has("force-refresh-hardware-probe") || Has("force_refresh_hardware_probe")) config.Flags.ForceRefreshHardwareProbe = true; + if (Has("allow-high-precision-hybrids")) config.Flags.AllowHighPrecisionHybrids = true; + if (Has("rebucket-learned-tensor-groups") || Has("rebucket-tensor-groups-from-db") || Has("relearn-tensor-groups-from-db")) + { + // Kept as a harmless compatibility alias. Rebucket is now enabled by default + // because it is the safe/idempotent path after regex profile changes. + config.Learning.RebucketLearnedTensorGroupsFromExistingTruth = true; + } + if (Has("no-rebucket-learned-tensor-groups") || Has("disable-tensor-group-rebucket") || Has("full-relearn-tensor-groups")) + config.Learning.RebucketLearnedTensorGroupsFromExistingTruth = false; + if (Has("skip-tensor-group-confirm") || Has("yes-tensor-groups")) + config.Learning.ConfirmTensorGroupProfile = false; + + config.Imatrix.ImatrixUrl = Prefer(Get("imatrix-url"), config.Imatrix.ImatrixUrl); + config.Imatrix.DatasetRepo = Prefer(Get("imatrix-dataset-repo"), config.Imatrix.DatasetRepo); + config.Imatrix.DatasetSplit = Prefer(Get("imatrix-dataset-split"), config.Imatrix.DatasetSplit); + config.Imatrix.DatasetConfig = Prefer(Get("imatrix-dataset-config"), config.Imatrix.DatasetConfig); + config.Imatrix.DatasetLocalFile = Prefer(Get("imatrix-dataset-local-file"), config.Imatrix.DatasetLocalFile); + + if (ulong.TryParse(Get("manual-max-predicted-size-bytes"), out var manualBytes)) + config.Prediction.ManualMaxPredictedSizeBytes = manualBytes; + + if (TryParseFiniteDouble(Get("prediction-default-bit-stress-threshold"), out var defaultBitStress) && defaultBitStress > 0d) + config.Prediction.DefaultBitStressThreshold = defaultBitStress; + + if (int.TryParse(Get("prediction-minimum-fit-rows"), out var minFitRows) && minFitRows >= 2) + config.Prediction.MinimumFitRows = minFitRows; + + var bitStressCandidates = ParseDoubleList(Get("prediction-bit-stress-threshold-candidates")); + if (bitStressCandidates.Count > 0) + config.Prediction.BitStressThresholdCandidates = bitStressCandidates; + + if (TryParseFiniteDouble(Get("selection-near-baseline-max-size-growth-percent"), out var nearPct) && nearPct >= 0d) + config.CandidateSelection.NearBaselineMaxSizeGrowthPercent = nearPct; + + var windows = ParseDoubleList(Get("selection-interior-window-fractions")); + if (windows.Count > 0) + config.CandidateSelection.InteriorWindowFractions = windows; + + if (int.TryParse(Get("selection-max-candidates-per-interior-window"), out var maxInterior) && maxInterior > 0) + config.CandidateSelection.MaxCandidatesPerInteriorWindow = maxInterior; + + if (int.TryParse(Get("selection-max-fallback-attempts-per-anchor"), out var maxFallbacks) && maxFallbacks > 0) + config.CandidateSelection.MaxFallbackAttemptsPerAnchor = maxFallbacks; + + if (TryParseFiniteDouble(Get("selection-minimum-kld-improvement-epsilon"), out var minKldEpsilon) && minKldEpsilon >= 0d) + config.CandidateSelection.MinimumKldImprovementEpsilon = minKldEpsilon; + + if (TryParseFiniteDouble(Get("selection-minimum-neighbor-gap-fraction"), out var neighborGap) && neighborGap >= 0d) + config.CandidateSelection.MinimumNeighborGapFractionOfGlobalSpan = neighborGap; + + if (TryParseFiniteDouble(Get("selection-near-lower-anchor-brutal-zone-fraction"), out var brutalZone) && brutalZone >= 0d) + config.CandidateSelection.NearLowerAnchorBrutalZoneFractionOfPairSpan = brutalZone; + + if (TryParseFiniteDouble(Get("selection-near-anchor-required-kld-gain-fraction"), out var brutalGain) && brutalGain >= 0d) + config.CandidateSelection.NearAnchorRequiredKldGainFractionOfPairGap = brutalGain; + + if (Has("allow-eight-bit-anchor-replacements")) + config.CandidateSelection.AllowEightBitAnchorReplacements = true; + + if (Has("validate-all-anomaly-strict-candidates-after-success")) + config.CandidateSelection.ValidateAllAnomalyStrictCandidatesAfterSuccess = true; + + if (TryParseBool(Get("selection-diversify-validation-candidates"), out var diversifyValidationCandidates)) + config.CandidateSelection.DiversifyValidationCandidates = diversifyValidationCandidates; + + if (int.TryParse(Get("selection-diversity-scan-multiplier"), out var diversityScanMultiplier) && diversityScanMultiplier > 0) + config.CandidateSelection.DiversityScanMultiplier = diversityScanMultiplier; + + if (int.TryParse(Get("selection-diversity-scan-min-candidates"), out var diversityScanMinCandidates) && diversityScanMinCandidates > 0) + config.CandidateSelection.DiversityScanMinCandidates = diversityScanMinCandidates; + + if (int.TryParse(Get("selection-diversity-scan-max-candidates"), out var diversityScanMaxCandidates) && diversityScanMaxCandidates > 0) + config.CandidateSelection.DiversityScanMaxCandidates = diversityScanMaxCandidates; + + if (TryParseBool(Get("selection-diversity-low-bit-only"), out var diversityLowBitOnly)) + config.CandidateSelection.DiversityLowBitOnly = diversityLowBitOnly; + + config.Output.OutputDir = Prefer(Get("output-dir"), config.Output.OutputDir); + config.Output.OutputNamePrefix = Prefer(Get("output-name-prefix"), config.Output.OutputNamePrefix) ?? "Model"; + if (Has("export-external-learned-baselines")) config.Output.ExportExternalLearnedBaselines = true; + if (Has("reuse-existing-final-artifacts")) config.Output.ReuseExistingFinalArtifacts = true; + + config.Identity.ArchitectureFamilyName = Prefer(Get("architecture-family"), config.Identity.ArchitectureFamilyName); + if (Has("allow-architecture-family-alias-override")) config.Identity.AllowArchitectureFamilyAliasOverride = true; + } + + private static bool TryParseFiniteDouble(string? value, out double result) + { + result = 0; + if (value == null) return false; + if (!double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out result) || !double.IsFinite(result)) + throw new ArgumentException($"Invalid numeric option value '{value}'. Use a finite number with a decimal point."); + return true; + } + + private static bool TryParseBool(string? value, out bool result) + { + result = false; + if (string.IsNullOrWhiteSpace(value)) + return false; + + var normalized = value.Trim(); + if (bool.TryParse(normalized, out result)) + return true; + + if (string.Equals(normalized, "1", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, "yes", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, "y", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, "on", StringComparison.OrdinalIgnoreCase)) + { + result = true; + return true; + } + + if (string.Equals(normalized, "0", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, "no", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, "n", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, "off", StringComparison.OrdinalIgnoreCase)) + { + result = false; + return true; + } + + return false; + } + + private static List ParseDoubleList(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + return new List(); + + return value + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(x => TryParseFiniteDouble(x, out var parsed) ? (double?)parsed : null) + .Where(x => x.HasValue) + .Select(x => x!.Value) + .ToList(); + } + + private static string? Prefer(string? preferred, string? fallback) + => string.IsNullOrWhiteSpace(preferred) ? fallback : preferred; + + private static bool IsEmptyFrontmatterValue(object? value) + { + if (value == null) + return true; + + if (value is string text) + return string.IsNullOrWhiteSpace(text); + + if (value is System.Collections.IEnumerable sequence && value is not string) + { + foreach (var item in sequence) + { + if (!IsEmptyFrontmatterValue(item)) + return false; + } + + return true; + } + + return false; + } + + private static string ResolveMagicQuantRoot(string? configured) + { + if (!string.IsNullOrWhiteSpace(configured)) + return Path.GetFullPath(configured); + + return Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), MagicConstants.MagicQuantFolder); + } + + + private static List NormalizeScratchRoots(IEnumerable? roots) + { + if (roots == null) + return new List(); + + return roots + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Select(x => Path.GetFullPath(x.Trim())) + .Distinct(OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal) + .ToList(); + } + private static string? NormalizeNullOrFullPath(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + return null; + + return Path.GetFullPath(value); + } +} diff --git a/src/MagicQuant/Configuration/YamlConfigurationDiagnostics.cs b/src/MagicQuant/Configuration/YamlConfigurationDiagnostics.cs new file mode 100644 index 0000000..edd7111 --- /dev/null +++ b/src/MagicQuant/Configuration/YamlConfigurationDiagnostics.cs @@ -0,0 +1,51 @@ +using System.Reflection; +using YamlDotNet.RepresentationModel; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace MagicQuant.Configuration; + +/// Checks document keys against the typed schema while leaving free-form metadata alone. +public static class YamlConfigurationDiagnostics +{ + public static IReadOnlyList Inspect(string yaml) + { + var stream = new YamlStream(); + stream.Load(new StringReader(yaml)); + if (stream.Documents.Count > 1) + throw new InvalidOperationException("Expected one YAML configuration document."); + var warnings = new List(); + if (stream.Documents.Count == 1) + InspectNode(stream.Documents[0].RootNode, typeof(MagicQuantYamlConfig), "", warnings); + return warnings; + } + + private static void InspectNode(YamlNode node, Type type, string path, List warnings) + { + // Dictionary keys (frontmatter, GPU indices) belong to the user, not the C# schema. + if (type == typeof(object) || type.GetInterfaces().Any(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IDictionary<,>))) + return; + if (node is YamlSequenceNode sequence && type.IsGenericType) + { + for (int i = 0; i < sequence.Children.Count; i++) + InspectNode(sequence.Children[i], type.GetGenericArguments()[0], $"{path}[{i}]", warnings); + return; + } + if (node is not YamlMappingNode mapping) + return; + var properties = type.GetProperties() + .Where(p => p.GetCustomAttribute() == null) + .ToDictionary(p => UnderscoredNamingConvention.Instance.Apply(p.Name), StringComparer.Ordinal); + foreach (var entry in mapping.Children) + { + string key = ((YamlScalarNode)entry.Key).Value ?? ""; + string fullKey = path.Length == 0 ? key : $"{path}.{key}"; + if (fullKey == "flags.force_relearn_baseline_tensor_mappings") + throw new InvalidOperationException("Removed destructive option flags.force_relearn_baseline_tensor_mappings. Use targeted learning options instead."); + if (!properties.TryGetValue(key, out var property)) + warnings.Add($"Unknown or inactive YAML setting '{fullKey}' (line {entry.Key.Start.Line}). It will be ignored."); + else + InspectNode(entry.Value, property.PropertyType, fullKey, warnings); + } + } +} diff --git a/src/MagicQuant/Helpers/CliHelpers.cs b/src/MagicQuant/Helpers/CliHelpers.cs new file mode 100644 index 0000000..68fa3d7 --- /dev/null +++ b/src/MagicQuant/Helpers/CliHelpers.cs @@ -0,0 +1,164 @@ +using System.Collections.Immutable; +using System.Diagnostics; +using System.Numerics; +using System.Text.RegularExpressions; +using MagicQuant.Commands; +using MagicQuant.Models; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Helpers; + +public static class CliHelpers +{ + public static void ValidateCombinationLogicWorks(bool realResults = false) + { + PrintTotalCombinationCount(); + + var expectedTotal = ComboCounter.CountAll(); + + AnsiConsole.MarkupLine( + realResults + ? $"[bold cyan]Total real combinations after model detection:[/] [bold yellow]{expectedTotal:N0}[/]" + : $"[bold cyan]Expected total combinations:[/] [bold yellow]{expectedTotal:N0}[/]"); + + var sw = Stopwatch.StartNew(); + long actualTotal = 0; + + foreach (var baseline in RuntimeSearchSpace.GetActiveCombinationBaselines().ToImmutableArray()) + { + AnsiConsole.MarkupLine( + $"[cyan]Base:[/] [bold]{string.Join("/", baseline.Names)}[/] [grey](RequiresImatrix={baseline.RequiresImatrix})[/]"); + + long baseTotal = 0; + + foreach (var batch in TensorConfigGenerator.GenerateTensorConfigBatches(baseline, batchSize: 10_000_000)) + { + baseTotal += batch.Count; + actualTotal += batch.Count; + + AnsiConsole.MarkupLine($" [green]Batch:[/] {batch.Count:N0} [grey]BaseRunning:[/] {baseTotal:N0}"); + batch.Clear(); + } + + AnsiConsole.MarkupLine($"[yellow]Base total:[/] {baseTotal:N0}"); + } + + sw.Stop(); + + bool match = actualTotal == expectedTotal; + + AnsiConsole.MarkupLine($"[bold green]Generated total:[/] {actualTotal:N0}"); + AnsiConsole.MarkupLine( + match + ? "[bold green] Counts match expected total[/]" + : $"[bold red] MISMATCH! Expected {expectedTotal:N0} but generated {actualTotal:N0}[/]"); + + var t = sw.Elapsed; + AnsiConsole.MarkupLine($"[bold]Elapsed:[/] {t.Hours}h {t.Minutes}m {t.Seconds}s {t.Milliseconds}ms"); + + Console.WriteLine(); + Console.WriteLine("---------------"); + Console.WriteLine(); + + var samplePlan = TensorConfigGenerator.GenerateInitialIsolationSamplePlan( + realResults ? Cache.UnusedTensorGroups : null); + + AnsiConsole.MarkupLine($"[bold green]Required pure baselines:[/] {samplePlan.PureBaselineCount:N0}"); + AnsiConsole.MarkupLine($"[bold green]Required base-only isolations:[/] {samplePlan.BaseOnlyIsolationCount:N0}"); + AnsiConsole.MarkupLine($"[bold green]Required smallest-probe isolations:[/] {samplePlan.GroupIsolationCount:N0}"); + AnsiConsole.MarkupLine($"[bold green]Total required startup samples:[/] {samplePlan.TotalCount:N0}"); + } + + public static void PrintTotalCombinationCount() + { + BigInteger total = ComboCounter.CountAll(); + + AnsiConsole.MarkupLine( + total > long.MaxValue + ? $"[red]Total potential combinations exceed Int64 range:[/] [bold yellow]{total:N0}[/]" + : $"[green]Total potential combinations:[/] [bold yellow]{total:N0}[/]"); + } + + public static List ParseArguments(string input) + { + var cliArgs = new List(); + var regex = new Regex(@"--(?[^\s=]+)(?:[\s=]+(?:""(?[^""]*)""|(?[^\s-]*)))?", RegexOptions.IgnoreCase); + var matches = regex.Matches(input); + + foreach (Match match in matches) + { + cliArgs.Add(new CliArg + { + Name = match.Groups["name"].Value, + Value = match.Groups["value"].Value + }); + } + + return cliArgs; + } + + public static List ParseArguments(IEnumerable arguments) + { + ArgumentNullException.ThrowIfNull(arguments); + + string[] tokens = arguments.ToArray(); + var cliArgs = new List(); + + for (int i = 0; i < tokens.Length; i++) + { + string token = tokens[i]; + if (!token.StartsWith("--", StringComparison.Ordinal) || token.Length <= 2) + continue; + + string option = token[2..]; + string name; + string value = string.Empty; + int equals = option.IndexOf('='); + + if (equals >= 0) + { + name = option[..equals]; + value = option[(equals + 1)..]; + } + else + { + name = option; + if (i + 1 < tokens.Length && !tokens[i + 1].StartsWith("--", StringComparison.Ordinal)) + value = tokens[++i]; + } + + cliArgs.Add(new CliArg + { + Name = name, + Value = value.Trim().Trim('"') + }); + } + + return cliArgs; + } + + public static void ShowHelp(Dictionary Factory)> commands) + { + AnsiConsole.Write(new Rule("[yellow]MagicQuant CLI[/]") { Justification = Justify.Left, Style = "grey" }); + AnsiConsole.WriteLine(); + + var table = new Table() + .AddColumn("[blue]Command[/]") + .AddColumn("[white]Description[/]") + .Border(TableBorder.Rounded) + .BorderColor(Color.Grey15); + + foreach (var cmd in commands) + table.AddRow($"[green]{cmd.Key}[/]", cmd.Value.Description); + + table.AddRow("[green]help[/]", "Show this help information"); + + AnsiConsole.Write(table); + AnsiConsole.MarkupLine("Usage: [bold]magicquant[/] [blue][[--option value]][/]"); + AnsiConsole.MarkupLine("Config: [green]--config[/] [grey][/] (CLI flags override YAML)"); + AnsiConsole.MarkupLine("Identity: [green]--architecture-family[/] [grey][/] | [green]--allow-architecture-family-alias-override[/]"); + AnsiConsole.WriteLine(); + } +} diff --git a/src/MagicQuant/Helpers/ComboLogic.cs b/src/MagicQuant/Helpers/ComboLogic.cs new file mode 100644 index 0000000..f217fdb --- /dev/null +++ b/src/MagicQuant/Helpers/ComboLogic.cs @@ -0,0 +1,85 @@ +using System.Collections.Immutable; +using System.Numerics; +using MQ.DB; +using MQ.DB.Models; + +namespace MagicQuant.Helpers; + +public static class ComboLogic +{ + private static readonly ImmutableArray GroupsOrdered = + TReg.All.OrderBy(g => g.UniqueId).ToImmutableArray(); + + public static ImmutableArray GetAllowedCandidateIdsPerGroup(BaselineQuants baseQuant) + { + bool imatrixAvailable = RuntimeSearchSpace.HasUsableImatrix(); + var builder = ImmutableArray.CreateBuilder(); + var unusedIds = Cache.UnusedTensorGroups.Select(x => x.UniqueId).ToHashSet(); + + foreach (var group in GroupsOrdered) + { + if (unusedIds.Contains(group.UniqueId)) + { + builder.Add([BaselineQuants.TensorConfigNullSlotValue]); + continue; + } + + var ids = new List(); + + foreach (var alias in BaselineQuants.GetExactHighPrecisionAliases(RuntimeSearchSpace.AllowHighPrecisionHybrids)) + { + if (RuntimeSearchSpace.IsBf16TensorChoiceSuppressed(group)) + continue; + + ids.Add(BaselineQuants.EncodeTensorConfigGroupSlot(alias)); + } + + var realCandidates = RuntimeSearchSpace.GetAllowedRealExplicitCombinationCandidatesForGroup(group); + ids.AddRange(realCandidates.Select(BaselineQuants.EncodeTensorConfigGroupSlot)); + + ids = ids.Distinct().ToList(); + + if (ids.Count == 0) + ids.Add(BaselineQuants.EncodeTensorConfigGroupSlot(BaselineQuants.GetDefaultExplicitFallbackBaseline())); + + builder.Add(ids.ToArray()); + } + + return builder.ToImmutable(); + } + + public static BigInteger CountCombinations(in BaselineQuants baseQuant) + { + var allowed = GetAllowedCandidateIdsPerGroup(baseQuant); + + BigInteger total = BigInteger.One; + for (int i = 0; i < allowed.Length; i++) + total *= allowed[i].Length; + + return total; + } +} + +public static class ComboCounter +{ + public static BigInteger CountForBase(BaselineQuants baseQuant) + { + var allowed = ComboLogic.GetAllowedCandidateIdsPerGroup(baseQuant); + + BigInteger total = BigInteger.One; + for (int i = 0; i < allowed.Length; i++) + total *= allowed[i].Length; + + return total; + } + + public static BigInteger CountAll() + { + BigInteger sum = BigInteger.Zero; + + foreach (var baseline in RuntimeSearchSpace.GetActiveCombinationBaselines()) + sum += CountForBase(baseline); + + return sum; + } +} diff --git a/src/MagicQuant/Helpers/DependencyManager.cs b/src/MagicQuant/Helpers/DependencyManager.cs new file mode 100644 index 0000000..8f1caa1 --- /dev/null +++ b/src/MagicQuant/Helpers/DependencyManager.cs @@ -0,0 +1,160 @@ +using System.Diagnostics; +using System.IO.Compression; +using System.Runtime.InteropServices; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Helpers; + +public static class DependencyManager +{ + // CMake Constants + private const string CmakeVersion = "3.29.0"; + private const string CmakeWinUrl = $"https://github.com/Kitware/CMake/releases/download/v{CmakeVersion}/cmake-{CmakeVersion}-windows-x86_64.zip"; + + public static async Task EnsureDependenciesAsync(SystemInfo sysInfo) + { + // 1. Check CMake (Download if missing on Windows) + string? cmakePath = GetCmakePath(); + if (cmakePath == null) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + await DownloadAndInstallCmakeAsync(); + } + else + { + // Linux usually handles this via apt earlier, but just in case: + throw new Exception("CMake is missing. Please run: sudo apt install cmake"); + } + } + + // 2. Check Visual Studio (Windows Only) + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + if (!CheckVisualStudio()) + { + PromptForVisualStudio(); + } + } + + // 3. Check GPU Toolkits (CUDA / ROCm / OneAPI) + await ValidateGpuToolkitAsync(sysInfo); + } + + private static async Task ValidateGpuToolkitAsync(SystemInfo sysInfo) + { + if (sysInfo.GpuInfo.FirstOrDefault()?.GpuVendor == GpuVendor.Nvidia) + { + // Check for NVCC + if (!CheckCommandExists("nvcc")) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + AnsiConsole.MarkupLine("[red]CUDA Toolkit not found![/]"); + AnsiConsole.MarkupLine("To use your NVIDIA GPU, you must install the CUDA Toolkit."); + AnsiConsole.MarkupLine("[link]https://developer.nvidia.com/cuda-downloads[/]"); + + if (!AnsiConsole.Confirm("Have you installed the CUDA Toolkit and are ready to retry?")) + { + throw new Exception("CUDA Toolkit required for Nvidia build."); + } + } + else + { + // Linux auto-install attempt or error + throw new Exception("CUDA Toolkit missing. Run: sudo apt install nvidia-cuda-toolkit"); + } + } + } + else if (sysInfo.GpuInfo.FirstOrDefault()?.GpuVendor == GpuVendor.Intel) + { + if (!CheckCommandExists("icx")) // Intel OneAPI Compiler + { + AnsiConsole.MarkupLine("[yellow]Warning: Intel OneAPI Base Toolkit not found.[/]"); + AnsiConsole.MarkupLine("For optimal Intel performance (SYCL), install OneAPI: [blue]https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html[/]"); + AnsiConsole.MarkupLine("Proceeding with CPU/Vulkan fallback if build fails."); + } + } + // AMD on Linux usually handled by "sudo apt install hipcc" or rocm libs + } + + // --- CMake Helpers --- + + public static string? GetCmakePath() + { + // 1. Check Global Path + if (CheckCommandExists("cmake")) return "cmake"; + + // 2. Check Local 'MagicQuant/cmake/bin' + string localPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + MagicConstants.MagicQuantFolder, "cmake", "bin", "cmake.exe"); + return File.Exists(localPath) ? localPath : null; + } + + private static async Task DownloadAndInstallCmakeAsync() + { + string magicPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), MagicConstants.MagicQuantFolder); + string zipPath = Path.Combine(magicPath, "cmake.zip"); + string extractPath = Path.Combine(magicPath, "cmake"); + + AnsiConsole.Status().Start("Downloading CMake...", ctx => + { + using var client = new HttpClient(); + var bytes = client.GetByteArrayAsync(CmakeWinUrl).Result; + File.WriteAllBytes(zipPath, bytes); + }); + + AnsiConsole.MarkupLine("Extracting CMake..."); + if (Directory.Exists(extractPath)) Directory.Delete(extractPath, true); + + ZipFile.ExtractToDirectory(zipPath, magicPath); + + // Rename the extracted folder (e.g., cmake-3.29-windows...) to just "cmake" + var extractedDir = Directory.GetDirectories(magicPath, "cmake-*").First(); + Directory.Move(extractedDir, extractPath); + + File.Delete(zipPath); + AnsiConsole.MarkupLine("[green]CMake installed successfully.[/]"); + } + + // --- Visual Studio Helpers --- + + private static bool CheckVisualStudio() + { + // Quick check for vswhere or cl.exe + return CheckCommandExists("cl") || File.Exists(@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"); + } + + private static void PromptForVisualStudio() + { + AnsiConsole.Write(new Rule("[red]Missing Visual Studio[/]")); + AnsiConsole.MarkupLine("MagicQuant requires [bold]Visual Studio Build Tools 2022[/] with C++ Desktop Development."); + AnsiConsole.MarkupLine("[blue]https://visualstudio.microsoft.com/downloads/#build-tools[/]"); + + if (!AnsiConsole.Confirm("Have you installed Visual Studio Build Tools?")) + { + throw new Exception("Visual Studio is required to compile on Windows."); + } + } + + private static bool CheckCommandExists(string cmd) + { + try + { + var psi = new ProcessStartInfo + { + FileName = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "where" : "which", + Arguments = cmd, + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true + }; + using var p = Process.Start(psi); + p?.WaitForExit(); + return p?.ExitCode == 0; + } + catch { return false; } + } +} diff --git a/src/MagicQuant/Helpers/EquivalentTruthSelectionHelper.cs b/src/MagicQuant/Helpers/EquivalentTruthSelectionHelper.cs new file mode 100644 index 0000000..c2aa24d --- /dev/null +++ b/src/MagicQuant/Helpers/EquivalentTruthSelectionHelper.cs @@ -0,0 +1,38 @@ +using MQ.DB.Models; + +namespace MagicQuant.Helpers; + +public static class EquivalentTruthSelectionHelper +{ + public static bool AreEquivalentTruths( + ulong leftSizeBytes, + double leftKld, + double leftPpl, + ulong rightSizeBytes, + double rightKld, + double rightPpl) + { + if (leftSizeBytes != rightSizeBytes) + return false; + + return Math.Abs(leftKld - rightKld) <= IsolationPruningConfig.FloatingPointEpsilon && + Math.Abs(leftPpl - rightPpl) <= IsolationPruningConfig.FloatingPointEpsilon; + } + + public static int GetBaselineSafetyRank( + BaselineQuants baseline, + bool isHybrid, + bool isExternalPureBaseline) + { + // Prefer the safest / most default representative when multiple rows have identical truth. + // 1) Higher BitRange is safer. + // 2) Higher ExplicitCandidateSortOrder wins ties inside the same BitRange. + // 3) Pure baseline beats hybrid when the measured truth is identical. + // 4) Internal/non-external beats external pure reference when still tied. + int rank = baseline.BitRange * 10_000; + rank += baseline.ExplicitCandidateSortOrder * 10; + rank += isHybrid ? 0 : 2; + rank += isExternalPureBaseline ? 0 : 1; + return rank; + } +} diff --git a/src/MagicQuant/Helpers/HardDeleteHelper.cs b/src/MagicQuant/Helpers/HardDeleteHelper.cs new file mode 100644 index 0000000..9615c84 --- /dev/null +++ b/src/MagicQuant/Helpers/HardDeleteHelper.cs @@ -0,0 +1,73 @@ +namespace MagicQuant.Helpers; + +public static class HardDeleteHelper +{ + public static async Task DeleteDirectoryIfExistsAsync( + string? directory, + CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(directory) || !Directory.Exists(directory)) + return; + + foreach (var file in Directory.EnumerateFiles(directory, "*", SearchOption.AllDirectories)) + { + ct.ThrowIfCancellationRequested(); + await DeleteFileIfExistsAsync(file); + } + + foreach (var sub in Directory.EnumerateDirectories(directory, "*", SearchOption.AllDirectories) + .OrderByDescending(x => x.Length)) + { + ct.ThrowIfCancellationRequested(); + if (Directory.Exists(sub)) + Directory.Delete(sub, recursive: false); + } + + if (Directory.Exists(directory)) + Directory.Delete(directory, recursive: false); + } + + public static async Task DeleteFileIfExistsAsync( + string? path, + int maxAttempts = 6, + int delayMs = 500) + { + if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) + return; + + Exception? lastError = null; + + for (int attempt = 1; attempt <= maxAttempts; attempt++) + { + try + { + var attributes = File.GetAttributes(path); + if ((attributes & FileAttributes.ReadOnly) != 0) + { + File.SetAttributes(path, attributes & ~FileAttributes.ReadOnly); + } + + File.Delete(path); + + if (!File.Exists(path)) + return; + } + catch (IOException ex) + { + lastError = ex; + } + catch (UnauthorizedAccessException ex) + { + lastError = ex; + } + + GC.Collect(); + GC.WaitForPendingFinalizers(); + await Task.Delay(delayMs); + } + + throw new IOException( + $"Failed to hard delete file '{path}' after {maxAttempts} attempts.", + lastError); + } +} diff --git a/src/MagicQuant/Helpers/HardwareHelper.cs b/src/MagicQuant/Helpers/HardwareHelper.cs new file mode 100644 index 0000000..5506d5e --- /dev/null +++ b/src/MagicQuant/Helpers/HardwareHelper.cs @@ -0,0 +1,710 @@ +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.RegularExpressions; +using MQ.DB.Models; + +namespace MagicQuant.Helpers; + +public static class HardwareHelper +{ + public static SystemInfo GetSystemInfo() + { + var detectedGpus = DetectGpus(); + var selectedGpus = SelectBestGpuVendorGroup(detectedGpus); + + if (selectedGpus.Count == 0) + { + selectedGpus = detectedGpus; + } + + return new SystemInfo + { + ThreadCount = Environment.ProcessorCount, + RamGb = GetTotalRamGb(), + GpuInfo = selectedGpus + }; + } + + private static List SelectBestGpuVendorGroup(List gpus) + { + if (gpus == null || gpus.Count == 0) + return new List(); + + var candidates = gpus + .Where(x => x != null) + .Where(x => x.GpuVendor != GpuVendor.Unknown && x.GpuVendor != GpuVendor.Cpu) + .Where(x => x.VramGb > 0.01) + .ToList(); + + if (candidates.Count == 0) + return new List(); + + var bestVendor = candidates + .GroupBy(x => x.GpuVendor) + .Select(g => new + { + Vendor = g.Key, + TotalVram = g.Sum(x => x.VramGb), + Count = g.Count() + }) + .OrderByDescending(x => x.TotalVram) + .ThenByDescending(x => x.Count) + .First() + .Vendor; + + return candidates + .Where(x => x.GpuVendor == bestVendor) + .OrderByDescending(x => x.VramGb) + .ThenBy(x => x.GpuName, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + public static double GetCudaVersion() + { + try + { + var result = RunProcess( + RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "nvcc.exe" : "nvcc", + "--version"); + + if (!result.Success || string.IsNullOrWhiteSpace(result.StdOut)) + return 0; + + var match = Regex.Match(result.StdOut, @"release (\d+\.\d+)"); + if (match.Success && + double.TryParse(match.Groups[1].Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var version)) + { + return version; + } + } + catch + { + // ignored + } + + return 0; + } + + private static double GetTotalRamGb() + { + try + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + // More reliable than GC.GetGCMemoryInfo for actual system RAM + var result = RunProcess( + "powershell", + "-NoProfile -ExecutionPolicy Bypass -Command \"(Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory\""); + + if (result.Success && + TryParseFirstInteger(result.StdOut, out var bytes) && + bytes > 0) + { + return BytesToGb(bytes); + } + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + var memInfo = "/proc/meminfo"; + if (File.Exists(memInfo)) + { + var line = File.ReadLines(memInfo) + .FirstOrDefault(x => x.StartsWith("MemTotal:", StringComparison.OrdinalIgnoreCase)); + + if (!string.IsNullOrWhiteSpace(line)) + { + var match = Regex.Match(line, @"MemTotal:\s+(\d+)\s+kB", RegexOptions.IgnoreCase); + if (match.Success && + ulong.TryParse(match.Groups[1].Value, out var kb)) + { + return kb / 1024.0 / 1024.0; + } + } + } + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + var result = RunProcess("sysctl", "-n hw.memsize"); + if (result.Success && + TryParseFirstInteger(result.StdOut, out var bytes) && + bytes > 0) + { + return BytesToGb(bytes); + } + } + } + catch + { + // ignored + } + + // Last-resort fallback. This is NOT actual total RAM, just available-to-GC-ish territory. + var fallback = GC.GetGCMemoryInfo().TotalAvailableMemoryBytes; + return fallback > 0 ? BytesToGb((ulong)fallback) : 0; + } + + private static List DetectGpus() + { + var gpus = new List(); + + try + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + gpus.AddRange(DetectGpusWindows()); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + gpus.AddRange(DetectGpusLinux()); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + gpus.AddRange(DetectGpusMac()); + } + } + catch + { + // ignored + } + + gpus = NormalizeAndDedupe(gpus); + + if (gpus.Count == 0) + { + gpus.Add(new GpuInfo + { + GpuVendor = GpuVendor.Cpu, + GpuName = "No discrete GPU detected", + VramGb = 0 + }); + } + + return gpus; + } + + // ========================= + // Windows + // ========================= + + private static IEnumerable DetectGpusWindows() +{ + var results = new List(); + + var nvidia = DetectNvidiaViaSmi().ToList(); + results.AddRange(nvidia); + + var ps = RunProcess( + "powershell", + "-NoProfile -ExecutionPolicy Bypass -Command \"Get-CimInstance Win32_VideoController | Select-Object Name,AdapterRAM,PNPDeviceID | ConvertTo-Json -Depth 3\""); + + if (ps.Success && !string.IsNullOrWhiteSpace(ps.StdOut)) + { + try + { + using var doc = JsonDocument.Parse(ps.StdOut); + + IEnumerable items = doc.RootElement.ValueKind switch + { + JsonValueKind.Array => doc.RootElement.EnumerateArray().ToArray(), + JsonValueKind.Object => new[] { doc.RootElement }, + _ => Array.Empty() + }; + + foreach (var item in items) + { + var name = item.TryGetProperty("Name", out var nameEl) + ? nameEl.GetString() ?? "Unknown" + : "Unknown"; + + var pnp = item.TryGetProperty("PNPDeviceID", out var pnpEl) + ? pnpEl.GetString() ?? string.Empty + : string.Empty; + + var vendor = DetectVendorFromNameOrId(name, pnp); + + // If we already have NVIDIA via nvidia-smi, skip weaker duplicate NVIDIA rows. + if (vendor == GpuVendor.Nvidia && nvidia.Count > 0) + continue; + + double vramGb = 0; + if (item.TryGetProperty("AdapterRAM", out var ramEl)) + { + if (ramEl.ValueKind == JsonValueKind.Number && ramEl.TryGetUInt64(out var bytes)) + { + vramGb = BytesToGb(bytes); + } + else if (ramEl.ValueKind == JsonValueKind.String && + ulong.TryParse(ramEl.GetString(), out var parsed)) + { + vramGb = BytesToGb(parsed); + } + } + + results.Add(new GpuInfo + { + GpuName = name, + GpuVendor = vendor, + VramGb = SanitizeGb(vramGb) + }); + } + } + catch + { + // ignored + } + } + + return results; +} + + // ========================= + // Linux + // ========================= + + private static IEnumerable DetectGpusLinux() + { + var results = new List(); + + // 1. NVIDIA: highest-confidence source + results.AddRange(DetectNvidiaViaSmi()); + + // 2. lspci for names/vendors + var lspci = RunProcess("bash", "-lc \"lspci -nn 2>/dev/null | grep -Ei 'vga|3d|display'\""); + var lspciEntries = new List<(string Name, GpuVendor Vendor)>(); + + if (lspci.Success && !string.IsNullOrWhiteSpace(lspci.StdOut)) + { + foreach (var line in SplitLines(lspci.StdOut)) + { + var name = ExtractGpuNameFromLspci(line); + var vendor = DetectVendorFromNameOrId(line, line); + + lspciEntries.Add((name, vendor)); + + results.Add(new GpuInfo + { + GpuName = name, + GpuVendor = vendor, + VramGb = 0 + }); + } + } + + // 3. AMD/Intel VRAM hints from /sys/class/drm + results = MergeLinuxSysFsData(results); + + return results; + } + + private static List MergeLinuxSysFsData(List existing) + { + try + { + var drmPath = "/sys/class/drm"; + if (!Directory.Exists(drmPath)) + return existing; + + var cardDirs = Directory.GetDirectories(drmPath, "card*") + .Where(x => !x.Contains("-", StringComparison.Ordinal)) // skip card0-DP-1 type connector entries + .OrderBy(x => x) + .ToList(); + + foreach (var cardDir in cardDirs) + { + var deviceDir = Path.Combine(cardDir, "device"); + if (!Directory.Exists(deviceDir)) + continue; + + string vendorId = ReadTrimmedFile(Path.Combine(deviceDir, "vendor")); + string deviceId = ReadTrimmedFile(Path.Combine(deviceDir, "device")); + + var vendor = DetectVendorFromPciVendorId(vendorId); + if (vendor == GpuVendor.Unknown) + continue; + + double vramGb = 0; + + // AMD dedicated VRAM often exposed here on amdgpu + var amdVramPath = Path.Combine(deviceDir, "mem_info_vram_total"); + if (File.Exists(amdVramPath) && + ulong.TryParse(ReadTrimmedFile(amdVramPath), out var amdBytes)) + { + vramGb = BytesToGb(amdBytes); + } + + // Intel integrated usually won’t have dedicated VRAM here. + // Leave as 0 instead of inventing nonsense. + + // Try to match an existing entry by vendor with 0 VRAM and patch it in. + var possibleMatches = existing + .Where(x => x.GpuVendor == vendor && x.VramGb <= 0.01) + .ToList(); + + GpuInfo? existingMatch = possibleMatches.Count == 1 ? possibleMatches[0] : null; + + if (existingMatch != null && vramGb > 0) + { + existingMatch.VramGb = SanitizeGb(vramGb); + } + else + { + existing.Add(new GpuInfo + { + GpuVendor = vendor, + GpuName = string.IsNullOrWhiteSpace(deviceId) + ? vendor.ToString() + : $"{vendor} GPU", + VramGb = SanitizeGb(vramGb) + }); + } + } + } + catch + { + // ignored + } + + return existing; + } + + // ========================= + // macOS + // ========================= + + private static IEnumerable DetectGpusMac() + { + var results = new List(); + + var sp = RunProcess("system_profiler", "SPDisplaysDataType -json"); + if (!sp.Success || string.IsNullOrWhiteSpace(sp.StdOut)) + return results; + + try + { + using var doc = JsonDocument.Parse(sp.StdOut); + + if (!doc.RootElement.TryGetProperty("SPDisplaysDataType", out var displays) || + displays.ValueKind != JsonValueKind.Array) + { + return results; + } + + foreach (var gpu in displays.EnumerateArray()) + { + string name = + GetJsonString(gpu, "sppci_model") ?? + GetJsonString(gpu, "_name") ?? + "Unknown"; + + double vramGb = 0; + + // Intel/older Macs may expose strings like "1536 MB" + var vramText = + GetJsonString(gpu, "spdisplays_vram") ?? + GetJsonString(gpu, "spdisplays_vram_shared") ?? + GetJsonString(gpu, "sppci_vram"); + + if (!string.IsNullOrWhiteSpace(vramText)) + { + vramGb = ParseMemoryStringToGb(vramText); + } + + // Apple Silicon often won’t expose dedicated VRAM because memory is unified. + // So leaving 0 here is more honest than lying. + + results.Add(new GpuInfo + { + GpuName = name, + GpuVendor = DetectVendorFromNameOrId(name, name), + VramGb = SanitizeGb(vramGb) + }); + } + } + catch + { + // ignored + } + + return results; + } + + // ========================= + // Shared NVIDIA path + // ========================= + + private static IEnumerable DetectNvidiaViaSmi() + { + var results = new List(); + + string exe = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "nvidia-smi.exe" : "nvidia-smi"; + var smi = RunProcess(exe, "--query-gpu=gpu_uuid,name,memory.total --format=csv,noheader,nounits"); + + if (!smi.Success || string.IsNullOrWhiteSpace(smi.StdOut)) + return results; + + foreach (var line in SplitLines(smi.StdOut)) + { + var parts = line.Split(',', StringSplitOptions.TrimEntries); + + if (parts.Length < 3) + continue; + + var uuid = parts[0].Trim(); + var name = parts[1].Trim(); + + double vramGb = 0; + if (double.TryParse(parts[2].Trim(), NumberStyles.Any, CultureInfo.InvariantCulture, out var memMb)) + { + vramGb = memMb / 1024.0; + } + + results.Add(new GpuInfo + { + UniqueId = uuid, + GpuVendor = GpuVendor.Nvidia, + GpuName = string.IsNullOrWhiteSpace(name) ? "NVIDIA GPU" : name, + VramGb = SanitizeGb(vramGb) + }); + } + + return results; + } + + // ========================= + // Helpers + // ========================= + + private static List NormalizeAndDedupe(List gpus) + { + var final = new List(); + + foreach (var gpu in gpus) + { + var normalizedName = string.IsNullOrWhiteSpace(gpu.GpuName) + ? "Unknown" + : Regex.Replace(gpu.GpuName.Trim(), @"\s+", " "); + + var vendor = gpu.GpuVendor == GpuVendor.Unknown + ? DetectVendorFromNameOrId(normalizedName, gpu.UniqueId ?? normalizedName) + : gpu.GpuVendor; + + GpuInfo? existing = null; + + // Only merge by UniqueId if we actually have one + if (!string.IsNullOrWhiteSpace(gpu.UniqueId)) + { + existing = final.FirstOrDefault(x => + !string.IsNullOrWhiteSpace(x.UniqueId) && + string.Equals(x.UniqueId, gpu.UniqueId, StringComparison.OrdinalIgnoreCase)); + } + + // If no UniqueId, do NOT aggressively merge by name/vendor. + // That breaks multi-GPU systems with identical cards. + if (existing == null) + { + final.Add(new GpuInfo + { + UniqueId = gpu.UniqueId, + GpuVendor = vendor, + GpuName = normalizedName, + VramGb = SanitizeGb(gpu.VramGb) + }); + } + else + { + if (existing.VramGb <= 0.01 && gpu.VramGb > existing.VramGb) + existing.VramGb = SanitizeGb(gpu.VramGb); + + if (string.Equals(existing.GpuName, "Unknown", StringComparison.OrdinalIgnoreCase) && + !string.Equals(normalizedName, "Unknown", StringComparison.OrdinalIgnoreCase)) + { + existing.GpuName = normalizedName; + } + + if (existing.GpuVendor == GpuVendor.Unknown && vendor != GpuVendor.Unknown) + existing.GpuVendor = vendor; + } + } + + if (final.Any(x => x.GpuVendor != GpuVendor.Cpu)) + { + final.RemoveAll(x => x.GpuVendor == GpuVendor.Cpu); + } + + return final; + } + + private static GpuVendor DetectVendorFromNameOrId(string? name, string? idText) + { + var haystack = $"{name} {idText}".ToLowerInvariant(); + + if (haystack.Contains("nvidia") || haystack.Contains("geforce") || haystack.Contains("quadro") || haystack.Contains("tesla")) + return GpuVendor.Nvidia; + + if (haystack.Contains("amd") || haystack.Contains("advanced micro devices") || haystack.Contains("radeon") || haystack.Contains("firepro")) + return GpuVendor.Amd; + + if (haystack.Contains("intel") || haystack.Contains("arc") || haystack.Contains("uhd") || haystack.Contains("iris")) + return GpuVendor.Intel; + + return GpuVendor.Unknown; + } + + private static GpuVendor DetectVendorFromPciVendorId(string? vendorId) + { + if (string.IsNullOrWhiteSpace(vendorId)) + return GpuVendor.Unknown; + + var v = vendorId.Trim().ToLowerInvariant(); + + return v switch + { + "0x10de" => GpuVendor.Nvidia, + "0x1002" => GpuVendor.Amd, + "0x8086" => GpuVendor.Intel, + _ => GpuVendor.Unknown + }; + } + + private static string ExtractGpuNameFromLspci(string line) + { + if (string.IsNullOrWhiteSpace(line)) + return "Unknown"; + + var idx = line.IndexOf(':'); + if (idx >= 0 && idx < line.Length - 1) + { + var right = line[(idx + 1)..].Trim(); + + // Strip "VGA compatible controller:" / "3D controller:" / "Display controller:" + right = Regex.Replace( + right, + @"^(VGA compatible controller|3D controller|Display controller)\s*:\s*", + "", + RegexOptions.IgnoreCase).Trim(); + + return right; + } + + return line.Trim(); + } + + private static string? GetJsonString(JsonElement element, string propertyName) + { + if (!element.TryGetProperty(propertyName, out var prop)) + return null; + + return prop.ValueKind == JsonValueKind.String ? prop.GetString() : prop.ToString(); + } + + private static double ParseMemoryStringToGb(string text) + { + if (string.IsNullOrWhiteSpace(text)) + return 0; + + var match = Regex.Match(text, @"([\d.]+)\s*(TB|GB|MB|KB)", RegexOptions.IgnoreCase); + if (!match.Success) + return 0; + + if (!double.TryParse(match.Groups[1].Value, NumberStyles.Any, CultureInfo.InvariantCulture, out var value)) + return 0; + + var unit = match.Groups[2].Value.ToUpperInvariant(); + return unit switch + { + "TB" => value * 1024.0, + "GB" => value, + "MB" => value / 1024.0, + "KB" => value / 1024.0 / 1024.0, + _ => 0 + }; + } + + private static bool TryParseFirstInteger(string? text, out ulong value) + { + value = 0; + if (string.IsNullOrWhiteSpace(text)) + return false; + + var match = Regex.Match(text, @"\d+"); + return match.Success && ulong.TryParse(match.Value, out value); + } + + private static string ReadTrimmedFile(string path) + { + try + { + return File.Exists(path) ? File.ReadAllText(path).Trim() : string.Empty; + } + catch + { + return string.Empty; + } + } + + private static IEnumerable SplitLines(string text) + { + return text.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + } + + private static double BytesToGb(ulong bytes) + { + return bytes / 1024.0 / 1024.0 / 1024.0; + } + + private static double SanitizeGb(double gb) + { + if (double.IsNaN(gb) || double.IsInfinity(gb) || gb < 0) + return 0; + + // Keep this nice and user-facing + return Math.Round(gb, 2); + } + + private static ProcessResult RunProcess(string fileName, string arguments, int timeoutMs = 5000) + { + try + { + var psi = new ProcessStartInfo + { + FileName = fileName, + Arguments = arguments, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + using var process = new Process { StartInfo = psi }; + process.Start(); + + var stdOutTask = process.StandardOutput.ReadToEndAsync(); + var stdErrTask = process.StandardError.ReadToEndAsync(); + + if (!process.WaitForExit(timeoutMs)) + { + try { process.Kill(entireProcessTree: true); } catch { } + return ProcessResult.Failure("Process timed out."); + } + + Task.WaitAll(stdOutTask, stdErrTask); + + return new ProcessResult( + process.ExitCode == 0, + stdOutTask.Result ?? string.Empty, + stdErrTask.Result ?? string.Empty, + process.ExitCode); + } + catch (Exception ex) + { + return ProcessResult.Failure(ex.Message); + } + } + + private readonly record struct ProcessResult(bool Success, string StdOut, string StdErr, int ExitCode) + { + public static ProcessResult Failure(string error) => new(false, string.Empty, error, -1); + } +} \ No newline at end of file diff --git a/src/MagicQuant/Helpers/IsolationPruningConfig.cs b/src/MagicQuant/Helpers/IsolationPruningConfig.cs new file mode 100644 index 0000000..41eb01c --- /dev/null +++ b/src/MagicQuant/Helpers/IsolationPruningConfig.cs @@ -0,0 +1,14 @@ +namespace MagicQuant.Helpers; + +public static class IsolationPruningConfig +{ + public static double MinimumIsolationReductionToContinueRatio => Config.Current.IsolationPruning.MinimumIsolationReductionToContinueRatio; + public static double MinimumIsolationReductionToSuppressBf16Ratio => Config.Current.IsolationPruning.MinimumIsolationReductionToSuppressBf16Ratio; + public static double MaximumIsolationPplDeltaPercent => Config.Current.IsolationPruning.MaximumIsolationPplDeltaPercent; + public static double MaximumIsolationKld => Config.Current.IsolationPruning.MaximumIsolationKld; + public static double BadTradeMaxSizeDeltaPercent => Config.Current.IsolationPruning.BadTradeMaxSizeDeltaPercent; + public static double BadTradeKldMultiplier => Config.Current.IsolationPruning.BadTradeKldMultiplier; + public static double BadTradePplMultiplier => Config.Current.IsolationPruning.BadTradePplMultiplier; + public static double FloatingPointEpsilon => Config.Current.IsolationPruning.FloatingPointEpsilon; + public static double MinimumMeaningfulBaseOnlyReductionRatio => Config.Current.IsolationPruning.MinimumMeaningfulBaseOnlyReductionRatio; +} diff --git a/src/MagicQuant/Helpers/JsonHelper.cs b/src/MagicQuant/Helpers/JsonHelper.cs new file mode 100644 index 0000000..2505e08 --- /dev/null +++ b/src/MagicQuant/Helpers/JsonHelper.cs @@ -0,0 +1,106 @@ +using System.Text.Json; +using Spectre.Console; +using MQ.DB; +using MQ.DB.Models; + +namespace MagicQuant.Helpers; + +public static class JsonHelper +{ + // Priority list of keys to look for + private static readonly List DtypeKeys = new() + { + "torch_dtype", + "dtype", + "prec", + "precision" + }; + + public static void DetectAndSetTorchType(string modelDir) + { + string configPath = Path.Combine(modelDir, "config.json"); + + if (!File.Exists(configPath)) + { + throw new FileNotFoundException($"Could not find 'config.json' in {modelDir}"); + } + + try + { + string jsonContent = File.ReadAllText(configPath); + using JsonDocument doc = JsonDocument.Parse(jsonContent); + + // Recursive search for the key + string? dtypeValue = FindKeyRecursive(doc.RootElement, DtypeKeys); + + if (string.IsNullOrWhiteSpace(dtypeValue)) + { + AnsiConsole.MarkupLine("[yellow]Warning:[/] Could not find 'torch_dtype' in config.json. Defaulting to [bold]BF16[/]."); + Cache.TorchType = Cache.MainTorchType.BF16; + return; + } + + // Parse the value + Cache.TorchType = ParseTorchType(dtypeValue); + AnsiConsole.MarkupLine($"[grey]Detected Model Type:[/] [cyan]{Cache.TorchType}[/] (from '{dtypeValue}')"); + } + catch (Exception ex) + { + AnsiConsole.MarkupLine($"[red]Error parsing config.json:[/] {ex.Message}"); + // Fail safe or throw depending on strictness. + // Usually safe to default if we assume modern models. + Cache.TorchType = Cache.MainTorchType.BF16; + } + } + + private static string? FindKeyRecursive(JsonElement element, List targetKeys) + { + if (element.ValueKind == JsonValueKind.Object) + { + // 1. Check current level first (Optimization) + foreach (var prop in element.EnumerateObject()) + { + if (targetKeys.Contains(prop.Name, StringComparer.OrdinalIgnoreCase) && + prop.Value.ValueKind == JsonValueKind.String) + { + return prop.Value.GetString(); + } + } + + // 2. Recurse into children + foreach (var prop in element.EnumerateObject()) + { + // Skip if not object or array to save time + if (prop.Value.ValueKind == JsonValueKind.Object || prop.Value.ValueKind == JsonValueKind.Array) + { + string? found = FindKeyRecursive(prop.Value, targetKeys); + if (found != null) return found; + } + } + } + else if (element.ValueKind == JsonValueKind.Array) + { + foreach (var item in element.EnumerateArray()) + { + string? found = FindKeyRecursive(item, targetKeys); + if (found != null) return found; + } + } + + return null; + } + + private static Cache.MainTorchType ParseTorchType(string value) + { + // Normalize + string v = value.ToLowerInvariant().Trim(); + + return v switch + { + "bfloat16" => Cache.MainTorchType.BF16, + "float16" => Cache.MainTorchType.F16, + "float32" => Cache.MainTorchType.F32, + _ => Cache.MainTorchType.BF16 // Default fallback + }; + } +} \ No newline at end of file diff --git a/src/MagicQuant/Helpers/LinuxHelper.cs b/src/MagicQuant/Helpers/LinuxHelper.cs new file mode 100644 index 0000000..9910aeb --- /dev/null +++ b/src/MagicQuant/Helpers/LinuxHelper.cs @@ -0,0 +1,36 @@ +using System.Diagnostics; +using Spectre.Console; + +namespace MagicQuant.Helpers; + +public class LinuxHelper +{ + public static async Task RefreshSudoCredentialsAsync() + { + AnsiConsole.MarkupLine("[grey]Verifying sudo access for system installs...[/]"); + + // "sudo -v" updates the user's cached credentials. + // It will prompt for a password if necessary. + var psi = new ProcessStartInfo + { + FileName = "sudo", + Arguments = "-v", + UseShellExecute = false // Let standard input handle the password prompt + }; + + using var p = Process.Start(psi) ?? throw new InvalidOperationException("Could not start sudo."); + try { await p.WaitForExitAsync(MagicQuant.Runtime.RunCancellation.Token); } + catch (OperationCanceledException) + { + try { if (!p.HasExited) p.Kill(entireProcessTree: true); } + catch (InvalidOperationException) { } + await p.WaitForExitAsync(CancellationToken.None); + throw; + } + + if (p.ExitCode != 0) + { + throw new Exception("Sudo access denied or cancelled."); + } + } +} diff --git a/src/MagicQuant/Helpers/LlamaBuilder.cs b/src/MagicQuant/Helpers/LlamaBuilder.cs new file mode 100644 index 0000000..c0a8885 --- /dev/null +++ b/src/MagicQuant/Helpers/LlamaBuilder.cs @@ -0,0 +1,342 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; +using LibGit2Sharp; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Helpers; + +public class LlamaBuilder +{ + private const string LlamaCppRepositoryUrl = "https://github.com/ggerganov/llama.cpp.git"; + + private readonly string _llamaRoot; + private readonly SystemInfo _sysInfo; + + public LlamaBuilder(string magicRoot, SystemInfo sysInfo) + { + _llamaRoot = Path.Combine(magicRoot, MagicConstants.LlamaRepoName); + _sysInfo = sysInfo; + + Cache.LlamaRoot = _llamaRoot; + Cache.LlamaBin = Path.Combine(_llamaRoot, "build", "bin"); + Cache.ConvertScript = ResolveConvertScriptPath(_llamaRoot); + } + + public string GetLlamaBinPath() => Path.Combine(_llamaRoot, "build", "bin"); + + public async Task PrepareAndBuildAsync(bool forceRebuild) + { + // 1. Validate ALL dependencies before doing anything. + await DependencyManager.EnsureDependenciesAsync(_sysInfo); + + // 2. Ensure the llama.cpp checkout is real and buildable. + EnsureLlamaRepository(forceRebuild); + + Cache.LlamaRoot = _llamaRoot; + Cache.LlamaBin = Path.Combine(_llamaRoot, "build", "bin"); + Cache.ConvertScript = ResolveConvertScriptPath(_llamaRoot); + + ValidateLlamaSourceTreeOrThrow(); + + // 3. Setup Build Directory. + string buildDir = Path.Combine(_llamaRoot, "build"); + EnsureBuildDirectory(buildDir, forceRebuild); + + // 4. Get the CMake Executable (System or Local). + string cmakeExe = DependencyManager.GetCmakePath() ?? "cmake"; + + // 5. Generate Build Files. + var cmakeArgs = GetOptimalCmakeArgs(); + AnsiConsole.MarkupLine($"[grey]Configuring build with: {Markup.Escape(FormatArgsForDisplay(cmakeArgs))}[/]"); + + if (!await RunProcessAsync(cmakeExe, cmakeArgs, buildDir)) + throw new Exception("CMake configuration failed."); + + // 6. Compile. + AnsiConsole.MarkupLine("[cyan]Compiling llama.cpp (Release Mode)...[/]"); + + var buildArgs = new List + { + "--build", + ".", + "--config", + "Release", + "-j", + Environment.ProcessorCount.ToString() + }; + + if (!await RunProcessAsync(cmakeExe, buildArgs, buildDir)) + throw new Exception("Build failed."); + + AnsiConsole.MarkupLine("[green]✔ Build Success![/]"); + } + + private void EnsureLlamaRepository(bool forceRebuild) + { + bool rootExists = Directory.Exists(_llamaRoot); + bool rootIsValid = IsValidLlamaSourceTree(); + + if (forceRebuild && rootExists) + { + AnsiConsole.MarkupLine("[yellow]Update requested: removing existing llama.cpp checkout...[/]"); + DeleteDirectoryOrThrow(_llamaRoot, "update was requested"); + rootExists = false; + rootIsValid = false; + } + + if (rootExists && !rootIsValid) + { + var escapedRoot = Markup.Escape(_llamaRoot); + AnsiConsole.MarkupLine($"[yellow]Existing llama.cpp directory is invalid or incomplete: {escapedRoot}[/]"); + AnsiConsole.MarkupLine("[grey]Missing CMakeLists.txt or repository metadata. Removing it so MagicQuant can redeploy a clean checkout.[/]"); + DeleteDirectoryOrThrow(_llamaRoot, "existing llama.cpp checkout is invalid/incomplete"); + rootExists = false; + } + + if (!rootExists) + { + CloneLlamaRepository(); + return; + } + + AnsiConsole.MarkupLine("[grey]Valid llama.cpp repository already exists. Skipping clone.[/]"); + } + + private void CloneLlamaRepository() + { + var escapedRoot = Markup.Escape(_llamaRoot); + AnsiConsole.MarkupLine($"Cloning llama.cpp to [blue]{escapedRoot}[/]..."); + AnsiConsole.MarkupLine("[grey](This includes submodules and may take a moment.)[/]"); + + Directory.CreateDirectory(Path.GetDirectoryName(_llamaRoot)!); + + var cloneOptions = new CloneOptions + { + RecurseSubmodules = true + }; + + try + { + Repository.Clone(LlamaCppRepositoryUrl, _llamaRoot, cloneOptions); + } + catch + { + if (Directory.Exists(_llamaRoot) && !IsValidLlamaSourceTree()) + DeleteDirectoryOrThrow(_llamaRoot, "clone failed and left a partial checkout"); + + throw; + } + + ValidateLlamaSourceTreeOrThrow(); + } + + private bool IsValidLlamaSourceTree() + { + if (!Directory.Exists(_llamaRoot)) + return false; + + // CMakeLists.txt is the non-negotiable build root. The previous bug was + // caused by trusting Directory.Exists(_llamaRoot) even when this file was gone. + if (!File.Exists(Path.Combine(_llamaRoot, "CMakeLists.txt"))) + return false; + + // Prefer a real git checkout for auto-managed installs. If a user points at a + // custom source tree, that path is handled by InitializeLlamaCpp custom args. + if (!Directory.Exists(Path.Combine(_llamaRoot, ".git"))) + return false; + + return true; + } + + private void ValidateLlamaSourceTreeOrThrow() + { + string cmakeLists = Path.Combine(_llamaRoot, "CMakeLists.txt"); + if (!File.Exists(cmakeLists)) + { + throw new DirectoryNotFoundException( + $"llama.cpp checkout is not buildable. Expected CMakeLists.txt at: {cmakeLists}. " + + "Delete the llama.cpp directory or rerun initialize-llama-cpp --update so MagicQuant can redeploy it."); + } + } + + private void EnsureBuildDirectory(string buildDir, bool forceRebuild) + { + if (Directory.Exists(buildDir)) + { + if (forceRebuild) + { + DeleteDirectoryOrThrow(buildDir, "clean rebuild requested"); + } + else if (!BuildCacheMatchesCurrentSource(buildDir)) + { + AnsiConsole.MarkupLine("[yellow]Existing CMake build cache points at a different or invalid source tree. Recreating build directory...[/]"); + DeleteDirectoryOrThrow(buildDir, "CMake cache does not match the active llama.cpp source tree"); + } + } + + Directory.CreateDirectory(buildDir); + } + + private bool BuildCacheMatchesCurrentSource(string buildDir) + { + string cacheFile = Path.Combine(buildDir, "CMakeCache.txt"); + if (!File.Exists(cacheFile)) + return true; + + try + { + foreach (string line in File.ReadLines(cacheFile)) + { + if (!line.StartsWith("CMAKE_HOME_DIRECTORY:INTERNAL=", StringComparison.Ordinal)) + continue; + + string cachedSource = line["CMAKE_HOME_DIRECTORY:INTERNAL=".Length..].Trim(); + if (string.IsNullOrWhiteSpace(cachedSource)) + return false; + + string normalizedCached = NormalizePath(cachedSource); + string normalizedCurrent = NormalizePath(_llamaRoot); + + return string.Equals(normalizedCached, normalizedCurrent, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) + ? StringComparison.OrdinalIgnoreCase + : StringComparison.Ordinal); + } + + return true; + } + catch + { + return false; + } + } + + private List GetOptimalCmakeArgs() + { + var args = new List + { + _llamaRoot, + "-DCMAKE_BUILD_TYPE=Release" + }; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + args.Add("-G"); + args.Add("Ninja"); + } + + switch (_sysInfo.GpuInfo.FirstOrDefault()?.GpuVendor) + { + case GpuVendor.Nvidia: + args.Add("-DGGML_CUDA=ON"); + args.Add("-DCMAKE_CUDA_ARCHITECTURES=native"); + break; + + case GpuVendor.Amd: + args.Add("-DGGML_HIPBLAS=ON"); + break; + + case GpuVendor.Intel: + args.Add("-DGGML_SYCL=ON"); + break; + } + + return args; + } + + private async Task RunProcessAsync(string exe, IReadOnlyList args, string workingDir) + { + var psi = new ProcessStartInfo + { + FileName = exe, + WorkingDirectory = workingDir, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + + foreach (string arg in args) + psi.ArgumentList.Add(arg); + + var result = await new MagicQuant.Runtime.ProcessRunner().RunAsync(psi, + onLine: (line, _) => AnsiConsole.WriteLine(line)); + return result.Success; + } + + private static string ResolveConvertScriptPath(string llamaRoot) + { + // llama.cpp has kept this at repo root for the relevant toolchain. Keep a + // tiny candidate list so a future minor layout change does not poison Cache. + string[] candidates = + { + Path.Combine(llamaRoot, "convert_hf_to_gguf.py"), + Path.Combine(llamaRoot, "convert.py") + }; + + return candidates.FirstOrDefault(File.Exists) ?? candidates[0]; + } + + private static void DeleteDirectoryOrThrow(string path, string reason) + { + if (!Directory.Exists(path)) + return; + + try + { + MakeDirectoryWritable(path); + Directory.Delete(path, recursive: true); + } + catch (Exception ex) + { + throw new IOException( + $"Could not remove directory '{path}' while repairing llama.cpp ({reason}). " + + "Close any terminals/editors using that path or delete it manually, then rerun initialize-llama-cpp.", ex); + } + } + + private static void MakeDirectoryWritable(string root) + { + try + { + foreach (string file in Directory.EnumerateFiles(root, "*", SearchOption.AllDirectories)) + { + var attributes = File.GetAttributes(file); + if ((attributes & FileAttributes.ReadOnly) != 0) + File.SetAttributes(file, attributes & ~FileAttributes.ReadOnly); + } + + foreach (string directory in Directory.EnumerateDirectories(root, "*", SearchOption.AllDirectories)) + { + var attributes = File.GetAttributes(directory); + if ((attributes & FileAttributes.ReadOnly) != 0) + File.SetAttributes(directory, attributes & ~FileAttributes.ReadOnly); + } + } + catch + { + // Best-effort only. Directory.Delete will throw a clearer failure if this mattered. + } + } + + private static string NormalizePath(string path) + { + return Path.GetFullPath(path) + .TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + } + + private static string FormatArgsForDisplay(IEnumerable args) + { + return string.Join(" ", args.Select(QuoteIfNeeded)); + } + + private static string QuoteIfNeeded(string arg) + { + if (string.IsNullOrEmpty(arg)) + return "\"\""; + + return arg.Any(char.IsWhiteSpace) + ? $"\"{arg.Replace("\"", "\\\"")}\"" + : arg; + } +} diff --git a/src/MagicQuant/Helpers/MagicQuantDiagnostics.cs b/src/MagicQuant/Helpers/MagicQuantDiagnostics.cs new file mode 100644 index 0000000..916640f --- /dev/null +++ b/src/MagicQuant/Helpers/MagicQuantDiagnostics.cs @@ -0,0 +1,56 @@ +using System.Runtime.CompilerServices; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Helpers; + +public static class MagicQuantDiagnostics +{ + private const string VerboseEnv = "MAGICQUANT_DIAG_VERBOSE_ISOLATION_PRUNING"; + private const string FocusGroupEnv = "MAGICQUANT_DIAG_GROUP"; + + public static bool VerboseIsolationPruning => + string.Equals(Environment.GetEnvironmentVariable(VerboseEnv), "1", StringComparison.OrdinalIgnoreCase) || + string.Equals(Environment.GetEnvironmentVariable(VerboseEnv), "true", StringComparison.OrdinalIgnoreCase); + + public static string? FocusGroup => Environment.GetEnvironmentVariable(FocusGroupEnv); + + public static bool ShouldLogGroup(TensorGroup group) + { + if (!VerboseIsolationPruning) + return false; + + if (string.IsNullOrWhiteSpace(FocusGroup)) + return true; + + return string.Equals(group.Name, FocusGroup, StringComparison.OrdinalIgnoreCase) + || string.Equals(group.UniqueId.ToString(), FocusGroup, StringComparison.OrdinalIgnoreCase); + } + + public static void Log(string tag, string message) + { + if (!VerboseIsolationPruning) + return; + AnsiConsole.MarkupLine($"[grey][diag:{Markup.Escape(tag)}][/]: {Markup.Escape(message)}"); + } + + public static string CandidateLabel(BaselineQuants candidate) => $"{candidate.Names[0]}(id={candidate.UniqueId})"; + + public static void LogRuntimeMutation( + string phase, + TensorGroup group, + BaselineQuants? candidate, + string reason, + int before, + int after, + [CallerMemberName] string caller = "") + { + if (!ShouldLogGroup(group)) + return; + + var candidateText = candidate == null ? "" : CandidateLabel(candidate); + Log("runtime-ban", + $"phase={phase} group={group.Name}(id={group.UniqueId}) candidate={candidateText} reason=\"{reason}\" allowedBefore={before} allowedAfter={after} caller={caller}"); + } +} diff --git a/src/MagicQuant/Helpers/MagicQuantModelId.cs b/src/MagicQuant/Helpers/MagicQuantModelId.cs new file mode 100644 index 0000000..8792ba0 --- /dev/null +++ b/src/MagicQuant/Helpers/MagicQuantModelId.cs @@ -0,0 +1,98 @@ +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using Blake3; + + +namespace MagicQuant.Helpers; + +public static class MagicQuantModelId +{ + private const string IdFileName = "MagicQuant.id.json"; + private const string IdPrefix = "mq-blake3:"; + + private sealed class ModelIdWrapper + { + public string Id { get; set; } = default!; + } + + public static string GetOrCreateModelId(string modelDirectory) + { + if (string.IsNullOrWhiteSpace(modelDirectory)) + throw new ArgumentException("Model directory path is null or empty.", nameof(modelDirectory)); + + if (!Directory.Exists(modelDirectory)) + throw new DirectoryNotFoundException($"Directory does not exist: {modelDirectory}"); + + string idFilePath = Path.Combine(modelDirectory, IdFileName); + + // 🚀 Fast path: cached ID exists + if (File.Exists(idFilePath)) + { + var wrapper = JsonSerializer.Deserialize(File.ReadAllText(idFilePath)); + if (string.IsNullOrWhiteSpace(wrapper?.Id)) + throw new InvalidDataException($"{IdFileName} exists but is invalid (missing Id)."); + + return wrapper.Id; + } + + // 🔍 Find safetensors + var safetensors = Directory + .EnumerateFiles(modelDirectory, "*.safetensors", SearchOption.TopDirectoryOnly) + .OrderBy(f => f, StringComparer.OrdinalIgnoreCase) + .ToArray(); + + if (safetensors.Length == 0) + throw new InvalidOperationException($"No .safetensors files found in directory: {modelDirectory}"); + + // 🧠 Hash tensor payloads deterministically + using var hasher = Hasher.New(); + + foreach (var file in safetensors) + HashSafetensorPayload(file, hasher); + + // ✅ Blake3.NET: Finalize() returns Blake3.Hash which stringifies to hex + var hash = hasher.Finalize(); + string finalHashHex = hash.ToString(); // hex digest (lowercase) + string finalId = IdPrefix + finalHashHex; + + // 💾 Persist ID + var output = new ModelIdWrapper { Id = finalId }; + + File.WriteAllText( + idFilePath, + JsonSerializer.Serialize(output, new JsonSerializerOptions { WriteIndented = true }), + Encoding.UTF8 + ); + + return finalId; + } + + /// + /// Hashes ONLY the tensor payload of a safetensors file (skips header + JSON metadata). + /// Safetensors format: [u64 header_len][header_json_bytes][tensor_bytes...] + /// + private static void HashSafetensorPayload(string filePath, Hasher hasher) + { + using var stream = File.OpenRead(filePath); + using var reader = new BinaryReader(stream); + + // UInt64 metadata length (little endian) + ulong metadataLength = reader.ReadUInt64(); + + long tensorDataOffset = 8L + checked((long)metadataLength); + + if (tensorDataOffset >= stream.Length) + throw new InvalidDataException($"Safetensors file is malformed (bad header length): {filePath}"); + + stream.Position = tensorDataOffset; + + byte[] buffer = new byte[1024 * 1024]; // 1MB + int bytesRead; + + while ((bytesRead = stream.Read(buffer, 0, buffer.Length)) > 0) + hasher.Update(buffer.AsSpan(0, bytesRead)); + } +} \ No newline at end of file diff --git a/src/MagicQuant/Helpers/NativePrecisionNormalization.cs b/src/MagicQuant/Helpers/NativePrecisionNormalization.cs new file mode 100644 index 0000000..479aa38 --- /dev/null +++ b/src/MagicQuant/Helpers/NativePrecisionNormalization.cs @@ -0,0 +1,70 @@ +using MQ.DB.Models; + +namespace MagicQuant.Helpers; + +public static class NativePrecisionNormalization +{ + public static string NormalizeLearnedFinalQuantTypeForApplication(string? observedFinalQuantType) + { + if (string.IsNullOrWhiteSpace(observedFinalQuantType)) + return string.Empty; + + var canonical = Canonicalize(observedFinalQuantType); + var native = TensorWeightScheme.GetCurrentNativePrecisionScheme(); + + // Keep raw F32 as-is. Do not silently collapse F32 to BF16/F16. + if (canonical == "F32") + return TensorWeightScheme.F32.Names[0]; + + // Treat F16/BF16 as "native high precision kept" when replaying + // learned behavior into the active source model. + if (canonical == "F16" || canonical == "BF16") + return native.Names[0]; + + return observedFinalQuantType.Trim(); + } + + public static IReadOnlyCollection ResolveSchemeIdsForLearnedFinalQuantType(string? observedFinalQuantType) + { + var result = new HashSet(); + + if (string.IsNullOrWhiteSpace(observedFinalQuantType)) + return result; + + var canonical = Canonicalize(observedFinalQuantType); + + // Keep F32 exact if observed. + if (canonical == "F32") + { + result.Add(TensorWeightScheme.F32.UniqueId); + return result; + } + + // Treat F16/BF16 as native high precision for pruning/application logic. + if (canonical == "F16" || canonical == "BF16") + { + result.Add(TensorWeightScheme.GetCurrentNativePrecisionScheme().UniqueId); + return result; + } + + foreach (var scheme in TensorWeightScheme.All) + { + if (scheme.Names.IsDefaultOrEmpty) + continue; + + if (scheme.Names.Any(x => Canonicalize(x) == canonical)) + result.Add(scheme.UniqueId); + } + + return result; + } + + private static string Canonicalize(string value) + { + return value + .Trim() + .Replace("-", "_") + .Replace(" ", string.Empty) + .ToUpperInvariant(); + } +} diff --git a/src/MagicQuant/Helpers/PythonManager.cs b/src/MagicQuant/Helpers/PythonManager.cs new file mode 100644 index 0000000..c381b1a --- /dev/null +++ b/src/MagicQuant/Helpers/PythonManager.cs @@ -0,0 +1,224 @@ +using System.Diagnostics; +using System.IO.Compression; +using System.Runtime.InteropServices; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Helpers; + +public class PythonManager +{ + private readonly string _basePath; + private readonly string _envPath; + + public PythonManager(string basePath) + { + _basePath = basePath; + _envPath = Path.Combine(basePath, MagicConstants.EnvName); + } + + public async Task GetInstalledVersionAsync(string packageName) + { + // NOTE: + // - Empty stdout is treated as NOT installed + // - stderr is captured + // - Python errors fail fast instead of lying + + string script = + $"import importlib.metadata, sys\n" + + $"try:\n" + + $" print(importlib.metadata.version('{packageName}'))\n" + + $"except Exception:\n" + + $" print('NONE')\n"; + + var psi = new ProcessStartInfo(GetPythonExecutable()); + psi.ArgumentList.Add("-c"); + psi.ArgumentList.Add(script); + var result = await new MagicQuant.Runtime.ProcessRunner().RunAsync(psi); + if (!result.Success) + throw new InvalidOperationException($"Python package check failed for '{packageName}'.\n{result.StdErr}"); + string stdout = result.StdOut; + + string version = stdout.Trim(); + + // CRITICAL FIX: + // Empty output MUST be treated as not installed + if (string.IsNullOrEmpty(version) || version == "NONE") + return null; + + return version; + } + + + public string GetPythonExecutable() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + return Path.Combine(_envPath, "python.exe"); + + return Path.Combine(_envPath, "bin", "python"); + } + + public async Task SetupEnvironmentAsync() + { + AnsiConsole.MarkupLine("[cyan]Configuring Python Environment...[/]"); + + if (CheckSuccessMarker()) + { + AnsiConsole.MarkupLine("[green]✔ Python Environment is ready.[/]"); + return; + } + + // Clean slate if corrupt + if (Directory.Exists(_envPath)) Directory.Delete(_envPath, true); + Directory.CreateDirectory(_envPath); + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + await SetupWindowsEmbedAsync(); + } + else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + await SetupLinuxVenvAsync(); + } + + // Install Pip Runner logic + await SetupPipRunnerAsync(); + + WriteSuccessMarker(); + } + + private async Task SetupWindowsEmbedAsync() + { + string zipPath = Path.Combine(_basePath, MagicConstants.WinPythonZip); + + // Download + if (!File.Exists(zipPath)) + { + using var client = new HttpClient(); + AnsiConsole.MarkupLine($"Downloading Python Embeddable from [blue]{MagicConstants.WinPythonUrl}[/]"); + var data = await client.GetByteArrayAsync(MagicConstants.WinPythonUrl); + await File.WriteAllBytesAsync(zipPath, data); + } + + // Extract + AnsiConsole.MarkupLine("Extracting Python..."); + ZipFile.ExtractToDirectory(zipPath, _envPath); + + // Cleanup Zip + File.Delete(zipPath); + + // Modify .pth file to allow importing site-packages (Crucial for pip) + string? pthFile = Directory.GetFiles(_envPath, "*._pth").FirstOrDefault(); + if (pthFile != null) + { + var lines = await File.ReadAllLinesAsync(pthFile); + var newLines = lines.Select(l => l.Trim() == "#import site" ? "import site" : l).ToList(); + await File.WriteAllLinesAsync(pthFile, newLines); + } + } + + private async Task SetupLinuxVenvAsync() + { + AnsiConsole.MarkupLine("Creating venv..."); + // FIX 1: Added _basePath as working dir, and null for env vars + await RunShellCommand("python3", $"-m venv \"{_envPath}\"", _basePath, null); + } + + private async Task SetupPipRunnerAsync() + { + string source = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Helpers", "pip_runner.py"); + string dest = Path.Combine(_envPath, "pip_runner.py"); + + if (File.Exists(source)) + { + File.Copy(source, dest, true); + AnsiConsole.MarkupLine("Copied pip_runner.py."); + } + else + { + AnsiConsole.MarkupLine("[yellow]Warning: pip_runner.py not found in Helpers.[/]"); + } + + string python = GetPythonExecutable(); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + await RunShellCommand("cmd.exe", $"/c \"{python}\" pip_runner.py install --upgrade pip setuptools wheel", + _envPath, null); + } + else + { + await RunShellCommand(python, "-m pip install --upgrade pip setuptools wheel", _basePath, null); + } + } + + public async Task RunPipAsync(string pipArgs, Dictionary? envVars = null) + { + string python = GetPythonExecutable(); + string exe, finalArgs; + + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + exe = "cmd.exe"; + finalArgs = $"/c \"{python}\" pip_runner.py {pipArgs}"; + await RunShellCommand(exe, finalArgs, _envPath, envVars); + } + else + { + exe = python; + finalArgs = $"-m pip {pipArgs}"; + await RunShellCommand(exe, finalArgs, _envPath, envVars); + } + } + + public Task RunPipInstallAsync(string installArgs, Dictionary? envVars = null) + => RunPipAsync($"install {installArgs}", envVars); + + + private bool CheckSuccessMarker() => File.Exists(Path.Combine(_envPath, MagicConstants.SuccessJson)); + + private void WriteSuccessMarker() => + File.WriteAllText(Path.Combine(_envPath, MagicConstants.SuccessJson), "{\"status\":\"success\"}"); + + public Task RunPythonScriptAsync(string scriptPath, string args = "", Dictionary? envVars = null) + { + return RunShellCommand(GetPythonExecutable(), $"\"{scriptPath}\" {args}", _envPath, envVars); + } + + public async Task RunPythonScriptAsync(string scriptPath, IReadOnlyList args, + Dictionary? envVars = null, CancellationToken ct = default) + { + var start = new MagicQuant.Runtime.NativeCommand(GetPythonExecutable(), [scriptPath, .. args]).CreateStartInfo(envVars); + start.WorkingDirectory = _envPath; + var result = await new MagicQuant.Runtime.ProcessRunner().RunAsync(start, + onLine: (line, _) => AnsiConsole.WriteLine(line), ct: ct); + if (!result.Success) throw new InvalidOperationException($"Python script '{scriptPath}' failed (exit {result.ExitCode}). {result.StdErr}"); + } + + // Legacy string arguments are retained here; ProcessRunner owns native lifetime. + private async Task RunShellCommand(string exe, string args, string workingDir, + Dictionary? envVars = null) + { + var psi = new ProcessStartInfo + { + FileName = exe, + Arguments = args, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true + }; + + if (!string.IsNullOrEmpty(workingDir)) + psi.WorkingDirectory = workingDir; + + if (envVars != null) + foreach (var kvp in envVars) + psi.Environment[kvp.Key] = kvp.Value; + + var result = await new MagicQuant.Runtime.ProcessRunner().RunAsync(psi, + onLine: (line, error) => AnsiConsole.MarkupLine($"[{(error ? "red" : "grey")}]{Markup.Escape(line)}[/]")); + if (!result.Success) + throw new InvalidOperationException($"Command failed (exit {result.ExitCode}): {exe} {args}"); + } +} diff --git a/src/MagicQuant/Helpers/RuntimeSearchSpace.cs b/src/MagicQuant/Helpers/RuntimeSearchSpace.cs new file mode 100644 index 0000000..0206367 --- /dev/null +++ b/src/MagicQuant/Helpers/RuntimeSearchSpace.cs @@ -0,0 +1,309 @@ +using MQ.DB; +using MQ.DB.Models; + +namespace MagicQuant.Helpers; + +public sealed class RuntimeLearnedBaselineBanInfo +{ + public BaselineQuants Candidate { get; init; } = default!; + public IReadOnlyList ExpectedTensorWeightSchemeIds { get; init; } = Array.Empty(); + public IReadOnlyList MatchedTensorWeightSchemeIds { get; init; } = Array.Empty(); + public IReadOnlyList MissingTensorWeightSchemeIds { get; init; } = Array.Empty(); + public string Note { get; init; } = string.Empty; +} + +public static class RuntimeSearchSpace +{ + private static readonly Dictionary> ExplicitCandidateBansByGroup = new(); + private static readonly Dictionary> ExplicitCandidateBanReasonsByGroup = new(); + private static readonly Dictionary> LearnedPrunesByGroupAndCandidate = new(); + private static readonly HashSet DisabledCombinationBaselineIds = new(); + private static readonly HashSet Bf16SuppressedTensorChoiceGroupIds = new(); + private static bool _imatrixAvailable; + + public static bool AllowHighPrecisionHybrids { get; set; } + + public static void ResetForNewModel() + { + ExplicitCandidateBansByGroup.Clear(); + ExplicitCandidateBanReasonsByGroup.Clear(); + LearnedPrunesByGroupAndCandidate.Clear(); + DisabledCombinationBaselineIds.Clear(); + Bf16SuppressedTensorChoiceGroupIds.Clear(); + _imatrixAvailable = false; + AllowHighPrecisionHybrids = false; + } + + public static void SetImatrixAvailability(bool available) => _imatrixAvailable = available; + + public static void ResetForCompatibilityPass() + { + ExplicitCandidateBansByGroup.Clear(); + ExplicitCandidateBanReasonsByGroup.Clear(); + LearnedPrunesByGroupAndCandidate.Clear(); + DisabledCombinationBaselineIds.Clear(); + Bf16SuppressedTensorChoiceGroupIds.Clear(); + } + + public static bool HasUsableImatrix() => _imatrixAvailable; + + public static void BanCombinationCandidateForGroup( + TensorGroup group, + BaselineQuants candidate, + string phase = "Unknown", + string reason = "unspecified") + { + int before = GetAllowedRealExplicitCombinationCandidatesForGroup(group).Count; + if (!ExplicitCandidateBansByGroup.TryGetValue(group.UniqueId, out var set)) + { + set = new HashSet(); + ExplicitCandidateBansByGroup[group.UniqueId] = set; + } + + set.Add(candidate.UniqueId); + if (!ExplicitCandidateBanReasonsByGroup.TryGetValue(group.UniqueId, out var reasonMap)) + { + reasonMap = new Dictionary(); + ExplicitCandidateBanReasonsByGroup[group.UniqueId] = reasonMap; + } + reasonMap[candidate.UniqueId] = reason; + int after = GetAllowedRealExplicitCombinationCandidatesForGroup(group).Count; + MagicQuantDiagnostics.LogRuntimeMutation(phase, group, candidate, reason, before, after); + } + + public static bool UnbanCombinationCandidateForGroup( + TensorGroup group, + BaselineQuants candidate, + string phase = "Unknown", + string reason = "unspecified") + { + int before = GetAllowedRealExplicitCombinationCandidatesForGroup(group).Count; + + if (!ExplicitCandidateBansByGroup.TryGetValue(group.UniqueId, out var set) || + !set.Remove(candidate.UniqueId)) + { + return false; + } + + if (set.Count == 0) + ExplicitCandidateBansByGroup.Remove(group.UniqueId); + + if (ExplicitCandidateBanReasonsByGroup.TryGetValue(group.UniqueId, out var reasonMap)) + { + reasonMap.Remove(candidate.UniqueId); + if (reasonMap.Count == 0) + ExplicitCandidateBanReasonsByGroup.Remove(group.UniqueId); + } + + int after = GetAllowedRealExplicitCombinationCandidatesForGroup(group).Count; + MagicQuantDiagnostics.LogRuntimeMutation(phase, group, candidate, $"restored: {reason}", before, after); + return true; + } + + public static void BanCombinationCandidateForGroupDueToLearnedSchemeMismatch( + TensorGroup group, + BaselineQuants candidate, + IReadOnlyCollection expectedTensorWeightSchemeIds, + IReadOnlyCollection matchedTensorWeightSchemeIds, + string note) + { + BanCombinationCandidateForGroup(group, candidate, phase: "LearnedPrune", reason: note); + + if (!LearnedPrunesByGroupAndCandidate.TryGetValue(group.UniqueId, out var byCandidate)) + { + byCandidate = new Dictionary(); + LearnedPrunesByGroupAndCandidate[group.UniqueId] = byCandidate; + } + + var expected = expectedTensorWeightSchemeIds + .Distinct() + .OrderBy(x => x) + .ToList(); + var matched = matchedTensorWeightSchemeIds + .Distinct() + .OrderBy(x => x) + .ToList(); + var missing = expected.Except(matched).OrderBy(x => x).ToList(); + + byCandidate[candidate.UniqueId] = new RuntimeLearnedBaselineBanInfo + { + Candidate = candidate, + ExpectedTensorWeightSchemeIds = expected, + MatchedTensorWeightSchemeIds = matched, + MissingTensorWeightSchemeIds = missing, + Note = note + }; + } + + public static void ClearLearnedBaselinePruneBookkeeping() => LearnedPrunesByGroupAndCandidate.Clear(); + + public static void ClearLearnedBaselinePruneForGroupCandidate(TensorGroup group, BaselineQuants candidate) + { + if (!LearnedPrunesByGroupAndCandidate.TryGetValue(group.UniqueId, out var byCandidate)) + return; + + byCandidate.Remove(candidate.UniqueId); + if (byCandidate.Count == 0) + LearnedPrunesByGroupAndCandidate.Remove(group.UniqueId); + } + + public static void BanAllExplicitCombinationCandidatesForGroup(TensorGroup group, string phase = "Unknown", string reason = "ban-all") + { + foreach (var candidate in GetRealExplicitCombinationCandidatesForGroup(group)) + BanCombinationCandidateForGroup(group, candidate, phase, reason); + } + + public static IReadOnlyList GetRuntimeExplicitCandidateBansForGroup(TensorGroup group) + { + if (!ExplicitCandidateBansByGroup.TryGetValue(group.UniqueId, out var set)) + return Array.Empty(); + + return BaselineQuants.GetAllRecognizedBaselines() + .Where(x => set.Contains(x.UniqueId)) + .OrderBy(x => x.UniqueId) + .ToList(); + } + + public static IReadOnlyDictionary GetRuntimeExplicitCandidateBanReasonsForGroup(TensorGroup group) + { + if (!ExplicitCandidateBanReasonsByGroup.TryGetValue(group.UniqueId, out var reasons)) + return new Dictionary(); + return reasons; + } + + public static bool IsCombinationCandidateRuntimeBannedForGroup(TensorGroup group, BaselineQuants candidate) + => ExplicitCandidateBansByGroup.TryGetValue(group.UniqueId, out var set) && set.Contains(candidate.UniqueId); + + public static IReadOnlyList GetRealExplicitCombinationCandidatesForGroup(TensorGroup group) + { + return BaselineQuants.GetGroupCombinationCandidates(_imatrixAvailable, allowHighPrecisionHybrids: false) + .Where(x => !x.BannedGroupIds.Contains(group.UniqueId)) + .OrderBy(x => x.ExplicitCandidateSortOrder) + .ThenBy(x => x.UniqueId) + .ToList(); + } + + public static IReadOnlyList GetAllowedRealExplicitCombinationCandidatesForGroup(TensorGroup group) + { + return GetRealExplicitCombinationCandidatesForGroup(group) + .Where(x => !IsCombinationCandidateRuntimeBannedForGroup(group, x)) + .ToList(); + } + + public static bool HasAnyExplicitCombinationCandidateAllowed(TensorGroup group) + => GetAllowedRealExplicitCombinationCandidatesForGroup(group).Count > 0; + + public static bool IsGroupExplicitCandidateBanned(TensorGroup group) => !HasAnyExplicitCombinationCandidateAllowed(group); + + public static IReadOnlyList GetGroupsWithExplicitQuantBanned() + => TReg.All.Where(IsGroupExplicitCandidateBanned).OrderBy(x => x.UniqueId).ToList(); + + public static bool HasLearnedBaselineMissingPrunesForGroup(TensorGroup group) + => LearnedPrunesByGroupAndCandidate.TryGetValue(group.UniqueId, out var byCandidate) && byCandidate.Count > 0; + + public static IReadOnlyList GetGroupsWithLearnedBaselineMissingPrunes() + => TReg.All.Where(HasLearnedBaselineMissingPrunesForGroup).OrderBy(x => x.UniqueId).ToList(); + + public static IReadOnlyList GetLearnedBaselineMissingPrunedCandidatesForGroup(TensorGroup group) + { + if (!LearnedPrunesByGroupAndCandidate.TryGetValue(group.UniqueId, out var byCandidate)) + return Array.Empty(); + + return byCandidate + .OrderBy(x => x.Key) + .Select(x => x.Value) + .ToList(); + } + + public static void SuppressBf16TensorChoice(TensorGroup group, string phase = "Unknown", string reason = "suppressed") + { + Bf16SuppressedTensorChoiceGroupIds.Add(group.UniqueId); + if (MagicQuantDiagnostics.ShouldLogGroup(group)) + MagicQuantDiagnostics.Log("runtime-ban", $"phase={phase} group={group.Name}(id={group.UniqueId}) bf16Suppressed=true reason=\"{reason}\""); + } + + public static bool IsBf16TensorChoiceSuppressed(TensorGroup group) + => Bf16SuppressedTensorChoiceGroupIds.Contains(group.UniqueId) && HasAnyExplicitCombinationCandidateAllowed(group); + + public static IReadOnlyList GetBf16SuppressedGroups() + => TReg.All.Where(IsBf16TensorChoiceSuppressed).OrderBy(x => x.UniqueId).ToList(); + + public static string GetDisplayStateForGroup(TensorGroup group) + { + if (Cache.UnusedTensorGroups.Any(x => x.UniqueId == group.UniqueId)) + return "unused->NULL"; + + if (IsGroupExplicitCandidateBanned(group)) + return "explicit-banned->Q8-fallback"; + + if (IsBf16TensorChoiceSuppressed(group)) + return "BF16-suppressed"; + + if (HasLearnedBaselineMissingPrunesForGroup(group)) + return "learned-pruned"; + + return "variable"; + } + + public static (bool ExplicitAllowed, bool Bf16Allowed) GetFinalAllowedQuantFamiliesForGroup(TensorGroup group) + { + bool explicitAllowed = HasAnyExplicitCombinationCandidateAllowed(group); + bool bf16Allowed = !IsBf16TensorChoiceSuppressed(group) || !explicitAllowed; + return (explicitAllowed, bf16Allowed); + } + + public static IReadOnlyList GetActiveCombinationBaselines() + { + var active = BaselineQuants.GetCombinationCarrierBaselines(_imatrixAvailable) + .Where(x => !DisabledCombinationBaselineIds.Contains(x.UniqueId)) + .OrderBy(x => x.UniqueId) + .ToList(); + + if (active.Count == 0) + return new[] { BaselineQuants.Q8_0 }; + + return active; + } + + public static bool DisableCombinationBaseline(BaselineQuants baseline, bool allowDisablingLast = false, string phase = "Unknown", string reason = "disabled") + { + if (!baseline.IsCombinationCarrierCandidate || DisabledCombinationBaselineIds.Contains(baseline.UniqueId)) + return false; + + int currentlyActive = GetActiveCombinationBaselines().Count; + if (!allowDisablingLast && currentlyActive <= 1) + return false; + + DisabledCombinationBaselineIds.Add(baseline.UniqueId); + MagicQuantDiagnostics.Log("runtime-ban", $"phase={phase} baseline={baseline.Names[0]}(id={baseline.UniqueId}) reason=\"{reason}\""); + return true; + } + + public static bool IsCombinationBaselineDisabled(BaselineQuants baseline) + => DisabledCombinationBaselineIds.Contains(baseline.UniqueId); + + [Obsolete("Use BanCombinationCandidateForGroup.")] + public static void BanSchemeForGroup(TensorGroup group, TensorWeightScheme scheme) + => BanCombinationCandidateForGroup(group, BaselineQuants.FromTensorSchemeId(scheme.UniqueId)); + + [Obsolete("Use BanCombinationCandidateForGroupDueToLearnedSchemeMismatch.")] + public static void BanSchemeForGroupByLearnedBaselineAbsence(TensorGroup group, TensorWeightScheme scheme, BaselineQuants sourceBaseline) + => BanCombinationCandidateForGroupDueToLearnedSchemeMismatch( + group, + BaselineQuants.FromTensorSchemeId(scheme.UniqueId), + expectedTensorWeightSchemeIds: [scheme.UniqueId], + matchedTensorWeightSchemeIds: Array.Empty(), + note: $"Legacy scheme-ban shim invoked for source baseline '{sourceBaseline.Names[0]}'."); + + [Obsolete("Use BanAllExplicitCombinationCandidatesForGroup.")] + public static void BanAllExplicitTensorSchemesForGroup(TensorGroup group) + => BanAllExplicitCombinationCandidatesForGroup(group); + + [Obsolete("Use IsCombinationCandidateRuntimeBannedForGroup.")] + public static bool IsSchemeRuntimeBannedForGroup(TensorGroup group, TensorWeightScheme scheme) + => IsCombinationCandidateRuntimeBannedForGroup(group, BaselineQuants.FromTensorSchemeId(scheme.UniqueId)); + + [Obsolete("Use IsGroupExplicitCandidateBanned.")] + public static bool IsGroupExplicitQuantBanned(TensorGroup group) + => IsGroupExplicitCandidateBanned(group); +} \ No newline at end of file diff --git a/src/MagicQuant/Helpers/SearchSpaceDebugPrinter.cs b/src/MagicQuant/Helpers/SearchSpaceDebugPrinter.cs new file mode 100644 index 0000000..e900bed --- /dev/null +++ b/src/MagicQuant/Helpers/SearchSpaceDebugPrinter.cs @@ -0,0 +1,182 @@ +using System; +using System.Linq; +using System.Collections.Generic; +using System.Numerics; +using MagicQuant.Services; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Helpers; + +public static class SearchSpaceDebugPrinter +{ + public static void PrintCurrentSearchSpace(string title = "Current Runtime Search Space") + { + AnsiConsole.Write(new Rule($"[yellow]{Markup.Escape(title)}[/]") { Justification = Justify.Left }); + + var activeBaselines = RuntimeSearchSpace.GetActiveCombinationBaselines().ToList(); + var disabledBaselines = BaselineQuants.All + .Where(x => x.IsCombinationCarrierCandidate) + .Where(x => RuntimeSearchSpace.IsCombinationBaselineDisabled(x)) + .OrderBy(x => x.UniqueId) + .ToList(); + + AnsiConsole.MarkupLine($"[green]Active combo baselines:[/] {activeBaselines.Count}"); + foreach (var baseline in activeBaselines) + AnsiConsole.MarkupLine($" [cyan]- {string.Join("/", baseline.Names)}[/] (Id={baseline.UniqueId})"); + + if (disabledBaselines.Count > 0) + { + AnsiConsole.MarkupLine($"[yellow]Disabled combo baselines:[/] {disabledBaselines.Count}"); + foreach (var baseline in disabledBaselines) + AnsiConsole.MarkupLine($" [grey]- {string.Join("/", baseline.Names)}[/] (Id={baseline.UniqueId})"); + } + + var explicitBannedGroups = RuntimeSearchSpace.GetGroupsWithExplicitQuantBanned(); + if (explicitBannedGroups.Count > 0) + { + AnsiConsole.MarkupLine($"[yellow]Explicit-quant-banned groups:[/] {explicitBannedGroups.Count}"); + foreach (var group in explicitBannedGroups) + AnsiConsole.MarkupLine($" [yellow]- {group.Name}[/]"); + } + + var bf16SuppressedGroups = RuntimeSearchSpace.GetBf16SuppressedGroups(); + if (bf16SuppressedGroups.Count > 0) + { + AnsiConsole.MarkupLine($"[yellow]BF16 tensor-choice suppressed groups:[/] {bf16SuppressedGroups.Count}"); + foreach (var group in bf16SuppressedGroups) + AnsiConsole.MarkupLine($" [yellow]- {group.Name}[/]"); + } + + var learnedPrunedGroups = RuntimeSearchSpace.GetGroupsWithLearnedBaselineMissingPrunes(); + if (learnedPrunedGroups.Count > 0) + { + AnsiConsole.MarkupLine($"[yellow]Learned-baseline-pruned groups:[/] {learnedPrunedGroups.Count}"); + + foreach (var group in learnedPrunedGroups) + { + var learned = RuntimeSearchSpace.GetLearnedBaselineMissingPrunedCandidatesForGroup(group); + + var parts = learned.Select(x => + $"{x.Candidate.Names[0]} (expected={FormatSchemeIds(x.ExpectedTensorWeightSchemeIds)}, matched={FormatSchemeIds(x.MatchedTensorWeightSchemeIds)}, missing={FormatSchemeIds(x.MissingTensorWeightSchemeIds)})"); + + AnsiConsole.MarkupLine( + $" [yellow]- {Markup.Escape(group.Name)}[/] :: [grey]{Markup.Escape(string.Join(", ", parts))}[/]"); + } + } + + foreach (var baseline in activeBaselines) + { + AnsiConsole.Write(new Rule($"[blue]Base: {Markup.Escape(string.Join("/", baseline.Names))}[/]") + { + Justification = Justify.Left + }); + + var allowed = ComboLogic.GetAllowedCandidateIdsPerGroup(baseline); + BigInteger baseCount = BigInteger.One; + + for (int i = 0; i < TReg.All.Length; i++) + { + var group = TReg.All.OrderBy(x => x.UniqueId).ElementAt(i); + var ids = allowed[i]; + baseCount *= ids.Length; + + var names = ids.Select(id => + { + if (BaselineQuants.IsNullTensorConfigGroupSlot(id)) + return "NULL"; + + return BaselineQuants.DecodeTensorConfigGroupSlotToBaseline(id).Names[0]; + }).ToList(); + + string state = RuntimeSearchSpace.GetDisplayStateForGroup(group); + + AnsiConsole.MarkupLine( + $" [cyan]{Markup.Escape(group.Name)}[/] => [green]{ids.Length}[/] choice(s) " + + $"[grey][[{Markup.Escape(state)}]][/] :: {Markup.Escape(string.Join(", ", names))}"); + + if (MagicQuantDiagnostics.ShouldLogGroup(group)) + { + var raw = RuntimeSearchSpace.GetRealExplicitCombinationCandidatesForGroup(group); + var runtimeBanned = raw.Where(x => RuntimeSearchSpace.IsCombinationCandidateRuntimeBannedForGroup(group, x)).ToList(); + var staticBanned = BaselineQuants.GetGroupCombinationCandidates(RuntimeSearchSpace.HasUsableImatrix(), false) + .Where(x => x.BannedGroupIds.Contains(group.UniqueId)) + .OrderBy(x => x.UniqueId) + .ToList(); + var allowedReal = RuntimeSearchSpace.GetAllowedRealExplicitCombinationCandidatesForGroup(group); + var reasonMap = RuntimeSearchSpace.GetRuntimeExplicitCandidateBanReasonsForGroup(group); + var why = ids.Length == 1 ? "single final choice after bans/suppression" : "multi-choice"; + MagicQuantDiagnostics.Log("search-space", + $"group={group.Name}(id={group.UniqueId}) unused={Cache.UnusedTensorGroups.Any(x=>x.UniqueId==group.UniqueId)} explicitBanned={RuntimeSearchSpace.IsGroupExplicitCandidateBanned(group)} bf16Suppressed={RuntimeSearchSpace.IsBf16TensorChoiceSuppressed(group)} rawExplicit={raw.Count} staticBanned={staticBanned.Count} runtimeBanned={runtimeBanned.Count} allowedExplicit={allowedReal.Count} finalChoices={string.Join(",", names)} why={why}"); + if (runtimeBanned.Count > 0) + MagicQuantDiagnostics.Log("search-space", $"group={group.Name} runtimeBanned: {string.Join(", ", runtimeBanned.Select(x => $"{x.Names[0]}(id={x.UniqueId})"))}"); + if (reasonMap.Count > 0) + { + var grouped = runtimeBanned + .GroupBy(x => reasonMap.TryGetValue(x.UniqueId, out var r) ? r : "unspecified") + .Select(g => $"{g.Key}: {string.Join(", ", g.Select(x => x.Names[0]))}"); + MagicQuantDiagnostics.Log("search-space", $"group={group.Name} restrictionReasons={string.Join(" | ", grouped)}"); + } + if (staticBanned.Count > 0) + MagicQuantDiagnostics.Log("search-space", $"group={group.Name} staticBanned: {string.Join(", ", staticBanned.Select(x => $"{x.Names[0]}(id={x.UniqueId})"))}"); + } + } + + AnsiConsole.MarkupLine($" [bold green]Base total:[/] {baseCount:N0}"); + } + + AnsiConsole.MarkupLine($"[bold yellow]Grand total:[/] {ComboCounter.CountAll():N0}"); + } + public static void PrintIsolationGroupDecisions( + string title, + IEnumerable decisions, + string winningLabel = "Winning candidate") + { + var ordered = decisions + .OrderBy(x => x.GroupName, StringComparer.Ordinal) + .ToList(); + + if (ordered.Count == 0) + return; + + AnsiConsole.Write(new Rule($"[yellow]{Markup.Escape(title)}[/]") { Justification = Justify.Left }); + + foreach (var gd in ordered) + { + AnsiConsole.Write( + new Rule($"[yellow]Isolation Group: {Markup.Escape(gd.GroupName)}[/]") + { + Justification = Justify.Left + }); + + AnsiConsole.MarkupLine($"[green]Best savings:[/] {gd.BestReductionRatio:P2}"); + AnsiConsole.MarkupLine($"[green]{Markup.Escape(winningLabel)}:[/] {Markup.Escape(gd.WinningCandidate ?? "n/a")}"); + AnsiConsole.MarkupLine($"[green]Explicit quant banned:[/] {(gd.ExplicitQuantBanned ? "[red]yes[/]" : "[green]no[/]")}"); + AnsiConsole.MarkupLine($"[green]BF16 suppressed:[/] {(gd.Bf16Suppressed ? "[yellow]yes[/]" : "[green]no[/]")}"); + + foreach (var line in gd.Candidates) + AnsiConsole.MarkupLine($" [grey]- {Markup.Escape(line)}[/]"); + } + } + + private static string FormatSchemeIds(IEnumerable schemeIds) + { + var ids = schemeIds + .Distinct() + .OrderBy(x => x) + .ToList(); + + if (ids.Count == 0) + return ""; + + var parts = ids.Select(id => + { + var scheme = TensorWeightScheme.All.FirstOrDefault(x => x.UniqueId == id); + return scheme?.Names[0] ?? id.ToString(); + }); + + return string.Join("/", parts); + } + +} diff --git a/src/MagicQuant/Helpers/TensorConfigGenerator.cs b/src/MagicQuant/Helpers/TensorConfigGenerator.cs new file mode 100644 index 0000000..1441a61 --- /dev/null +++ b/src/MagicQuant/Helpers/TensorConfigGenerator.cs @@ -0,0 +1,462 @@ +using System.Collections.Concurrent; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Helpers; + +public static class TensorConfigGenerator +{ + public static RequiredSampleGenerationResult GenerateInitialIsolationSamplePlan( + List? missingTensorGroups = null) + { + if (missingTensorGroups != null && !missingTensorGroups.Any()) + missingTensorGroups = null; + + var missingIds = missingTensorGroups?.Select(x => x.UniqueId).ToHashSet() ?? new HashSet(); + var activeGroups = TReg.All + .Where(x => !missingIds.Contains(x.UniqueId)) + .OrderBy(x => x.UniqueId) + .ToList(); + + var result = new RequiredSampleGenerationResult(); + var nativeExactScheme = TensorWeightScheme.GetCurrentNativePrecisionScheme(); + + var alreadyAddedPureBaselineIds = new HashSet(); + + void AddPureBaselinePlan(BaselineQuants baseline) + { + if (!alreadyAddedPureBaselineIds.Add(baseline.UniqueId)) + return; + + result.Plans.Add(new RequiredSamplePlan + { + Kind = RequiredSampleKind.PureBaseline, + Key = $"pure:{baseline.UniqueId}", + Description = $"Pure baseline build for {string.Join("/", baseline.Names)}", + Quant = HybridQuant.CreatePureBaseline(baseline), + TestedBaselineId = baseline.UniqueId, + TestedBaselineCanonicalKey = baseline.CanonicalKey + }); + + result.PureBaselineCount++; + } + + foreach (var baseline in BaselineQuants.GetLearningBaselines(RuntimeSearchSpace.HasUsableImatrix())) + AddPureBaselinePlan(baseline); + + // Q8 remains a required system anchor even when the user disables standard baselines. + AddPureBaselinePlan(BaselineQuants.Q8_0); + + foreach (var baseline in RuntimeSearchSpace.GetActiveCombinationBaselines()) + { + result.Plans.Add(new RequiredSamplePlan + { + Kind = RequiredSampleKind.BaseOnlyIsolation, + Key = $"baseonly:{baseline.UniqueId}", + Description = + $"Base-only isolation for {string.Join("/", baseline.Names)} with all active groups forced native.", + Quant = HybridQuant.CreateExactBlanket( + baseQuant: baseline, + groups: activeGroups, + exactScheme: nativeExactScheme), + TestedBaselineId = baseline.UniqueId, + TestedBaselineCanonicalKey = baseline.CanonicalKey + }); + + result.BaseOnlyIsolationCount++; + } + + var carrier = BaselineQuants.Q8_0; + + result.Plans.Add(new RequiredSamplePlan + { + Kind = RequiredSampleKind.BaseOnlyIsolation, + Key = $"carrier-baseonly:{carrier.UniqueId}", + Description = "Carrier base-only isolation on Q8 with all active groups forced native.", + Quant = HybridQuant.CreateExactBlanket( + baseQuant: carrier, + groups: activeGroups, + exactScheme: nativeExactScheme), + TestedBaselineId = carrier.UniqueId, + TestedBaselineCanonicalKey = carrier.CanonicalKey + }); + + result.BaseOnlyIsolationCount++; + + foreach (var group in activeGroups) + { + var smallest = GetSmallestAllowedProbeCandidateForGroup(group); + if (smallest == null) + continue; + + var quant = HybridQuant.CreateExactBlanket( + baseQuant: carrier, + groups: activeGroups, + exactScheme: nativeExactScheme); + + quant.SetLearnedCandidateOverride(group, smallest); + + result.Plans.Add(new RequiredSamplePlan + { + Kind = RequiredSampleKind.GroupIsolationProbe, + Key = $"probe:{carrier.UniqueId}:{group.UniqueId}:{smallest.UniqueId}", + Description = $"Smallest-first probe for group '{group.Name}' using '{smallest.Names[0]}'.", + Quant = quant, + TargetGroupId = group.UniqueId, + TestedCandidateId = smallest.UniqueId, + TestedCandidateCanonicalKey = smallest.CanonicalKey, + TestedBaselineId = carrier.UniqueId, + TestedBaselineCanonicalKey = carrier.CanonicalKey, + IsSmallestProbe = true + }); + + result.GroupIsolationCount++; + } + + AnsiConsole.MarkupLine($"[bold green]Pure baselines required:[/] {result.PureBaselineCount:N0}"); + AnsiConsole.MarkupLine( + $"[bold green]Base-only isolation samples required:[/] {result.BaseOnlyIsolationCount:N0}"); + AnsiConsole.MarkupLine( + $"[bold green]Smallest-probe isolation samples required:[/] {result.GroupIsolationCount:N0}"); + AnsiConsole.MarkupLine($"[bold green]Total initial startup samples:[/] {result.TotalCount:N0}"); + EmitSamplePlanDiagnostics("initial", result.Plans); + + return result; + } + + public static RequiredSampleGenerationResult GenerateContinuationIsolationSamplePlan( + IEnumerable groupIdsToContinue, + List? missingTensorGroups = null) + { + if (missingTensorGroups != null && !missingTensorGroups.Any()) + missingTensorGroups = null; + + var continueIds = groupIdsToContinue.Distinct().ToHashSet(); + var missingIds = missingTensorGroups?.Select(x => x.UniqueId).ToHashSet() ?? new HashSet(); + + var activeGroups = TReg.All + .Where(x => continueIds.Contains(x.UniqueId)) + .Where(x => !missingIds.Contains(x.UniqueId)) + .OrderBy(x => x.UniqueId) + .ToList(); + + var result = BuildIsolationCoverageContinuationPlan(activeGroups, missingIds); + + AnsiConsole.MarkupLine( + $"[bold green]Continuation isolation samples required:[/] {result.GroupIsolationCount:N0}"); + EmitSamplePlanDiagnostics("continuation", result.Plans); + return result; + } + + /// + /// Builds archival-only isolation coverage for any continuation-style samples that were not part + /// of the live startup+continuation pruning plan. This is intentionally kept separate from the + /// current run's pruning inputs so search-space behavior stays unchanged while the database still + /// gains full isolated-sample coverage for future prediction/reporting flows. + /// + public static RequiredSampleGenerationResult GenerateArchivalIsolationCoverageSamplePlan( + IEnumerable? groupIdsToArchive = null, + IEnumerable? existingPlanKeys = null, + List? missingTensorGroups = null) + { + if (missingTensorGroups != null && !missingTensorGroups.Any()) + missingTensorGroups = null; + + var missingIds = missingTensorGroups?.Select(x => x.UniqueId).ToHashSet() ?? new HashSet(); + var archiveIds = groupIdsToArchive? + .Distinct() + .ToHashSet() + ?? new HashSet(); + + var existingKeys = existingPlanKeys? + .Where(x => !string.IsNullOrWhiteSpace(x)) + .ToHashSet(StringComparer.Ordinal) + ?? new HashSet(StringComparer.Ordinal); + + var activeGroups = TReg.All + .Where(x => !missingIds.Contains(x.UniqueId)) + .Where(x => archiveIds.Count == 0 || archiveIds.Contains(x.UniqueId)) + .OrderBy(x => x.UniqueId) + .ToList(); + + var result = BuildIsolationCoverageContinuationPlan(activeGroups, missingIds); + + if (existingKeys.Count > 0) + { + result.Plans = result.Plans + .Where(x => !existingKeys.Contains(x.Key)) + .ToList(); + } + + result.GroupIsolationCount = result.Plans.Count(x => + x.Kind == RequiredSampleKind.GroupIsolationProbe || + x.Kind == RequiredSampleKind.GroupIsolationContinuation); + + return result; + } + + private static RequiredSampleGenerationResult BuildIsolationCoverageContinuationPlan( + IReadOnlyCollection activeGroups, + HashSet missingIds) + { + var result = new RequiredSampleGenerationResult(); + if (activeGroups.Count == 0) + return result; + + var carrier = BaselineQuants.Q8_0; + var nativeExactScheme = TensorWeightScheme.GetCurrentNativePrecisionScheme(); + + var blanketGroups = TReg.All + .Where(x => !missingIds.Contains(x.UniqueId)) + .ToList(); + + // IMPORTANT: + // Isolation coverage is NOT the same thing as runtime search-space eligibility. + // + // Prediction/RankSafe can reference external/custom/virtual anchor baseline ids + // for any active tensor group. Therefore every candidate identity needs a logical + // isolation snapshot for every active group. + // + // Physical work is still protected by isolation dedupe. If moe_router + UD-Q5_K_XL + // materializes to the same tensor->quant map as moe_router + Q8_0/F32, the artifact + // benchmark may be cloned/reused. But the duplicate TensorConfig identity must still + // exist in SQLite so RankSafe can resolve it exactly. + var candidates = GetIsolationCoverageCandidates() + .ToList(); + + foreach (var group in activeGroups.OrderBy(x => x.UniqueId)) + { + var smallest = GetSmallestAllowedProbeCandidateForGroup(group); + + foreach (var candidate in candidates) + { + if (smallest != null && candidate.UniqueId == smallest.UniqueId) + continue; + + var quant = HybridQuant.CreateExactBlanket( + baseQuant: carrier, + groups: blanketGroups, + exactScheme: nativeExactScheme); + + quant.SetLearnedCandidateOverride(group, candidate); + + result.Plans.Add(new RequiredSamplePlan + { + Kind = RequiredSampleKind.GroupIsolationContinuation, + Key = $"cont:{carrier.UniqueId}:{group.UniqueId}:{candidate.UniqueId}", + Description = + $"Continuation isolation coverage for group '{group.Name}' using '{candidate.Names[0]}'.", + Quant = quant, + TargetGroupId = group.UniqueId, + TestedCandidateId = candidate.UniqueId, + TestedCandidateCanonicalKey = candidate.CanonicalKey, + TestedBaselineId = carrier.UniqueId, + TestedBaselineCanonicalKey = carrier.CanonicalKey + }); + + result.GroupIsolationCount++; + } + } + + return result; + } + + private static BaselineQuants? GetSmallestAllowedProbeCandidateForGroup(TensorGroup group) + { + return GetIsolationCoverageCandidates() + .Where(x => !x.BannedGroupIds.Contains(group.UniqueId)) + .FirstOrDefault(); + } + + private static IEnumerable GetIsolationCoverageCandidates() + { + var hasUsableImatrix = RuntimeSearchSpace.HasUsableImatrix(); + + // This is intentionally broad. It is the identity universe RankSafe / virtual + // anchors may need exact snapshots for, not the narrowed per-group search space. + return BaselineQuants + .GetGroupCombinationCandidatesSmallestFirst( + hasUsableImatrix, + allowHighPrecisionHybrids: RuntimeSearchSpace.AllowHighPrecisionHybrids) + .Where(x => !BaselineQuants.IsNativeExactAlias(x.UniqueId)) + .GroupBy(x => x.UniqueId) + .Select(g => g.First()) + .OrderBy(x => x.ExplicitCandidateSortOrder) + .ThenBy(x => x.UniqueId); + } + + private static void EmitSamplePlanDiagnostics(string phase, IReadOnlyCollection plans) + { + if (!MagicQuantDiagnostics.VerboseIsolationPruning) + return; + + var byGroup = plans.Where(x => x.TargetGroupId.HasValue).GroupBy(x => x.TargetGroupId!.Value); + foreach (var set in byGroup) + { + var group = TReg.All.First(x => x.UniqueId == set.Key); + if (!MagicQuantDiagnostics.ShouldLogGroup(group)) + continue; + var planned = set.Select(x => BaselineQuants.FromId(x.TestedCandidateId!.Value)).ToList(); + var smallest = GetSmallestAllowedProbeCandidateForGroup(group); + var allowed = RuntimeSearchSpace.GetAllowedRealExplicitCombinationCandidatesForGroup(group); + var raw = RuntimeSearchSpace.GetRealExplicitCombinationCandidatesForGroup(group); + var staticBanned = BaselineQuants + .GetGroupCombinationCandidates(RuntimeSearchSpace.HasUsableImatrix(), false) + .Where(x => x.BannedGroupIds.Contains(group.UniqueId)).ToList(); + var runtimeBanned = raw.Where(x => RuntimeSearchSpace.IsCombinationCandidateRuntimeBannedForGroup(group, x)) + .ToList(); + var notAllowed = planned.Where(x => allowed.All(a => a.UniqueId != x.UniqueId)).ToList(); + MagicQuantDiagnostics.Log("sample-plan", + $"phase={phase} group={group.Name}(id={group.UniqueId}) plannedCount={planned.Count} smallestProbe={(smallest == null ? "" : MagicQuantDiagnostics.CandidateLabel(smallest))}"); + MagicQuantDiagnostics.Log("sample-plan", + $"planned={string.Join(", ", planned.Select(MagicQuantDiagnostics.CandidateLabel))}"); + MagicQuantDiagnostics.Log("sample-plan", + $"allowedAtPlan={string.Join(", ", allowed.Select(MagicQuantDiagnostics.CandidateLabel))}"); + MagicQuantDiagnostics.Log("sample-plan", + $"staticBanned={string.Join(", ", staticBanned.Select(MagicQuantDiagnostics.CandidateLabel))}"); + MagicQuantDiagnostics.Log("sample-plan", + $"runtimeBanned={string.Join(", ", runtimeBanned.Select(MagicQuantDiagnostics.CandidateLabel))}"); + MagicQuantDiagnostics.Log("sample-plan", + $"plannedButNotAllowed={string.Join(", ", notAllowed.Select(MagicQuantDiagnostics.CandidateLabel))}"); + } + } + + public static List GenerateRequiredDataSampleCombos(List? missingTensorGroups = null) + { + return GenerateInitialIsolationSamplePlan(missingTensorGroups) + .Plans + .Select(x => x.Quant) + .ToList(); + } + + public static IEnumerable> GenerateTensorConfigBatches( + BaselineQuants baseQuant, + int batchSize = 10_000_000, + CancellationToken ct = default) + { + if (batchSize <= 0) + throw new ArgumentOutOfRangeException(nameof(batchSize)); + + if (TReg.All.IsDefault) + throw new InvalidOperationException("TensorRegistry.All is default (uninitialized)."); + + var allowed = ComboLogic.GetAllowedCandidateIdsPerGroup(baseQuant); + + if (allowed.IsDefault) + throw new InvalidOperationException("Allowed candidate array is default (uninitialized)."); + + if (allowed.Length == 0) + yield break; + + for (int i = 0; i < allowed.Length; i++) + { + if (allowed[i] == null) + throw new InvalidOperationException( + $"Allowed[{i}] is null for base {string.Join("/", baseQuant.Names)}."); + + if (allowed[i].Length == 0) + throw new InvalidOperationException( + $"Allowed[{i}] is empty for base {string.Join("/", baseQuant.Names)}."); + } + + int dims = allowed.Length; + int dop = ComputeWorkerThreads(GetThreadCountSafe()); + byte baseId = baseQuant.UniqueId; + + var queue = new BlockingCollection>(boundedCapacity: Math.Max(2, dop * 2)); + + var producer = Task.Run(() => + { + try + { + Parallel.ForEach( + Partitioner.Create(0, allowed[0].Length), + new ParallelOptions { MaxDegreeOfParallelism = dop, CancellationToken = ct }, + range => + { + var batch = new List(Math.Min(batchSize, 250_000)); + var idx = new int[dims]; + + var d0 = allowed[0]; + var d1 = allowed[1]; + var d2 = allowed[2]; + var d3 = allowed[3]; + var d4 = allowed[4]; + var d5 = allowed[5]; + var d6 = allowed[6]; + var d7 = allowed[7]; + var d8 = allowed[8]; + + for (int i0 = range.Item1; i0 < range.Item2; i0++) + { + ct.ThrowIfCancellationRequested(); + + idx[0] = i0; + Array.Clear(idx, 1, dims - 1); + + while (true) + { + batch.Add(new TensorConfig( + baseId, + d0[idx[0]], d1[idx[1]], d2[idx[2]], d3[idx[3]], d4[idx[4]], + d5[idx[5]], d6[idx[6]], d7[idx[7]], d8[idx[8]])); + + if (batch.Count >= batchSize) + { + queue.Add(batch, ct); + batch = new List(Math.Min(batchSize, 250_000)); + } + + int d = dims - 1; + while (d >= 1) + { + idx[d]++; + if (idx[d] < allowed[d].Length) + break; + + idx[d] = 0; + d--; + } + + if (d < 1) + break; + } + } + + if (batch.Count > 0) + queue.Add(batch, ct); + }); + } + finally + { + queue.CompleteAdding(); + } + }, ct); + + foreach (var batch in queue.GetConsumingEnumerable(ct)) + yield return batch; + + producer.GetAwaiter().GetResult(); + } + + + private static int GetThreadCountSafe() + { + int tc = Cache.SysInfo?.ThreadCount ?? Environment.ProcessorCount; + return Math.Max(1, tc); + } + + private static int ComputeWorkerThreads(int threadCount) + { + if (threadCount <= 1) + return 1; + + int workers = + threadCount < 16 + ? threadCount - 1 + : (int)Math.Floor(threadCount * 0.90); + + return Math.Clamp(workers, 1, Math.Max(1, threadCount - 1)); + } +} \ No newline at end of file diff --git a/src/MagicQuant/Helpers/pip_runner.py b/src/MagicQuant/Helpers/pip_runner.py new file mode 100644 index 0000000..e4ea566 --- /dev/null +++ b/src/MagicQuant/Helpers/pip_runner.py @@ -0,0 +1,44 @@ +import os +import sys +import runpy + +# Get the current script directory +script_dir = os.path.dirname(os.path.abspath(__file__)) + +# Construct paths for key directories +lib_dir = os.path.join(script_dir, 'Lib') +site_packages_dir = os.path.join(lib_dir, 'site-packages') + +# Add the necessary paths to sys.path +sys.path.insert(0, lib_dir) +sys.path.insert(0, site_packages_dir) + +# Check if pip is available +try: + import pip +except ImportError: + print("pip is not available in the current environment.", file=sys.stderr) + sys.exit(1) + +def run_pip_command(command): + """ + Run pip commands dynamically using pip directly as a module. + """ + # Prepare arguments for pip by splitting the command string + sys.argv = ['pip'] + command.split() + + # Run pip using runpy to run pip as a module + try: + runpy.run_module('pip', run_name="__main__") + except Exception as e: + print(f"Failed to run pip command: {e}", file=sys.stderr) + +# Main execution logic +if __name__ == "__main__": + # If there are command-line arguments, use them + if len(sys.argv) > 1: + # Use arguments passed to the script (excluding the script name) + run_pip_command(' '.join(sys.argv[1:])) + else: + # Default to checking pip version if no arguments are passed + run_pip_command('--version') diff --git a/src/MagicQuant/Interfaces/ICommand.cs b/src/MagicQuant/Interfaces/ICommand.cs new file mode 100644 index 0000000..0bd90a9 --- /dev/null +++ b/src/MagicQuant/Interfaces/ICommand.cs @@ -0,0 +1,7 @@ +namespace MagicQuant.Commands; +using MagicQuant.Models; + +public interface ICommand +{ + Task Run(List args); +} \ No newline at end of file diff --git a/src/MagicQuant/MagicQuant.csproj b/src/MagicQuant/MagicQuant.csproj new file mode 100644 index 0000000..3a5621d --- /dev/null +++ b/src/MagicQuant/MagicQuant.csproj @@ -0,0 +1,55 @@ + + + + Exe + true + magicquant + MagicQuant + 0.1.0 + MagicCodingMan + Benchmark-driven GGUF quantization and mixed-precision tensor-group hybrid discovery for llama.cpp. + GGUF;quantization;llama.cpp;LLM;mixed-precision;benchmark + https://github.com/magiccodingman/MagicQuant + https://github.com/magiccodingman/MagicQuant + git + true + AGPL-3.0-only + README.md + icon.png + net10.0 + enable + enable + + + + + + + + + + + + + + PreserveNewest + PreserveNewest + + + Always + Always + + + + + + + + + + + + + + + diff --git a/src/MagicQuant/Models/AnomalyDetectionModels.cs b/src/MagicQuant/Models/AnomalyDetectionModels.cs new file mode 100644 index 0000000..06a4dac --- /dev/null +++ b/src/MagicQuant/Models/AnomalyDetectionModels.cs @@ -0,0 +1,241 @@ +using MQ.DB.Models; + +namespace MagicQuant.Models; + +public enum QuantMovementKind +{ + Same = 0, + Downgrade = 1, + Upgrade = 2, + LateralOrEquivalent = 3, + Unknown = 4 +} + +public enum AnomalyMovementClassification +{ + MonotoneDowngrade = 1, + MixedTrade = 2, + MonotoneUpgrade = 3, + LateralOrProviderEquivalent = 4, + Unknown = 5, + NoMovement = 6 +} + +public enum AnomalyRuleDirection +{ + Beneficial = 1, + Harmful = 2, + SuppressionOnly = 3 +} + +public enum AnomalyRuleStatus +{ + Confirmed = 1, + Rejected = 2, + Suppressed = 3, + Retired = 4 +} + +public enum AnomalySeedClass +{ + ConfirmedHistoricalCounterfactual = 1, + HistoricalMissingTwin = 2, + PredictionSpaceSmoke = 3, + ExploratorySingle = 4, + ExploratoryPair = 5, + ConfirmedAnomalyNeighborhoodProbe = 6, + SynergyTransferProbe = 7, + SynergyCompositionProbe = 8 +} + +public enum AnomalyProbeClassification +{ + BeneficialAnomaly = 1, + CounterfactualMdaViolation = 2, + HarmfulInteraction = 3, + RejectedSmoke = 4, + NormalGravity = 5, + SuppressionOnly = 6, + SingleGroupInversion = 7, + PairSynergy = 8, + HigherOrderSynergy = 9, + ContextOnly = 10, + MissingTwin = 11, + MissingProbeBenchmark = 12, + SuperSynergy = 13, + AdditiveComposition = 14, + RedundantComposition = 15, + HarmfulInterference = 16, + CompositionRejected = 17, + ContaminatingPassenger = 18 +} + +public sealed class AnomalyChangedGroup +{ + public TensorGroup Group { get; init; } = default!; + public byte CandidateQuantId { get; init; } + public byte ReferenceQuantId { get; init; } + public byte CandidateStoredSlot { get; init; } + public byte ReferenceStoredSlot { get; init; } + public QuantMovementKind Movement { get; init; } +} + +public sealed class AnomalyMovementAnalysis +{ + public AnomalyMovementClassification Classification { get; init; } + public IReadOnlyList ChangedGroups { get; init; } = Array.Empty(); + public int UpgradeCount { get; init; } + public int DowngradeCount { get; init; } + public int SameCount { get; init; } + public int UnknownCount { get; init; } + public int LateralCount { get; init; } + public int NetBitDelta { get; init; } +} + +public sealed class AnomalySmokeCandidate +{ + public string Source { get; init; } = string.Empty; + public TensorConfig CandidateConfig { get; init; } + public TensorConfig TwinConfig { get; init; } + public HybridQuant CandidateQuant => (HybridQuant)CandidateConfig; + public HybridQuant TwinQuant => (HybridQuant)TwinConfig; + public AnomalyMovementAnalysis Movement { get; init; } = new(); + public double CandidatePredictedKld { get; init; } + public double TwinPredictedKld { get; init; } + public ulong? CandidatePredictedSizeBytes { get; init; } + public ulong? TwinPredictedSizeBytes { get; init; } + public ulong? PredictedSizeSavingsBytes { get; init; } + public ulong? ActualSizeSavingsBytes { get; init; } + public bool PlannedProbeWillMeasureSize { get; init; } + public string TwinLookupMode { get; init; } = string.Empty; + public string RejectionReason { get; init; } = string.Empty; + public bool MatchedConfirmedAnomalyPattern { get; init; } + public bool TwinFoundInLookupDictionary { get; init; } + public AnomalySeedClass SeedClass { get; init; } = AnomalySeedClass.PredictionSpaceSmoke; + public double PredictionSpaceGapVsTwin { get; init; } + public ulong? CandidatePredictionRank { get; init; } + public ulong? TwinPredictionRank { get; init; } + public double SmokeScore { get; init; } + public string SmokeStrength { get; init; } = string.Empty; + public bool HasActualTwin { get; init; } + public double? CandidateActualKld { get; init; } + public double? TwinActualKld { get; init; } + public ulong? CandidateActualSizeBytes { get; init; } + public ulong? TwinActualSizeBytes { get; init; } + public bool IsConfirmedFromHistory { get; init; } + public string Message { get; init; } = string.Empty; +} + +public sealed class AnomalyProbePlan +{ + public AnomalySmokeCandidate Seed { get; init; } = default!; + public TensorConfig ReferenceConfig { get; init; } + public TensorConfig ProbeConfig { get; init; } + public IReadOnlyList ProbeGroups { get; init; } = Array.Empty(); + public string ProbeType { get; init; } = string.Empty; + public string HypothesisLabel { get; init; } = string.Empty; + public AnomalySeedClass SeedClass { get; init; } + public AnomalySeedClass ProbePriorityClass { get; init; } +} + +public sealed class AnomalyProbeResult +{ + public AnomalyProbePlan Plan { get; init; } = default!; + public BenchmarkSnapshotRecord? ProbeSnapshot { get; init; } + public BenchmarkSnapshotRecord? ReferenceSnapshot { get; init; } + public AnomalyProbeClassification Classification { get; init; } + public AnomalyRuleDirection RuleDirection { get; init; } + public bool Accepted { get; init; } + public double ActualGainVsTwin { get; init; } + public string FailureCode { get; init; } = string.Empty; + public string Message { get; init; } = string.Empty; +} + +public sealed class AnomalyAdjustmentSummary +{ + public int AppliedRuleCount { get; init; } + public long MatchedRowCount { get; init; } + public string DuckDbPath { get; init; } = string.Empty; + public IReadOnlyList RuleMatches { get; init; } = Array.Empty(); +} + +public sealed class AnomalyRunResult +{ + public IReadOnlyList SmokeCandidates { get; init; } = Array.Empty(); + public IReadOnlyList ProbePlans { get; init; } = Array.Empty(); + public IReadOnlyList ProbeResults { get; init; } = Array.Empty(); + public AnomalyAdjustmentSummary AdjustmentSummary { get; init; } = new(); + public object? BestAnomalyReconciliation { get; init; } +} + +public sealed class AnomalySmokeScanDiagnostics +{ + public long PredictedRowsScanned { get; set; } + public long SparseRowsSkipped { get; set; } + public long SparseRowsNormalized { get; set; } + public long Bf16ExactRowsSkipped { get; set; } + public long PureReferenceRowsSkipped { get; set; } + public long ContextualRowsScanned { get; set; } + public long TwinLookupCount { get; set; } + public long DictionaryTwinHits { get; set; } + public long MissingTwins { get; set; } + public long FallbackDbTwinLookups { get; set; } + public long MovementNotMonotoneDowngrade { get; set; } + public long MixedTradeIgnored { get; set; } + public long SizeSavingsBelowThreshold { get; set; } + public long PredictionSpaceGapTooLarge { get; set; } + public long QueuedSmokeCandidates { get; set; } + public long LoadPredictedRowsMs { get; set; } + public long BuildLookupDictionaryMs { get; set; } + public long ScanRowsMs { get; set; } + public IReadOnlyList RejectedPreview { get; set; } = Array.Empty(); + public IReadOnlyList ClosestGapFailures { get; set; } = Array.Empty(); +} + +public sealed class ProbePlanningDiagnostics +{ + public int ExistingRuleKeysLoaded { get; set; } + public int SkippedExistingRuleOrSuppression { get; set; } + public int SkippedDuplicate { get; set; } + public int SkippedInvalidMovement { get; set; } + public int SkippedBudget { get; set; } + public int ProbesQueued { get; set; } + public int ExpansionProbesQueued { get; set; } + public int CompositionProbesQueued { get; set; } + public int TransferProbesQueued { get; set; } + public int ExploratoryPairProbesQueued { get; set; } + public int SkippedContaminationSuppression { get; set; } +} + +public sealed class SynergyCompositionProbeRecord +{ + public string CompositionId { get; init; } = string.Empty; + public IReadOnlyList SourceTemplateIds { get; init; } = Array.Empty(); + public IReadOnlyList SourceTemplateLabels { get; init; } = Array.Empty(); + public Dictionary CandidateEffectiveGroups { get; init; } = new(StringComparer.OrdinalIgnoreCase); + public Dictionary TwinEffectiveGroups { get; init; } = new(StringComparer.OrdinalIgnoreCase); + public int CombinedGroupCount { get; init; } + public string Classification { get; init; } = string.Empty; + public double? ActualCandidateKld { get; init; } + public double? ActualTwinKld { get; init; } + public double? ActualGainVsTwin { get; init; } + public double? PredictedCandidateKld { get; init; } + public double? PredictedTwinKld { get; init; } + public double? PredictionSpaceGap { get; init; } + public IReadOnlyList Notes { get; init; } = Array.Empty(); +} + +public sealed class SynergyWingSummary +{ + public string Zone { get; init; } = string.Empty; + public int SmokeCount { get; set; } + public int ConfirmedBeneficialTemplates { get; set; } + public int HarmfulTemplates { get; set; } + public int SuppressionOnlyTemplates { get; set; } + public int CandidateRowsAdjustedPositively { get; set; } + public int CandidateRowsDemoted { get; set; } + public int ValidationSuccessCount { get; set; } + public int ValidationFailureCount { get; set; } + public int FinalSurvivorsFromZone { get; set; } + public string Explanation { get; set; } = string.Empty; +} diff --git a/src/MagicQuant/Models/CliArg.cs b/src/MagicQuant/Models/CliArg.cs new file mode 100644 index 0000000..52df970 --- /dev/null +++ b/src/MagicQuant/Models/CliArg.cs @@ -0,0 +1,7 @@ +namespace MagicQuant.Models; + +public class CliArg +{ + public string? Name { get; set; } + public string? Value { get; set; } +} \ No newline at end of file diff --git a/src/MagicQuant/Models/HybridFinalizationModels.cs b/src/MagicQuant/Models/HybridFinalizationModels.cs new file mode 100644 index 0000000..67276d8 --- /dev/null +++ b/src/MagicQuant/Models/HybridFinalizationModels.cs @@ -0,0 +1,240 @@ +using System.Security.Cryptography; +using System.Text; +using MQ.DB.Models; + +namespace MagicQuant.Models; + +public static class TensorConfigIdentity +{ + public static string ToKey(TensorConfig config) + { + return string.Join(":", + config.BaseQuant, + config.Embeddings, + config.LmHead, + config.AttnQ, + config.AttnKV, + config.AttnOutput, + config.FfnUpGate, + config.FfnDown, + config.MoeExperts, + config.MoeRouter); + } + + public static bool IsPureBaseline(TensorConfig config) + { + return config.Embeddings == BaselineQuants.TensorConfigNullSlotValue && + config.LmHead == BaselineQuants.TensorConfigNullSlotValue && + config.AttnQ == BaselineQuants.TensorConfigNullSlotValue && + config.AttnKV == BaselineQuants.TensorConfigNullSlotValue && + config.AttnOutput == BaselineQuants.TensorConfigNullSlotValue && + config.FfnUpGate == BaselineQuants.TensorConfigNullSlotValue && + config.FfnDown == BaselineQuants.TensorConfigNullSlotValue && + config.MoeExperts == BaselineQuants.TensorConfigNullSlotValue && + config.MoeRouter == BaselineQuants.TensorConfigNullSlotValue; + } + + public static IReadOnlyList<(TensorGroup Group, byte StoredValue)> EnumerateGroupSlots(TensorConfig config) + { + return + [ + (TReg.Embeddings, config.Embeddings), + (TReg.LmHead, config.LmHead), + (TReg.AttnQ, config.AttnQ), + (TReg.AttnKV, config.AttnKV), + (TReg.AttnOutput, config.AttnOutput), + (TReg.FfnUpGate, config.FfnUpGate), + (TReg.FfnDown, config.FfnDown), + (TReg.MoeExperts, config.MoeExperts), + (TReg.MoeRouter, config.MoeRouter) + ]; + } + + public static string StableHash(string value) + { + var bytes = SHA256.HashData(Encoding.UTF8.GetBytes(value)); + return Convert.ToHexString(bytes).ToLowerInvariant(); + } +} + +public sealed class EffectiveStateResolutionResult +{ + public TensorConfig Config { get; init; } + public string EffectiveStateKey { get; init; } = string.Empty; + public bool HasUnknownMappings { get; init; } + public IReadOnlyList Warnings { get; init; } = Array.Empty(); + public IReadOnlyDictionary GroupStates { get; init; } = new Dictionary(StringComparer.Ordinal); + public string BaseState { get; init; } = string.Empty; +} + +public sealed class PredictedCandidateEvaluation +{ + public TensorConfig Config { get; init; } + public HybridQuant Quant { get; init; } = default!; + public ulong PredictedSizeBytes { get; init; } + public double PredictedKldCost { get; init; } + public double PredictedPplCost { get; init; } + public double CompositeScore { get; init; } + public string EffectiveStateKey { get; init; } = string.Empty; + public bool HasUnknownMappings { get; init; } + public byte BaseBitRange { get; init; } + public bool IsPureBaseline { get; init; } + public List Notes { get; init; } = new(); +} + +public sealed class BitRangeBucketDefinition +{ + public string Key { get; init; } = string.Empty; + public byte LowerBitRange { get; init; } + public byte UpperBitRange { get; init; } + public ulong LowerAnchorSizeBytes { get; init; } + public ulong UpperAnchorSizeBytes { get; init; } + public bool WasSkipped { get; init; } + public string? SkipReason { get; init; } +} + +public sealed class BucketedCandidate +{ + public PredictedCandidateEvaluation Evaluation { get; init; } = default!; + public BitRangeBucketDefinition Bucket { get; init; } = default!; +} + +public sealed class BucketPruneDiagnostics +{ + public string BucketKey { get; init; } = string.Empty; + public ulong LowerAnchorSizeBytes { get; init; } + public ulong UpperAnchorSizeBytes { get; init; } + public int IncomingCount { get; set; } + public int RemovedCount { get; set; } + public int KeptCount { get; set; } + public bool Skipped { get; set; } + public string? SkipReason { get; set; } + public Dictionary RemovalReasons { get; } = new(StringComparer.OrdinalIgnoreCase); + public List Notes { get; } = new(); + + public void CountReason(string reason) + { + RemovalReasons.TryGetValue(reason, out var current); + RemovalReasons[reason] = current + 1; + } +} + +public sealed class SurvivalStageReport +{ + public long StartingCount { get; set; } + public long EndingCount { get; set; } + public long RemovedCount => StartingCount - EndingCount; + public Dictionary RemovalCounts { get; } = new(StringComparer.OrdinalIgnoreCase); + public List BucketDiagnostics { get; } = new(); + public List Notes { get; } = new(); + + public void AddRemoval(string reason, long count) + { + RemovalCounts.TryGetValue(reason, out var current); + RemovalCounts[reason] = current + count; + } +} + +public sealed class BenchmarkSnapshotRecord +{ + public TensorConfig Config { get; init; } + public HybridQuant Quant { get; init; } = default!; + public string DisplayName { get; init; } = string.Empty; + public string ProviderName { get; init; } = string.Empty; + public string BaselineFamily { get; init; } = string.Empty; + + /// + /// True only for MagicQuant-discovered mixed tensor configurations. + /// Exact/base-only blankets and uniform external rebuilt baselines are not hybrids. + /// + public bool IsHybrid { get; init; } + + public bool IsExternalPureBaseline { get; init; } + + /// + /// True when MagicQuant rebuilt/materialized an external provider baseline for equal-footing + /// benchmarking/export, but did not invent a mixed MagicQuant hybrid recipe. + /// + public bool IsExternalRebuiltBaseline { get; init; } + + /// + /// True when the tensor config contains materialized tensor-group overrides, even if those + /// overrides are only exact/native anchors or a uniform external baseline rebuild. + /// + public bool IsMaterializedTensorMapped { get; init; } + + public ulong SizeBytes { get; init; } + public double Kld { get; init; } + public double Ppl { get; init; } + public string? OutputModelPath { get; init; } + public string? ExternalRepositoryUrl { get; init; } +} + +public sealed class FinalSelectionRow +{ + public int Id { get; set; } + public bool Enabled { get; set; } = true; + public BenchmarkSnapshotRecord Snapshot { get; init; } = default!; + + // Planned public/export identity. The CLI previews these names and the export + // service reuses them, so a user never sees one name in the selection UI and + // a different name in the produced GGUF/README. + public string PlannedFileName { get; set; } = string.Empty; + public string PlannedDisplayName { get; set; } = string.Empty; + public string PlannedProviderName { get; set; } = string.Empty; + public string PlannedQuantFamily { get; set; } = string.Empty; +} + +public sealed class ExportedArtifactRecord +{ + public BenchmarkSnapshotRecord Snapshot { get; init; } = default!; + public string DisplayName { get; init; } = string.Empty; + public string ProviderName { get; init; } = string.Empty; + public string BaselineFamily { get; init; } = string.Empty; + public bool IsExternalReference { get; init; } + public string? FileName { get; init; } + public string? FullPath { get; init; } + public string DownloadTarget { get; init; } = string.Empty; + public ulong ExpectedSizeBytes { get; init; } + public ulong? ActualSizeBytes { get; set; } + public EffectiveStateResolutionResult? EffectiveState { get; init; } +} + +public sealed class HybridMapEntry +{ + public string ExportedFileName { get; set; } = string.Empty; + public string DisplayName { get; set; } = string.Empty; + public string ProviderSource { get; set; } = string.Empty; + public string BaselineFamily { get; set; } = string.Empty; + public string? OriginalReferenceBaseline { get; set; } + public Dictionary TensorGroups { get; set; } = new(StringComparer.Ordinal); + public string EffectiveQuantStateKey { get; set; } = string.Empty; + public bool HasUnknownMappings { get; set; } + public List Warnings { get; set; } = new(); + public bool UsedImatrix { get; set; } + public ulong ExpectedSizeBytes { get; set; } + public double ExpectedSizeGB { get; set; } + public double ExpectedSizeGiB { get; set; } + public ulong? ActualSizeBytes { get; set; } + public double? ActualSizeGB { get; set; } + public double? ActualSizeGiB { get; set; } + public string? OriginalExternalSource { get; set; } +} + +public sealed class FinalRealEliminationResult +{ + public IReadOnlyList Survivors { get; init; } = Array.Empty(); + public IReadOnlyList Eliminated { get; init; } = Array.Empty(); +} + +public sealed class CombinationSurvivalExecutionResult +{ + public IReadOnlyList BenchmarkedSnapshots { get; init; } = Array.Empty(); + public IReadOnlyList BrutalSurvivors { get; init; } = Array.Empty(); + public IReadOnlyList SelectedRows { get; init; } = Array.Empty(); + public IReadOnlyList ExportedArtifacts { get; init; } = Array.Empty(); + public IReadOnlyList BucketDiagnostics { get; init; } = Array.Empty(); + public SurvivalStageReport SurvivalReport { get; init; } = new(); + public IReadOnlyList Eliminations { get; init; } = Array.Empty(); + public IReadOnlyList ValidationFailures { get; init; } = Array.Empty(); +} diff --git a/src/MagicQuant/Models/ImatrixModels.cs b/src/MagicQuant/Models/ImatrixModels.cs new file mode 100644 index 0000000..740d26a --- /dev/null +++ b/src/MagicQuant/Models/ImatrixModels.cs @@ -0,0 +1,62 @@ +namespace MagicQuant.Models; + +public enum ImatrixSourceKind +{ + Https = 1, + HfDataset = 2, + LocalDatasetFile = 3 +} + +public sealed class ImatrixRequest +{ + public bool UseImatrix { get; init; } + public bool ForceRebuild { get; init; } + + public string? ImatrixUrl { get; init; } + + public string? DatasetRepo { get; init; } + public string? DatasetSplit { get; init; } + public string? DatasetConfig { get; init; } + + public string? LocalDatasetFile { get; init; } + + public string ModelDirectory { get; init; } = default!; + public string MagicQuantDirectory { get; init; } = default!; +} + +public sealed class ImatrixEnsureResult +{ + public bool Enabled { get; init; } + public bool Available { get; init; } + public bool Rebuilt { get; init; } + public string? CanonicalImatrixPath { get; init; } + public ImatrixSourceKind? SourceKind { get; init; } +} + +public sealed class ImatrixSuccessSidecar +{ + public string Status { get; init; } = "success"; + public DateTime CompletedUtc { get; init; } + public string ArtifactType { get; init; } = "imatrix"; + public string CanonicalFileName { get; init; } = "imatrix.dat"; + public string CanonicalPath { get; init; } = string.Empty; + public string SourceKind { get; init; } = string.Empty; + public string SourceIdentity { get; init; } = string.Empty; + public string? Split { get; init; } + public string? Config { get; init; } + public string Sha256 { get; init; } = string.Empty; + public long FileSizeBytes { get; init; } + public string BuilderVersion { get; init; } = "mvp-v1"; +} + +public sealed class ImatrixMetadataSidecar +{ + public string SourceKind { get; init; } = string.Empty; + public string? OriginalUrl { get; init; } + public string? OriginalDownloadName { get; init; } + public string? DatasetRepo { get; init; } + public string? DatasetConfig { get; init; } + public string? DatasetSplit { get; init; } + public string? LocalDatasetFile { get; init; } + public string Notes { get; init; } = "Renamed to canonical imatrix.dat after acquisition/build"; +} diff --git a/src/MagicQuant/Models/Learning/TensorLearningModels.cs b/src/MagicQuant/Models/Learning/TensorLearningModels.cs new file mode 100644 index 0000000..4fbde50 --- /dev/null +++ b/src/MagicQuant/Models/Learning/TensorLearningModels.cs @@ -0,0 +1,61 @@ +using MQ.DB.Models; + +namespace MagicQuant.Models.Learning; + +public sealed class TensorGroupingResult +{ + public TensorGroup? PrimaryGroup { get; init; } + public IReadOnlyList MatchedGroups { get; init; } = []; + public bool IsBaseQuantException { get; init; } + public string? MatchedExceptionPattern { get; init; } +} + +public sealed class TensorGroupingAuditIssue +{ + public required string TensorName { get; init; } + public required string IssueKind { get; init; } + public IReadOnlyList MatchedGroups { get; init; } = []; + public string? MatchedExceptionPattern { get; init; } + public string? FinalQuantType { get; init; } + public string? LearningSource { get; init; } +} + +public sealed class TensorGroupingAuditResult +{ + public required IReadOnlyDictionary GroupedByTensor { get; init; } + public required IReadOnlyList Ambiguous { get; init; } + public required IReadOnlyList IllegalUnresolved { get; init; } + public required IReadOnlyList BaseQuantExceptions { get; init; } + + public bool HasFatalIssues => Ambiguous.Count > 0 || IllegalUnresolved.Count > 0; + public int FatalIssueCount => Ambiguous.Count + IllegalUnresolved.Count; +} + +public sealed record LearnedTensorTruth(string TensorName, string FinalQuantType, LearningSource Source); + +public enum LearningSource +{ + LogOnly = 1, + GgufOnly = 2, + Both = 3, + BothWithMismatch = 4, + InheritedFromNative = 5 +} + +public sealed class TensorTruthMismatch +{ + public required string TensorName { get; init; } + public required string LogQuantType { get; init; } + public required string GgufQuantType { get; init; } + public bool IsHighSeverity { get; init; } +} + +public sealed class TensorTruthVerificationResult +{ + public required IReadOnlyDictionary TruthByTensor { get; init; } + public IReadOnlyList HardMismatches { get; init; } = []; + public IReadOnlyList SoftMismatches { get; init; } = []; + public IReadOnlyList LogOnly { get; init; } = []; + + public bool HasFatalIssues => HardMismatches.Count > 0; +} diff --git a/src/MagicQuant/Models/PredictionSelectionModels.cs b/src/MagicQuant/Models/PredictionSelectionModels.cs new file mode 100644 index 0000000..494063e --- /dev/null +++ b/src/MagicQuant/Models/PredictionSelectionModels.cs @@ -0,0 +1,190 @@ +using MQ.DB.Models; + +namespace MagicQuant.Models; + +public sealed class RankSafePredictionRow +{ + public TensorConfig Config { get; init; } + public HybridQuant Quant { get; init; } = default!; + public ulong PredictedSizeBytes { get; set; } + public bool IsSizePredictable { get; set; } = true; + public double AdditiveKld { get; set; } + public double InteractionKld { get; set; } + public double PredictedKld { get; set; } + public double PredictionConfidence { get; set; } = 1.0d; + public double PredictedPpl { get; set; } + public double CrossTerm { get; set; } + public bool IsPureBaseline { get; init; } + public bool IsHybrid => !IsPureBaseline; + public bool IsPredictable { get; set; } = true; + public bool HasUnknownMappings { get; set; } + public string EffectiveStateKey { get; init; } = string.Empty; + public List Notes { get; init; } = new(); + + public double ActualKld { get; set; } = double.NaN; + public double ActualPpl { get; set; } = double.NaN; + public ulong? ActualSizeBytes { get; set; } + public int? ActualRank { get; set; } + public ulong? PredictedRank { get; set; } + public double AnomalyAdjustmentKld { get; set; } + + public double AbsoluteKldError => + double.IsNaN(ActualKld) ? double.NaN : Math.Abs(PredictedKld - ActualKld); + + public double SignedKldError => + double.IsNaN(ActualKld) ? double.NaN : PredictedKld - ActualKld; +} + +public sealed class RankSafePredictionSet +{ + public IReadOnlyList Rows { get; init; } = Array.Empty(); + public RankSafePredictionFit Fit { get; init; } = new(); + public IReadOnlyList Notes { get; init; } = Array.Empty(); + + public IReadOnlyList PredictableRows => + Rows.Where(x => x.IsPredictable).ToList(); +} + +public sealed class RankSafePredictionFit +{ + public double Alpha { get; init; } = 1.0d; + public double Beta { get; init; } = 0.0d; + public double BitStressThreshold { get; init; } = 8.0d; + public int FitRowCount { get; init; } + public double FitMae { get; init; } + public bool UsedFallback { get; init; } +} + + +public sealed class PredictedAnchorRow +{ + public required TensorConfig Config { get; init; } + public required string ConfigKey { get; init; } + public required string DisplayName { get; init; } + public required string BaselineCanonicalKey { get; init; } + public byte RuntimeBaselineId { get; init; } + public double PredictedKld { get; init; } + public ulong PredictedSizeBytes { get; init; } + public double PredictionConfidence { get; init; } + public ulong PredictionRank { get; init; } + public bool IsVirtualPredictionAnchor { get; init; } = true; +} + +public sealed class HybridSelectionAnchor +{ + public BenchmarkSnapshotRecord Snapshot { get; init; } = default!; + public string Key => TensorConfigIdentity.ToKey(Snapshot.Config); +} + +public enum HybridSelectionReason +{ + StrictDominanceReplacement = 1, + NearBaselineOnePercentReplacement = 2, + InteriorSubspaceDiscovery = 3, + + // SQLite/isolation-truth fallback candidates. These are intentionally not + // DuckDB prediction-space rows; they are conservative baseline-blanket + // tuning attempts used only after the normal selector cannot validate a win. + SmartStrictDominanceFallback = 4, + SmartNearBaselineFallback = 5, + SmartInteriorSubspaceFallback = 6 +} + +public sealed class HybridSelectionCandidate +{ + public RankSafePredictionRow Prediction { get; init; } = default!; + public HybridSelectionReason Reason { get; init; } + public BenchmarkSnapshotRecord LowerDamageAnchor { get; init; } = default!; + public BenchmarkSnapshotRecord HigherDamageAnchor { get; init; } = default!; + public ulong WindowMinSizeBytes { get; init; } + public ulong WindowMaxSizeBytes { get; init; } + public double LinearExpectedKld { get; init; } + public double PredictedGainOverLine { get; init; } + public int AttemptOrder { get; init; } + public string WindowLabel { get; init; } = string.Empty; + public ulong PredictionWindowMinSizeBytes { get; init; } + public ulong PredictionWindowMaxSizeBytes { get; init; } + public PredictedAnchorRow? HigherDamagePredictionAnchor { get; init; } + public PredictedAnchorRow? LowerDamagePredictionAnchor { get; init; } + + // Diagnostic-only context captured at selection time. These values do not + // change acceptance rules; they explain how the candidate was found, how + // many neighbors existed, and how hard the retry aperture was capped. + public long CandidatePoolSize { get; init; } + public long WindowCandidateCount { get; init; } + public long LineBeatingCandidateCount { get; init; } + public int FetchedCandidateCount { get; init; } + public int CandidatesAfterBrutalityCount { get; init; } + public int CandidateAttemptLimit { get; init; } + public int PhaseWindowIndex { get; init; } + public int PhaseWindowCount { get; init; } + public int RawSelectionRank { get; init; } + public string CandidateTheoryFamilyKey { get; init; } = string.Empty; + public int CandidateTheoryFamilyRank { get; init; } + public int CandidateTheoryFamilyMemberRank { get; init; } + public string CandidateTheoryFamilyDisplay { get; init; } = string.Empty; + public string DiversityMode { get; init; } = string.Empty; + public IReadOnlyList CandidateSelectionNotes { get; init; } = Array.Empty(); +} + +public sealed class CandidateValidationResult +{ + public HybridSelectionCandidate Candidate { get; init; } = default!; + public BenchmarkSnapshotRecord? Snapshot { get; init; } + public bool Accepted { get; init; } + public string Message { get; init; } = string.Empty; + public string FailureCode { get; init; } = string.Empty; +} + +public sealed class BaselineEliminationRecord +{ + public BenchmarkSnapshotRecord Eliminated { get; init; } = default!; + public BenchmarkSnapshotRecord Eliminator { get; init; } = default!; + public string Reason { get; init; } = string.Empty; + public bool EliminatorIsHybrid => Eliminator.IsHybrid; + public double EliminatedKld => Eliminated.Kld; + public double EliminatorKld => Eliminator.Kld; + public ulong EliminatedSizeBytes => Eliminated.SizeBytes; + public ulong EliminatorSizeBytes => Eliminator.SizeBytes; +} + +public sealed class RankSafeValidationSummary +{ + public int RowCount { get; init; } + public int PredictableCount { get; init; } + public double Mae { get; init; } + public double Rmse { get; init; } + public double MaxAbsoluteError { get; init; } + public double MeanSignedError { get; init; } + public double PairwiseAccuracyPercent { get; init; } + public long ConcordantPairs { get; init; } + public long DiscordantPairs { get; init; } + public long TiedPredictedPairs { get; init; } + public int ExactRankMatches { get; init; } + public int WithinOneRank { get; init; } + public int WithinTwoRanks { get; init; } + public int WithinFiveRanks { get; init; } + public int WithinTenRanks { get; init; } + public int WithinTwentyRanks { get; init; } +} + +public sealed class PredictionValidationExportResult +{ + public RankSafeValidationSummary Summary { get; init; } = new(); + public string CsvPath { get; init; } = string.Empty; + public string MarkdownPath { get; init; } = string.Empty; + public IReadOnlyList Rows { get; init; } = Array.Empty(); +} + +public sealed class PhaseValidationResult +{ + public IReadOnlyList AcceptedSnapshots { get; init; } = Array.Empty(); + public IReadOnlyList Attempts { get; init; } = Array.Empty(); +} + +public sealed class PredictionGuidedSelectionResult +{ + public IReadOnlyList Survivors { get; init; } = Array.Empty(); + public IReadOnlyList Eliminations { get; init; } = Array.Empty(); + public IReadOnlyList ValidationFailures { get; init; } = Array.Empty(); +} \ No newline at end of file diff --git a/src/MagicQuant/Models/RepositoryCloneModels.cs b/src/MagicQuant/Models/RepositoryCloneModels.cs new file mode 100644 index 0000000..c2dd59e --- /dev/null +++ b/src/MagicQuant/Models/RepositoryCloneModels.cs @@ -0,0 +1,52 @@ +using System.Text.Json.Serialization; + +namespace MagicQuant.Models; + +public sealed class MagicQuantCloneManifest +{ + public int SchemaVersion { get; set; } = 1; + public DateTime GeneratedUtc { get; set; } = DateTime.UtcNow; + public string Generator { get; set; } = "MagicQuant"; + public string? SourceRepository { get; set; } + public string? SourceJson { get; set; } + public string? SourceModelId { get; set; } + public string? SourceArchitectureFamily { get; set; } + public string? Notes { get; set; } + + public List Artifacts { get; set; } = new(); +} + +public sealed class MagicQuantCloneArtifact +{ + public string FileName { get; set; } = string.Empty; + public string DisplayName { get; set; } = string.Empty; + public string ShortName { get; set; } = string.Empty; + public string Provider { get; set; } = string.Empty; + public string QuantFamily { get; set; } = string.Empty; + public string BaseQuant { get; set; } = "Q8_0"; + public bool IsHybrid { get; set; } + public bool UsedImatrix { get; set; } + + public double? SourceKld { get; set; } + public double? SourcePpl { get; set; } + public double? SourcePplDeltaPercent { get; set; } + public ulong? SourceSizeBytes { get; set; } + public double? SourceSizeGB { get; set; } + public double? SourceSizeGiB { get; set; } + + /// + /// Exact tensor-name -> final GGUF quant type map read from the exported artifact. + /// This is the real clone payload. + /// + public Dictionary TensorTypes { get; set; } = new(StringComparer.Ordinal); +} + +public sealed class CloneArtifactBuildRecord +{ + public MagicQuantCloneArtifact ManifestArtifact { get; init; } = default!; + public string OutputPath { get; init; } = string.Empty; + public ulong ActualSizeBytes { get; set; } + public double? Kld { get; set; } + public double? Ppl { get; set; } + public double? PplDeltaPercent { get; set; } +} diff --git a/src/MagicQuant/Program.cs b/src/MagicQuant/Program.cs new file mode 100644 index 0000000..9ca83b5 --- /dev/null +++ b/src/MagicQuant/Program.cs @@ -0,0 +1,142 @@ +using MagicQuant.Commands; +using MagicQuant.Configuration; +using MagicQuant.Helpers; +using MagicQuant.Models; +using MagicQuant.Services; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +if (args.Length == 1 && args[0] == "--version") +{ + Console.WriteLine(typeof(CommandCatalog).Assembly.GetCustomAttributes(typeof(System.Reflection.AssemblyInformationalVersionAttribute), false) + .Cast().Single().InformationalVersion); + return; +} + +var commands = CommandCatalog.Create(); + +if (args.Length == 0 || CommandCatalog.IsHelp(args[0])) +{ + CliHelpers.ShowHelp(commands); + return; +} + +string commandInput = args[0]; + +if (!commands.TryGetValue(commandInput, out var commandInfo)) +{ + AnsiConsole.MarkupLine($"[red]Error:[/] The command [yellow]'{Markup.Escape(commandInput)}'[/] does not exist."); + CliHelpers.ShowHelp(commands); + Environment.ExitCode = 2; + return; +} + +List parsedArgs = CliHelpers.ParseArguments(args.Skip(1)); + +using var cancellation = new CancellationTokenSource(); +using var cancellationScope = MagicQuant.Runtime.RunCancellation.Use(cancellation.Token); +ConsoleCancelEventHandler onCancel = (_, e) => +{ + // First Ctrl+C cooperatively unwinds leases/processes; a second uses OS termination. + e.Cancel = !cancellation.IsCancellationRequested; + cancellation.Cancel(); +}; +Console.CancelKeyPress += onCancel; +RunProvenanceService? provenance = null; +string completionStatus = "failed"; +string? completionError = null; + +try +{ + // Help is a read-only operation: do not load config, clean caches, install + // dependencies, or open databases just to explain a command. + if (parsedArgs.Any(a => string.Equals(a.Name, "help", StringComparison.OrdinalIgnoreCase)) || + args.Skip(1).Any(a => a == "-h")) + { + await commandInfo.Factory().Run([new CliArg { Name = "help", Value = string.Empty }]); + return; + } + + if (commandInput.Equals("init-config", StringComparison.OrdinalIgnoreCase)) + { + InitConfig.ValidateTokens(args.Skip(1).ToArray()); + await commandInfo.Factory().Run(parsedArgs); + return; + } + + CliOptionValidator.Validate(parsedArgs); + var loaded = MagicQuantYamlLoader.Read(parsedArgs); + CommandPreflight.Validate(commandInput, loaded.Settings, parsedArgs); + if (parsedArgs.Any(a => string.Equals(a.Name, "check-config", StringComparison.OrdinalIgnoreCase))) + { + foreach (string warning in loaded.Warnings) AnsiConsole.WriteLine(warning); + AnsiConsole.WriteLine("Configuration and input paths are valid. No runtime setup was performed."); + return; + } + MagicQuantYamlLoader.Apply(loaded); + var loadedConfig = loaded.Settings; + provenance = new RunProvenanceService(commandInput, args, loaded); + var startupScratch = new ScratchStorageService(); + await startupScratch.CleanupStaleScratchArtifactsAsync(); + + TensorWeightScheme.ValidateSmallestConfiguration(); + BaselineQuants.ValidateIntegrityOrThrow(); + QuantizationService.ValidateQuantNameNormalizationOrThrow(); + + if (!commandInput.Equals("initialize-llama-cpp", StringComparison.OrdinalIgnoreCase)) + { + await AnsiConsole.Status() + .StartAsync("[grey]Checking environment dependencies...[/]", async _ => + { + var initializer = new InitializeLlamaCpp(); + var validationArgs = new List + { + new() { Name = "validate", Value = string.Empty } + }; + + if (!string.IsNullOrWhiteSpace(loadedConfig.Paths.LlamaRoot)) + validationArgs.Add(new CliArg { Name = "llama-root", Value = loadedConfig.Paths.LlamaRoot }); + if (!string.IsNullOrWhiteSpace(loadedConfig.Paths.LlamaBin)) + validationArgs.Add(new CliArg { Name = "llama-bin", Value = loadedConfig.Paths.LlamaBin }); + if (!string.IsNullOrWhiteSpace(loadedConfig.Paths.ConvertScript)) + validationArgs.Add(new CliArg { Name = "convert-script", Value = loadedConfig.Paths.ConvertScript }); + + await initializer.Run(validationArgs); + }); + + AnsiConsole.MarkupLine("[bold green][/] Environment validated."); + AnsiConsole.WriteLine(); + } + + if (!commandInput.Equals("initialize-llama-cpp", StringComparison.OrdinalIgnoreCase)) + await provenance.CaptureToolchainAsync(); + cancellation.Token.ThrowIfCancellationRequested(); + var commandInstance = commandInfo.Factory(); + await commandInstance.Run(parsedArgs); + if (commandInput.Equals("initialize-llama-cpp", StringComparison.OrdinalIgnoreCase)) + await provenance.CaptureToolchainAsync(); + cancellation.Token.ThrowIfCancellationRequested(); + completionStatus = "completed"; +} +catch (OperationCanceledException) +{ + AnsiConsole.WriteLine("Run canceled. Active native work has been stopped."); + Environment.ExitCode = 130; + completionStatus = "canceled"; +} +catch (Exception ex) +{ + completionError = ex.Message; + AnsiConsole.WriteException(ex); + Environment.ExitCode = 1; +} +finally +{ + Console.CancelKeyPress -= onCancel; + try { provenance?.Complete(completionStatus, completionError); } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + AnsiConsole.WriteLine($"Could not finalize local run provenance: {ex.Message}"); + } +} diff --git a/src/MagicQuant/Properties/AssemblyInfo.cs b/src/MagicQuant/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..5aa468b --- /dev/null +++ b/src/MagicQuant/Properties/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("MagicQuant.Tests")] diff --git a/src/MagicQuant/Runtime/IProcessRunner.cs b/src/MagicQuant/Runtime/IProcessRunner.cs new file mode 100644 index 0000000..cabb6f9 --- /dev/null +++ b/src/MagicQuant/Runtime/IProcessRunner.cs @@ -0,0 +1,10 @@ +using System.Diagnostics; + +namespace MagicQuant.Runtime; + +/// Inject native execution at IO boundaries without mocking numerical policy. +public interface IProcessRunner +{ + Task RunAsync(ProcessStartInfo start, string? logPath = null, + Action? onLine = null, CancellationToken ct = default); +} diff --git a/src/MagicQuant/Runtime/NativeCommand.cs b/src/MagicQuant/Runtime/NativeCommand.cs new file mode 100644 index 0000000..66159a3 --- /dev/null +++ b/src/MagicQuant/Runtime/NativeCommand.cs @@ -0,0 +1,20 @@ +using System.Diagnostics; +using System.Text.Json; + +namespace MagicQuant.Runtime; + +/// Executable and literal argv; never interpreted by a shell. +public sealed record NativeCommand(string Executable, IReadOnlyList Arguments) +{ + public ProcessStartInfo CreateStartInfo(IReadOnlyDictionary? environment = null) + { + var start = new ProcessStartInfo(Executable); + foreach (string arg in Arguments) start.ArgumentList.Add(arg); + if (environment != null) + foreach (var (key, value) in environment) start.Environment[key] = value; + return start; + } + + // Diagnostic representation only, not a command to execute or shell-escape. + public override string ToString() => JsonSerializer.Serialize(new { Executable, Arguments }); +} diff --git a/src/MagicQuant/Runtime/ProcessRunner.cs b/src/MagicQuant/Runtime/ProcessRunner.cs new file mode 100644 index 0000000..02f7307 --- /dev/null +++ b/src/MagicQuant/Runtime/ProcessRunner.cs @@ -0,0 +1,84 @@ +using System.Diagnostics; +using System.Text; + +namespace MagicQuant.Runtime; + +public sealed record ProcessResult(int ExitCode, string StdOut, string StdErr) +{ + public bool Success => ExitCode == 0; + public string CombinedOutput => StdOut + StdErr; +} + +/// +/// Owns native process lifetime, drains both pipes concurrently, and closes logs on +/// every exit path. Cancellation kills and reaps the child tree before callers may +/// dispose scratch leases. Exit codes remain explicit so each caller owns retry policy. +/// +public sealed class ProcessRunner : IProcessRunner +{ + public async Task RunAsync(ProcessStartInfo start, string? logPath = null, + Action? onLine = null, CancellationToken ct = default) + { + using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, RunCancellation.Token); + ct = cancellation.Token; + ct.ThrowIfCancellationRequested(); + start.RedirectStandardOutput = true; + start.RedirectStandardError = true; + start.UseShellExecute = false; + start.CreateNoWindow = true; + using var log = logPath == null ? null : new StreamWriter(new FileStream(logPath, FileMode.Create, FileAccess.Write, FileShare.Read)) { AutoFlush = true }; + using var process = new Process { StartInfo = start }; + if (!process.Start()) throw new InvalidOperationException($"Failed to start '{start.FileName}'."); + var stdout = new StringBuilder(); + var stderr = new StringBuilder(); + object sync = new(); + + async Task DrainAsync(StreamReader reader, StringBuilder buffer, bool error) + { + try + { + while (await reader.ReadLineAsync() is { } line) + { + lock (sync) + { + buffer.AppendLine(line); + log?.WriteLine(line); + onLine?.Invoke(line, error); + } + } + } + catch + { + // Wake the other drain when this pipe/callback fails. + try { if (!process.HasExited) process.Kill(entireProcessTree: true); } + catch (InvalidOperationException) { } + throw; + } + } + + Task drains = Task.WhenAll(DrainAsync(process.StandardOutput, stdout, false), DrainAsync(process.StandardError, stderr, true)); + try + { + // A logging callback or pipe failure must terminate the child as well, + // rather than letting it hang forever with an undrained output pipe. + Task exited = process.WaitForExitAsync(ct); + Task completed = await Task.WhenAny(exited, drains); + if (completed == drains) await drains; + await exited; + await drains.WaitAsync(ct); + ct.ThrowIfCancellationRequested(); + return new ProcessResult(process.ExitCode, stdout.ToString(), stderr.ToString()); + } + catch + { + if (!process.HasExited) + { + try { process.Kill(entireProcessTree: true); } + catch (InvalidOperationException) { /* The child exited between the check and kill. */ } + } + await process.WaitForExitAsync(CancellationToken.None); + try { await drains; } catch { /* Preserve the original cancellation/pipe failure. */ } + throw; + } + } +} diff --git a/src/MagicQuant/Runtime/RunCancellation.cs b/src/MagicQuant/Runtime/RunCancellation.cs new file mode 100644 index 0000000..143fb57 --- /dev/null +++ b/src/MagicQuant/Runtime/RunCancellation.cs @@ -0,0 +1,24 @@ +namespace MagicQuant.Runtime; + +/// +/// Cancellation for the current async command scope. Legacy service APIs without a +/// token still stop native work; new APIs should also accept explicit caller tokens. +/// This value flows into tasks and is restored when the command finishes. +/// +public static class RunCancellation +{ + private static readonly AsyncLocal Ambient = new(); + public static CancellationToken Token => Ambient.Value; + + public static IDisposable Use(CancellationToken token) + { + var previous = Ambient.Value; + Ambient.Value = token; + return new Scope(previous); + } + + private sealed class Scope(CancellationToken previous) : IDisposable + { + public void Dispose() => Ambient.Value = previous; + } +} diff --git a/src/MagicQuant/Services/AnomalyAdjustedPredictionService.cs b/src/MagicQuant/Services/AnomalyAdjustedPredictionService.cs new file mode 100644 index 0000000..3dedd8e --- /dev/null +++ b/src/MagicQuant/Services/AnomalyAdjustedPredictionService.cs @@ -0,0 +1,657 @@ +using DuckDB.NET.Data; +using MagicQuant.Models; +using MQ.DB; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; +using Spectre.Console; +using System.Globalization; +using System.Numerics; +using System.Text.Json; + +namespace MagicQuant.Services; + +public sealed class AnomalyAdjustedPredictionService +{ + private const double UpdateEpsilon = 1e-15d; + + private readonly RemainingCombinationStore _store; + + public AnomalyAdjustedPredictionService(RemainingCombinationStore store) + { + _store = store; + } + + public async Task ApplyAsync( + IReadOnlyCollection rules, + CancellationToken ct) + { + if (rules.Count == 0) + return new AnomalyAdjustmentSummary { DuckDbPath = _store.GetDatabaseFilePath() }; + + using var c = new DuckDBConnection($"Data Source={_store.GetDatabaseFilePath()}"); + await c.OpenAsync(ct); + await ConfigureSessionAsync(c, ct); + + await ExecuteAsync(c, $@" +UPDATE {CombinationDuckDbSchema.TableName} +SET AnomalyAdjustmentKld = 0.0, + FinalPredictedKld = BaseRankSafeKld, + PredictedKld = BaseRankSafeKld +WHERE BaseRankSafeKld IS NOT NULL;", ct); + + long totalMatched = 0; + var matchLogs = new List(); + + foreach (var rule in rules.OrderByDescending(x => x.Confidence).ThenBy(x => x.Id)) + { + if (IsDirection(rule, AnomalyRuleDirection.SuppressionOnly)) + { + var log = LogSuppressionOnly(rule); + matchLogs.Add(log); + AnsiConsole.MarkupLine( + $"[grey]Anomaly rule suppression-only:[/] rule=[cyan]{Markup.Escape(DescribeRule(rule))}[/] no prediction score mutation."); + continue; + } + + if (IsDirection(rule, AnomalyRuleDirection.Beneficial)) + { + AnsiConsole.MarkupLine("[grey]Broad beneficial same-selected-group adjustment disabled; applying pairwise twin ordering only.[/]"); + var result = await ApplyBeneficialPairwiseOrderingRuleAsync(c, rule, ct); + if (result.HasValue) + { + var pairwise = result.Value; + totalMatched += pairwise.MatchedCandidateRows; + matchLogs.Add(pairwise.LogObject); + } + + continue; + } + + if (IsDirection(rule, AnomalyRuleDirection.Harmful)) + { + var result = await ApplyBroadHarmfulDemotionRuleAsync(c, rule, ct); + if (result.HasValue) + { + var harmful = result.Value; + totalMatched += harmful.MatchedRows; + matchLogs.Add(harmful.LogObject); + } + + continue; + } + } + + await ReRankAsync(c, ct); + + return new AnomalyAdjustmentSummary + { + AppliedRuleCount = rules.Count, + MatchedRowCount = totalMatched, + DuckDbPath = _store.GetDatabaseFilePath(), + RuleMatches = matchLogs + }; + } + + private static async Task ApplyBeneficialPairwiseOrderingRuleAsync( + DuckDBConnection c, + AnomalyInteractionRule rule, + CancellationToken ct) + { + string candidateWhere = BuildRuleCandidateWhere(rule, "c"); + if (string.IsNullOrWhiteSpace(candidateWhere)) + return null; + + await ExecuteAsync(c, "DROP TABLE IF EXISTS temp_anomaly_rule_candidates;", ct); + await ExecuteAsync(c, "DROP TABLE IF EXISTS temp_anomaly_rule_twins;", ct); + await ExecuteAsync(c, "DROP TABLE IF EXISTS temp_anomaly_rule_updates;", ct); + + await ExecuteAsync(c, $@" +CREATE TEMP TABLE temp_anomaly_rule_candidates AS +SELECT {CombinationDuckDbSchema.QualifySlotColumnList("c")}, + COALESCE(c.BaseRankSafeKld, c.PredictedKld) AS CandidateBaseRankSafeKld, + COALESCE(c.FinalPredictedKld, c.PredictedKld, c.BaseRankSafeKld) AS CandidateCurrentFinalKld +FROM {CombinationDuckDbSchema.TableName} c +WHERE {candidateWhere};", ct); + + long matchedCandidateRows = await CountTempRowsAsync(c, "temp_anomaly_rule_candidates", ct); + if (matchedCandidateRows == 0) + return null; + + string twinJoin = BuildPairwiseTwinJoinPredicate(rule, "c", "t"); + if (string.IsNullOrWhiteSpace(twinJoin)) + return null; + + await ExecuteAsync(c, $@" +CREATE TEMP TABLE temp_anomaly_rule_twins AS +SELECT {CombinationDuckDbSchema.QualifySlotColumnList("c")}, + MIN(COALESCE(t.FinalPredictedKld, t.PredictedKld, t.BaseRankSafeKld)) AS TwinEffectiveKld +FROM temp_anomaly_rule_candidates c +JOIN {CombinationDuckDbSchema.TableName} t + ON {twinJoin} +WHERE COALESCE(t.FinalPredictedKld, t.PredictedKld, t.BaseRankSafeKld) IS NOT NULL +GROUP BY {CombinationDuckDbSchema.QualifySlotColumnList("c")};", ct); + + long twinRowsFound = await CountTempRowsAsync(c, "temp_anomaly_rule_twins", ct); + long missingTwinRows = Math.Max(0, matchedCandidateRows - twinRowsFound); + + double margin = Math.Max(0d, Config.AnomalyDetection.PredictionSpaceViolationMargin); + await ExecuteAsync(c, $@" +CREATE TEMP TABLE temp_anomaly_rule_updates AS +SELECT {CombinationDuckDbSchema.QualifySlotColumnList("c")}, + c.CandidateBaseRankSafeKld, + c.CandidateCurrentFinalKld, + tw.TwinEffectiveKld, + GREATEST(0.0, LEAST(c.CandidateCurrentFinalKld, tw.TwinEffectiveKld - {SqlDouble(margin)})) AS NewFinalKld, + c.CandidateCurrentFinalKld - GREATEST(0.0, LEAST(c.CandidateCurrentFinalKld, tw.TwinEffectiveKld - {SqlDouble(margin)})) AS OrderingAdjustmentApplied +FROM temp_anomaly_rule_candidates c +JOIN temp_anomaly_rule_twins tw + ON {CombinationDuckDbSchema.BuildSlotEqualityPredicate("c", "tw")} +WHERE GREATEST(0.0, LEAST(c.CandidateCurrentFinalKld, tw.TwinEffectiveKld - {SqlDouble(margin)})) < c.CandidateCurrentFinalKld - {SqlDouble(UpdateEpsilon)};", ct); + + var stats = await LoadPairwiseUpdateStatsAsync(c, ct); + + await ExecuteAsync(c, $@" +UPDATE {CombinationDuckDbSchema.TableName} t +SET FinalPredictedKld = u.NewFinalKld, + PredictedKld = u.NewFinalKld, + AnomalyAdjustmentKld = u.NewFinalKld - COALESCE(t.BaseRankSafeKld, t.PredictedKld, 0.0) +FROM temp_anomaly_rule_updates u +WHERE {CombinationDuckDbSchema.BuildSlotEqualityPredicate("t", "u")};", ct); + + double advisoryCap = Math.Max(0d, Config.AnomalyDetection.MaxConfirmedPairwiseOrderingAdjustmentKld); + bool exceededAdvisoryCap = advisoryCap > 0d && stats.MaxOrderingAdjustmentApplied > advisoryCap; + var actual = ExtractActualEffect(rule); + + var log = new + { + ruleId = rule.Id, + direction = rule.RuleDirection, + ruleType = rule.RuleType, + applicationMode = "pairwise-twin-ordering", + broadBeneficialSameSelectedGroupAdjustment = "disabled", + referenceQuant = SafeName(rule.ReferenceQuantId), + groupSetHash = rule.GroupSetHash, + matchedCandidateRows, + twinRowsFound, + missingTwinRows, + rowsReordered = stats.RowsReordered, + maxOrderingAdjustmentApplied = stats.MaxOrderingAdjustmentApplied, + meanOrderingAdjustmentApplied = stats.MeanOrderingAdjustmentApplied, + minCandidateBefore = stats.MinCandidateBefore, + meanTwinEffectiveKld = stats.MeanTwinEffectiveKld, + meanCandidateAfter = stats.MeanCandidateAfter, + predictionSpaceViolationMargin = margin, + advisoryMaxConfirmedPairwiseOrderingAdjustmentKld = advisoryCap, + exceededAdvisoryCap, + actualCandidateKld = actual.CandidateKld, + actualTwinKld = actual.TwinKld, + actualGainOrHarm = actual.GainOrHarm, + confidence = rule.Confidence, + groups = BuildGroupLog(rule) + }; + + AnsiConsole.MarkupLine( + $"[green]Applying beneficial anomaly rule as pairwise ordering:[/] rule=[cyan]{Markup.Escape(DescribeRule(rule))}[/] " + + $"matchedCandidateRows=[cyan]{matchedCandidateRows:N0}[/] twinRowsFound=[cyan]{twinRowsFound:N0}[/] missingTwinRows=[cyan]{missingTwinRows:N0}[/] " + + $"rowsReordered=[cyan]{stats.RowsReordered:N0}[/] maxOrderingAdjustmentApplied=[cyan]{stats.MaxOrderingAdjustmentApplied:0.000000}[/] " + + $"meanTwinEffectiveKld=[cyan]{stats.MeanTwinEffectiveKld:0.000000}[/] meanCandidateAfter=[cyan]{stats.MeanCandidateAfter:0.000000}[/]"); + + if (missingTwinRows > 0) + { + AnsiConsole.MarkupLine( + $"[yellow]Beneficial anomaly twin lookup miss:[/] rule=[cyan]{Markup.Escape(DescribeRule(rule))}[/] missingTwinRows=[cyan]{missingTwinRows:N0}[/]. No broad fallback boost was applied."); + } + + if (exceededAdvisoryCap) + { + AnsiConsole.MarkupLine( + $"[yellow]Pairwise ordering adjustment exceeded advisory cap:[/] maxApplied=[cyan]{stats.MaxOrderingAdjustmentApplied:0.000000}[/], advisoryCap=[cyan]{advisoryCap:0.000000}[/]. Confirmed pairwise ordering was preserved anyway."); + } + + return new BeneficialPairwiseResult(matchedCandidateRows, twinRowsFound, missingTwinRows, stats.RowsReordered, log); + } + + private static async Task ApplyBroadHarmfulDemotionRuleAsync( + DuckDBConnection c, + AnomalyInteractionRule rule, + CancellationToken ct) + { + string where = BuildRuleCandidateWhere(rule, null); + if (string.IsNullOrWhiteSpace(where)) + return null; + + double adjustment = rule.AppliedPredictionSpaceAdjustmentKld; + if (adjustment <= 0d) + adjustment = Math.Max(Config.AnomalyDetection.PredictionSpaceViolationMargin, Math.Abs(adjustment)); + + if (adjustment <= 0d) + return null; + + long before = await CountMatchesAsync(c, where, ct); + if (before == 0) + return null; + + var beforeStats = await LoadPredictionStatsAsync(c, where, ct); + + string expression = $"LEAST(COALESCE(AnomalyAdjustmentKld, 0.0) + ({SqlDouble(adjustment)}), LEAST({SqlDouble(Config.AnomalyDetection.MaxPositiveAdjustmentKld)}, COALESCE(BaseRankSafeKld, 0.0) * {SqlDouble(Config.AnomalyDetection.MaxAdjustmentFractionOfBaseKld)}))"; + + await ExecuteAsync(c, $@" +UPDATE {CombinationDuckDbSchema.TableName} +SET AnomalyAdjustmentKld = {expression}, + FinalPredictedKld = GREATEST(0.0, COALESCE(BaseRankSafeKld, PredictedKld, 0.0) + {expression}), + PredictedKld = GREATEST(0.0, COALESCE(BaseRankSafeKld, PredictedKld, 0.0) + {expression}) +WHERE {where};", ct); + + var afterStats = await LoadPredictionStatsAsync(c, where, ct); + var actual = ExtractActualEffect(rule); + var log = new + { + ruleId = rule.Id, + direction = rule.RuleDirection, + ruleType = rule.RuleType, + applicationMode = "broad-harmful-demotion", + referenceQuant = SafeName(rule.ReferenceQuantId), + groupSetHash = rule.GroupSetHash, + basePredictedKld = beforeStats.AverageBasePredictedKld, + adjustment, + adjustedPredictedKld = afterStats.AverageFinalPredictedKld, + actualCandidateKld = actual.CandidateKld, + actualTwinKld = actual.TwinKld, + actualGainOrHarm = actual.GainOrHarm, + broadHarmfulDemotionMatches = before, + totalAdjustedRows = before, + matchedRows = before, + confidence = rule.Confidence, + groups = BuildGroupLog(rule) + }; + + AnsiConsole.MarkupLine( + $"[yellow]Applying broad harmful demotion:[/] rule=[cyan]{Markup.Escape(DescribeRule(rule))}[/] " + + $"matchedRows=[cyan]{before:N0}[/] adjustment=[cyan]{adjustment:0.000000}[/] " + + $"beforeAvg=[cyan]{beforeStats.AverageFinalPredictedKld:0.000000}[/] afterAvg=[cyan]{afterStats.AverageFinalPredictedKld:0.000000}[/]"); + + return new BroadRuleResult(before, log); + } + + internal static string BuildRuleCandidateWhere(AnomalyInteractionRule rule, string? alias) + { + if (rule.GroupStates.Count == 0) + return string.Empty; + + if (BaselineQuants.IsNativeExactAlias(rule.ReferenceQuantId) || + rule.GroupStates.Any(x => BaselineQuants.IsNativeExactAlias(x.CandidateQuantId) || BaselineQuants.IsNativeExactAlias(x.ReferenceQuantId))) + { + return string.Empty; + } + + string q(string column) => string.IsNullOrWhiteSpace(alias) ? column : $"{alias}.{column}"; + + var predicates = new List + { + $"COALESCE({q("IsProtectedAnchor")}, FALSE) = FALSE", + $"{q("BaseRankSafeKld")} IS NOT NULL" + }; + + if (!Config.SynergyDetection.ContextScopedRuleApplicationEnabled) + predicates.Add($"{q("BaseQuant")} = {rule.ReferenceQuantId}"); + + foreach (var state in rule.GroupStates.OrderBy(x => x.SortOrder)) + { + string? column = ColumnNameForGroupId(state.TensorGroupId); + if (column == null) + return string.Empty; + + predicates.Add($"{EffectiveQuantSql(alias, column)} = {state.CandidateQuantId}"); + } + + string contextFidelityPredicate = BuildContextFidelityPredicate(rule, alias); + if (!string.IsNullOrWhiteSpace(contextFidelityPredicate)) + predicates.Add(contextFidelityPredicate); + + return string.Join(" AND ", predicates); + } + + internal static string BuildContextFidelityPredicate(AnomalyInteractionRule rule, string? alias) + { + if (!Config.SynergyDetection.ContextScopedRuleApplicationEnabled) + return string.Empty; + + var ruleGroupIds = rule.GroupStates + .Select(x => x.TensorGroupId) + .ToHashSet(); + var referenceContext = ParseReferenceContextKey(rule.ReferenceContextKey); + string[] terms = ActiveGroups() + .Where(group => !ruleGroupIds.Contains(group.UniqueId)) + .Select(group => new + { + Column = ColumnNameForGroupId(group.UniqueId), + ExpectedQuantId = referenceContext.GetValueOrDefault(group.UniqueId, rule.ReferenceQuantId) + }) + .Where(x => x.Column != null) + .Select(x => $"CASE WHEN {EffectiveQuantSql(alias, x.Column!)} = {x.ExpectedQuantId} THEN 0 ELSE 1 END") + .ToArray(); + + if (terms.Length == 0) + return string.Empty; + + int maximumContextMismatches = Math.Clamp( + Config.SynergyDetection.MaxNonRuleGroupContextMismatches, + 0, + terms.Length); + return $"(({string.Join(" + ", terms)}) <= {maximumContextMismatches})"; + } + + private static IReadOnlyDictionary ParseReferenceContextKey(string? contextKey) + { + var result = new Dictionary(); + if (string.IsNullOrWhiteSpace(contextKey)) + return result; + + foreach (string item in contextKey.Split('|', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + string[] parts = item.Split(':', 2, StringSplitOptions.TrimEntries); + if (parts.Length == 2 && byte.TryParse(parts[0], out byte groupId) && byte.TryParse(parts[1], out byte quantId)) + result[groupId] = quantId; + } + + return result; + } + + private static string BuildPairwiseTwinJoinPredicate(AnomalyInteractionRule rule, string candidateAlias, string twinAlias) + { + if (rule.GroupStates.Count == 0) + return string.Empty; + + var byColumn = new Dictionary(StringComparer.Ordinal); + foreach (var state in rule.GroupStates.OrderBy(x => x.SortOrder)) + { + string? column = ColumnNameForGroupId(state.TensorGroupId); + if (column == null) + return string.Empty; + + byColumn[column] = state; + } + + var predicates = new List + { + $"{twinAlias}.BaseQuant = {candidateAlias}.BaseQuant" + }; + + foreach (string column in CombinationDuckDbSchema.SlotColumns.Skip(1)) + { + if (!byColumn.TryGetValue(column, out var state)) + { + predicates.Add($"{twinAlias}.{column} = {candidateAlias}.{column}"); + continue; + } + + byte referenceStoredSlot = BaselineQuants.EncodeTensorConfigGroupSlotBaselineId(state.ReferenceQuantId); + predicates.Add($"({twinAlias}.{column} = {referenceStoredSlot} OR ({state.ReferenceQuantId} = {twinAlias}.BaseQuant AND {twinAlias}.{column} = 0))"); + } + + return string.Join(" AND ", predicates); + } + + private static string EffectiveQuantSql(string? alias, string column) + { + string prefix = string.IsNullOrWhiteSpace(alias) ? string.Empty : alias + "."; + return $"(CASE WHEN {prefix}{column} = 0 THEN {prefix}BaseQuant ELSE CAST({prefix}{column} AS INTEGER) - 1 END)"; + } + + private static IReadOnlyList ActiveGroups() + { + TensorGroup[] ordered = + [ + TReg.Embeddings, + TReg.LmHead, + TReg.AttnQ, + TReg.AttnKV, + TReg.AttnOutput, + TReg.FfnUpGate, + TReg.FfnDown, + TReg.MoeExperts, + TReg.MoeRouter + ]; + + return ordered + .Where(g => !Cache.UnusedTensorGroups.Any(u => u.UniqueId == g.UniqueId)) + .OrderBy(g => g.UniqueId) + .ToList(); + } + + private static string? ColumnNameForGroupId(byte groupId) + { + if (groupId == TReg.Embeddings.UniqueId) return "Embeddings"; + if (groupId == TReg.LmHead.UniqueId) return "LmHead"; + if (groupId == TReg.AttnQ.UniqueId) return "AttnQ"; + if (groupId == TReg.AttnKV.UniqueId) return "AttnKV"; + if (groupId == TReg.AttnOutput.UniqueId) return "AttnOutput"; + if (groupId == TReg.FfnUpGate.UniqueId) return "FfnUpGate"; + if (groupId == TReg.FfnDown.UniqueId) return "FfnDown"; + if (groupId == TReg.MoeExperts.UniqueId) return "MoeExperts"; + if (groupId == TReg.MoeRouter.UniqueId) return "MoeRouter"; + return null; + } + + private static async Task ReRankAsync(DuckDBConnection c, CancellationToken ct) + { + await ExecuteAsync(c, $@" +DROP TABLE IF EXISTS temp_anomaly_rerank; +CREATE TEMP TABLE temp_anomaly_rerank AS +SELECT {CombinationDuckDbSchema.SlotColumnList}, + CAST(ROW_NUMBER() OVER ( + ORDER BY COALESCE(FinalPredictedKld, PredictedKld) ASC, + PredictedSizeBytes ASC, + PredictionConfidence DESC, + BaseQuant ASC, + Embeddings ASC, + LmHead ASC, + AttnQ ASC, + AttnKV ASC, + AttnOutput ASC, + FfnUpGate ASC, + FfnDown ASC, + MoeExperts ASC, + MoeRouter ASC + ) AS UBIGINT) AS NewPredictionRank +FROM {CombinationDuckDbSchema.TableName} +WHERE COALESCE(FinalPredictedKld, PredictedKld) IS NOT NULL + AND PredictedSizeBytes IS NOT NULL + AND PredictionConfidence IS NOT NULL + AND {CombinationDuckDbSchema.ActiveCandidatePredicateSql}; + +UPDATE {CombinationDuckDbSchema.TableName} t +SET PredictionRank = r.NewPredictionRank +FROM temp_anomaly_rerank r +WHERE {CombinationDuckDbSchema.BuildSlotEqualityPredicate("t", "r")};", ct); + } + + private static async Task CountMatchesAsync(DuckDBConnection c, string where, CancellationToken ct) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = $"SELECT COUNT(*) FROM {CombinationDuckDbSchema.TableName} WHERE {where};"; + return ToInt64(await cmd.ExecuteScalarAsync(ct)); + } + + private static async Task CountTempRowsAsync(DuckDBConnection c, string tableName, CancellationToken ct) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = $"SELECT COUNT(*) FROM {tableName};"; + return ToInt64(await cmd.ExecuteScalarAsync(ct)); + } + + private static async Task LoadPredictionStatsAsync(DuckDBConnection c, string where, CancellationToken ct) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = $@" +SELECT AVG(COALESCE(BaseRankSafeKld, PredictedKld)), + AVG(COALESCE(FinalPredictedKld, PredictedKld)) +FROM {CombinationDuckDbSchema.TableName} +WHERE {where};"; + + using var r = await cmd.ExecuteReaderAsync(ct); + if (!await r.ReadAsync(ct)) + return new PredictionMatchStats(0d, 0d); + + return new PredictionMatchStats(ToDouble(r.GetValue(0)), ToDouble(r.GetValue(1))); + } + + private static async Task LoadPairwiseUpdateStatsAsync(DuckDBConnection c, CancellationToken ct) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = @" +SELECT COUNT(*), + COALESCE(MAX(OrderingAdjustmentApplied), 0.0), + COALESCE(AVG(OrderingAdjustmentApplied), 0.0), + COALESCE(MIN(CandidateCurrentFinalKld), 0.0), + COALESCE(AVG(TwinEffectiveKld), 0.0), + COALESCE(AVG(NewFinalKld), 0.0) +FROM temp_anomaly_rule_updates;"; + + using var r = await cmd.ExecuteReaderAsync(ct); + if (!await r.ReadAsync(ct)) + return new PairwiseUpdateStats(0, 0d, 0d, 0d, 0d, 0d); + + return new PairwiseUpdateStats( + ToInt64(r.GetValue(0)), + ToDouble(r.GetValue(1)), + ToDouble(r.GetValue(2)), + ToDouble(r.GetValue(3)), + ToDouble(r.GetValue(4)), + ToDouble(r.GetValue(5))); + } + + private static ActualRuleEffect ExtractActualEffect(AnomalyInteractionRule rule) + { + if (string.IsNullOrWhiteSpace(rule.MetadataJson)) + return new ActualRuleEffect(null, null, null, false); + + try + { + using var doc = JsonDocument.Parse(rule.MetadataJson); + var root = doc.RootElement; + double? candidate = TryGetDouble(root, "actualCandidateKld"); + double? twin = TryGetDouble(root, "actualTwinKld"); + double? gain = TryGetDouble(root, "actualGainOrHarm"); + return new ActualRuleEffect(candidate, twin, gain, candidate.HasValue && twin.HasValue && gain.HasValue); + } + catch + { + return new ActualRuleEffect(null, null, null, false); + } + } + + private static double? TryGetDouble(JsonElement element, string propertyName) + { + return element.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.Number && value.TryGetDouble(out var d) + ? d + : null; + } + + private static object LogSuppressionOnly(AnomalyInteractionRule rule) + { + return new + { + ruleId = rule.Id, + direction = rule.RuleDirection, + ruleType = rule.RuleType, + applicationMode = "suppression-only", + scoreMutation = false, + referenceQuant = SafeName(rule.ReferenceQuantId), + groupSetHash = rule.GroupSetHash, + matchedRows = 0, + totalAdjustedRows = 0, + confidence = rule.Confidence, + groups = BuildGroupLog(rule) + }; + } + + private static IReadOnlyList BuildGroupLog(AnomalyInteractionRule rule) + { + return rule.GroupStates + .OrderBy(x => x.SortOrder) + .Select(x => (object)new + { + x.TensorGroupId, + group = ColumnNameForGroupId(x.TensorGroupId), + candidate = SafeName(x.CandidateQuantId), + reference = SafeName(x.ReferenceQuantId), + x.Movement + }) + .ToList(); + } + + private static bool IsDirection(AnomalyInteractionRule rule, AnomalyRuleDirection direction) => + string.Equals(rule.RuleDirection, direction.ToString(), StringComparison.OrdinalIgnoreCase); + + private static string FmtNullable(double? value) => value.HasValue ? value.Value.ToString("0.000000", CultureInfo.InvariantCulture) : "n/a"; + + private static double ToDouble(object? value) + { + if (value is null || value is DBNull) + return 0d; + + if (value is BigInteger big) + return (double)big; + + return Convert.ToDouble(value, CultureInfo.InvariantCulture); + } + + private static long ToInt64(object? value) + { + if (value is null || value is DBNull) + return 0L; + + if (value is BigInteger big) + return (long)big; + + return Convert.ToInt64(value, CultureInfo.InvariantCulture); + } + + private static async Task ExecuteAsync(DuckDBConnection c, string sql, CancellationToken ct) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = sql; + await cmd.ExecuteNonQueryAsync(ct); + } + + private static async Task ConfigureSessionAsync(DuckDBConnection c, CancellationToken ct) + { + await ExecuteAsync(c, "SET preserve_insertion_order = false;", ct); + await ExecuteAsync(c, $"SET threads = {Math.Max(1, Environment.ProcessorCount)};", ct); + } + + private static string SqlDouble(double value) => value.ToString(CultureInfo.InvariantCulture); + + private static string DescribeRule(AnomalyInteractionRule rule) + { + return string.Join(" + ", rule.GroupStates + .OrderBy(x => x.SortOrder) + .Select(x => $"{ColumnNameForGroupId(x.TensorGroupId)}={SafeName(x.CandidateQuantId)}>{SafeName(x.ReferenceQuantId)}")) + + $" in {SafeName(rule.ReferenceQuantId)} context"; + } + + private readonly record struct PredictionMatchStats(double AverageBasePredictedKld, double AverageFinalPredictedKld); + private readonly record struct ActualRuleEffect(double? CandidateKld, double? TwinKld, double? GainOrHarm, bool HasActualEffect); + private readonly record struct BeneficialPairwiseResult(long MatchedCandidateRows, long TwinRowsFound, long MissingTwinRows, long RowsReordered, object LogObject); + private readonly record struct BroadRuleResult(long MatchedRows, object LogObject); + private readonly record struct PairwiseUpdateStats( + long RowsReordered, + double MaxOrderingAdjustmentApplied, + double MeanOrderingAdjustmentApplied, + double MinCandidateBefore, + double MeanTwinEffectiveKld, + double MeanCandidateAfter); + + private static string SafeName(byte quantId) + { + try + { + return BaselineQuants.FromId(quantId).Names[0]; + } + catch + { + return $"id:{quantId}"; + } + } +} diff --git a/src/MagicQuant/Services/AnomalyRuleRepository.cs b/src/MagicQuant/Services/AnomalyRuleRepository.cs new file mode 100644 index 0000000..2f7d266 --- /dev/null +++ b/src/MagicQuant/Services/AnomalyRuleRepository.cs @@ -0,0 +1,460 @@ +using System.Text.Json; +using MagicQuant.Models; +using Microsoft.EntityFrameworkCore; +using MQ.DB.Data; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; + +namespace MagicQuant.Services; + +public sealed class AnomalyRuleRepository +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + WriteIndented = false + }; + + private readonly QuantFidelityComparerService _movement; + + public AnomalyRuleRepository(QuantFidelityComparerService movement) + { + _movement = movement; + } + + public async Task StartSessionAsync(string sourceRunLabel, CancellationToken ct) + { + await using var db = new MagicQuantContext(); + var scope = await ResolveScopeAsync(db, ct); + var session = new AnomalyProbeSession + { + ArchitectureFamilyId = scope.ArchitectureFamilyId, + TensorGroupProfileId = scope.TensorGroupProfileId, + AiModelHashId = scope.AiModelHashId, + ImatrixDefinitionId = scope.ImatrixDefinitionId, + BenchmarkCategory = (byte)BenchmarkCategory.General, + StartedUtc = DateTime.UtcNow, + SourceRunLabel = sourceRunLabel, + ConfigJson = JsonSerializer.Serialize(Config.AnomalyDetection, JsonOptions) + }; + + db.AnomalyProbeSessions.Add(session); + await db.SaveChangesAsync(ct); + return session; + } + + public async Task CompleteSessionAsync(Guid sessionId, CancellationToken ct) + { + await using var db = new MagicQuantContext(); + var session = await db.AnomalyProbeSessions.FirstOrDefaultAsync(x => x.Id == sessionId, ct); + if (session == null) + return; + + session.CompletedUtc = DateTime.UtcNow; + await db.SaveChangesAsync(ct); + } + + public async Task> PersistProbeResultsAsync( + Guid sessionId, + IReadOnlyCollection results, + CancellationToken ct) + { + if (results.Count == 0) + return Array.Empty(); + + await using var db = new MagicQuantContext(); + var scope = await ResolveScopeAsync(db, ct); + var observations = new List(); + + foreach (var result in results) + { + if (!_movement.IsContextualQuantizedConfig(result.Plan.ReferenceConfig) || + !_movement.IsContextualQuantizedConfig(result.Plan.ProbeConfig)) + { + // Invalid/sparse/BF16-exact anomaly attempts are logged by the workflow and + // intentionally not persisted as contextual anomaly observations. + continue; + } + + _movement.EnsureAllActiveGroupsExplicit(result.Plan.ReferenceConfig, "persist-observation-reference"); + _movement.EnsureAllActiveGroupsExplicit(result.Plan.ProbeConfig, "persist-observation-probe"); + + var referenceCombo = await EnsureTensorComboAsync(db, result.Plan.ReferenceConfig, ct); + var probeCombo = await EnsureTensorComboAsync(db, result.Plan.ProbeConfig, ct); + var groups = result.Plan.ProbeGroups.OrderBy(x => x.Group.UniqueId).ToList(); + var movement = result.Plan.Seed.Movement; + + var observation = new AnomalyProbeObservation + { + SessionId = sessionId, + ArchitectureFamilyId = scope.ArchitectureFamilyId, + TensorGroupProfileId = scope.TensorGroupProfileId, + AiModelHashId = scope.AiModelHashId, + ImatrixDefinitionId = scope.ImatrixDefinitionId, + BenchmarkCategory = (byte)BenchmarkCategory.General, + ReferenceTensorComboId = referenceCombo.Id, + ProbeTensorComboId = probeCombo.Id, + ProbeType = result.Plan.ProbeType, + Classification = result.Classification.ToString(), + HypothesisLabel = result.Plan.HypothesisLabel, + MovementClassification = movement.Classification.ToString(), + ChangedGroupSetHash = _movement.BuildChangedGroupHash(groups), + ChangedGroupsJson = JsonSerializer.Serialize(groups.Select(ToGroupLog), JsonOptions), + CandidateQuantsJson = JsonSerializer.Serialize(groups.ToDictionary(x => x.Group.Name, x => BaselineQuants.FromId(x.CandidateQuantId).Names[0]), JsonOptions), + ReferenceEffectiveGroupsJson = JsonSerializer.Serialize(_movement.BuildEffectiveGroupVector(result.Plan.ReferenceConfig), JsonOptions), + CandidateEffectiveGroupsJson = JsonSerializer.Serialize(_movement.BuildEffectiveGroupVector(result.Plan.ProbeConfig), JsonOptions), + InactiveGroupsJson = JsonSerializer.Serialize(_movement.BuildInactiveGroupList(), JsonOptions), + ReferenceTensorConfigKey = TensorConfigIdentity.ToKey(result.Plan.ReferenceConfig), + ProbeTensorConfigKey = TensorConfigIdentity.ToKey(result.Plan.ProbeConfig), + IsContextualAnomalyProbe = true, + OldBf16Isolation = false, + AllActiveGroupsExplicit = _movement.HasAllActiveGroupsExplicit(result.Plan.ReferenceConfig) && _movement.HasAllActiveGroupsExplicit(result.Plan.ProbeConfig), + ReferenceQuantId = result.Plan.ReferenceConfig.BaseQuant, + ActualKld = result.ProbeSnapshot?.Kld ?? 0d, + PredictedKld = result.Plan.Seed.CandidatePredictedKld, + ReferenceActualKld = result.ReferenceSnapshot?.Kld ?? 0d, + ReferencePredictedKld = result.Plan.Seed.TwinPredictedKld, + ActualGainVsTwin = result.ActualGainVsTwin, + PredictionSpaceGapVsTwin = result.Plan.Seed.PredictionSpaceGapVsTwin, + SizeSavingsBytes = ComputeSizeSavings(result.ReferenceSnapshot, result.ProbeSnapshot), + UpgradeCount = movement.UpgradeCount, + DowngradeCount = movement.DowngradeCount, + SameCount = movement.SameCount, + UnknownCount = movement.UnknownCount, + NetBitDelta = movement.NetBitDelta, + RuleDirection = result.RuleDirection.ToString(), + Accepted = result.Accepted, + FailureCode = result.FailureCode, + Message = result.Message, + CreatedUtc = DateTime.UtcNow + }; + + db.AnomalyProbeObservations.Add(observation); + observations.Add(observation); + } + + await db.SaveChangesAsync(ct); + return observations; + } + + public async Task> UpsertRulesFromResultsAsync( + IReadOnlyCollection results, + CancellationToken ct) + { + var eligible = results + .Where(x => x.RuleDirection != AnomalyRuleDirection.SuppressionOnly || Config.AnomalyDetection.PersistSuppressionResults) + .Where(x => x.ReferenceSnapshot != null && x.ProbeSnapshot != null) + .Where(x => _movement.IsContextualQuantizedConfig(x.Plan.ReferenceConfig)) + .Where(x => _movement.IsContextualQuantizedConfig(x.Plan.ProbeConfig)) + .ToList(); + + if (eligible.Count == 0) + return Array.Empty(); + + await using var db = new MagicQuantContext(); + var scope = await ResolveScopeAsync(db, ct); + var upserted = new List(); + + foreach (var group in eligible.GroupBy(BuildRuleKey, StringComparer.Ordinal)) + { + var first = group.First(); + _movement.EnsureAllActiveGroupsExplicit(first.Plan.ReferenceConfig, "upsert-rule-reference"); + _movement.EnsureAllActiveGroupsExplicit(first.Plan.ProbeConfig, "upsert-rule-probe"); + var probeGroups = first.Plan.ProbeGroups.OrderBy(x => x.Group.UniqueId).ToList(); + string groupSetHash = _movement.BuildChangedGroupHash(probeGroups); + string direction = first.RuleDirection.ToString(); + byte referenceQuantId = first.Plan.ReferenceConfig.BaseQuant; + string referenceContextKey = _movement.ReferenceContextKey(first.Plan.ReferenceConfig); + + var rule = await db.AnomalyInteractionRules + .Include(x => x.GroupStates) + .FirstOrDefaultAsync(x => + x.ArchitectureFamilyId == scope.ArchitectureFamilyId && + x.TensorGroupProfileId == scope.TensorGroupProfileId && + x.AiModelHashId == scope.AiModelHashId && + x.ImatrixDefinitionId == scope.ImatrixDefinitionId && + x.BenchmarkCategory == (byte)BenchmarkCategory.General && + x.ReferenceQuantId == referenceQuantId && + x.ReferenceContextKey == referenceContextKey && + x.GroupSetHash == groupSetHash && + x.RuleDirection == direction, + ct); + + bool isNew = rule == null; + if (rule == null) + { + rule = new AnomalyInteractionRule + { + ArchitectureFamilyId = scope.ArchitectureFamilyId, + TensorGroupProfileId = scope.TensorGroupProfileId, + AiModelHashId = scope.AiModelHashId, + ImatrixDefinitionId = scope.ImatrixDefinitionId, + BenchmarkCategory = (byte)BenchmarkCategory.General, + ReferenceQuantId = referenceQuantId, + ReferenceContextKey = referenceContextKey, + ReferenceEffectiveGroupsJson = JsonSerializer.Serialize(_movement.BuildEffectiveGroupVector(first.Plan.ReferenceConfig), JsonOptions), + CandidateEffectiveGroupsJson = JsonSerializer.Serialize(_movement.BuildEffectiveGroupVector(first.Plan.ProbeConfig), JsonOptions), + InactiveGroupsJson = JsonSerializer.Serialize(_movement.BuildInactiveGroupList(), JsonOptions), + FullTensorConfigKey = TensorConfigIdentity.ToKey(first.Plan.ProbeConfig), + RuleDirection = direction, + GroupSetHash = groupSetHash, + CreatedUtc = DateTime.UtcNow + }; + db.AnomalyInteractionRules.Add(rule); + } + + var rows = group.ToList(); + rule.RuleType = ResolveRuleType(rows); + rule.RuleStatus = first.RuleDirection == AnomalyRuleDirection.Beneficial || first.RuleDirection == AnomalyRuleDirection.Harmful + ? AnomalyRuleStatus.Confirmed.ToString() + : AnomalyRuleStatus.Suppressed.ToString(); + rule.Status = rule.RuleStatus; + rule.MovementClassification = first.Plan.Seed.Movement.Classification.ToString(); + rule.GroupCount = probeGroups.Count; + rule.EvidenceCount = Math.Max(rule.EvidenceCount, 0) + rows.Count; + rule.MeanActualGainVsTwin = rows.Average(x => x.ActualGainVsTwin); + rule.BestActualGainVsTwin = rows.Max(x => x.ActualGainVsTwin); + rule.MeanPredictionSpaceGap = rows.Average(x => x.Plan.Seed.PredictionSpaceGapVsTwin); + rule.BestPredictionSpaceGap = rows.Min(x => x.Plan.Seed.PredictionSpaceGapVsTwin); + rule.ShrinkFactor = Config.AnomalyDetection.AnomalyAdjustmentShrinkFactor; + rule.Confidence = ComputeConfidence(rows); + rule.AppliedPredictionSpaceAdjustmentKld = ComputePredictionAdjustment(first, rule.Confidence); + rule.ReferenceContextKey = referenceContextKey; + rule.ReferenceEffectiveGroupsJson = JsonSerializer.Serialize(_movement.BuildEffectiveGroupVector(first.Plan.ReferenceConfig), JsonOptions); + rule.CandidateEffectiveGroupsJson = JsonSerializer.Serialize(_movement.BuildEffectiveGroupVector(first.Plan.ProbeConfig), JsonOptions); + rule.InactiveGroupsJson = JsonSerializer.Serialize(_movement.BuildInactiveGroupList(), JsonOptions); + rule.FullTensorConfigKey = TensorConfigIdentity.ToKey(first.Plan.ProbeConfig); + rule.UpdatedUtc = DateTime.UtcNow; + rule.MetadataJson = JsonSerializer.Serialize(new + { + source = "counterfactual-twin-probe", + isContextualAnomalyProbe = true, + oldBf16Isolation = false, + allActiveGroupsExplicit = true, + referenceEffectiveGroups = _movement.BuildEffectiveGroupVector(first.Plan.ReferenceConfig), + candidateEffectiveGroups = _movement.BuildEffectiveGroupVector(first.Plan.ProbeConfig), + inactiveGroups = _movement.BuildInactiveGroupList(), + first.Plan.ProbeType, + first.Plan.HypothesisLabel, + actualCandidateKld = first.ProbeSnapshot?.Kld, + actualTwinKld = first.ReferenceSnapshot?.Kld, + actualGainOrHarm = first.ActualGainVsTwin, + adjustmentReason = first.ReferenceSnapshot != null && first.ProbeSnapshot != null + ? "measured-actual-counterfactual-effect" + : "prediction-space-gap-fallback", + groups = probeGroups.Select(ToGroupLog).ToList() + }, JsonOptions); + + if (!isNew) + db.AnomalyInteractionRuleGroupStates.RemoveRange(rule.GroupStates); + + rule.GroupStates = probeGroups.Select((x, i) => new AnomalyInteractionRuleGroupState + { + RuleId = rule.Id, + TensorGroupId = x.Group.UniqueId, + CandidateQuantId = x.CandidateQuantId, + ReferenceQuantId = x.ReferenceQuantId, + Movement = x.Movement.ToString(), + SortOrder = i + }).ToList(); + + upserted.Add(rule); + } + + await db.SaveChangesAsync(ct); + return upserted; + } + + public async Task> LoadApplicableRulesAsync(CancellationToken ct) + { + await using var db = new MagicQuantContext(); + var scope = await ResolveScopeAsync(db, ct); + var minConfidence = Config.AnomalyDetection.MinRuleConfidenceToApply; + + var rules = await db.AnomalyInteractionRules + .AsNoTracking() + .Include(x => x.GroupStates) + .Where(x => x.ArchitectureFamilyId == scope.ArchitectureFamilyId) + .Where(x => x.TensorGroupProfileId == scope.TensorGroupProfileId) + .Where(x => x.AiModelHashId == scope.AiModelHashId) + .Where(x => x.ImatrixDefinitionId == scope.ImatrixDefinitionId) + .Where(x => x.BenchmarkCategory == (byte)BenchmarkCategory.General) + .Where(x => x.RuleStatus == AnomalyRuleStatus.Confirmed.ToString()) + .Where(x => x.Confidence >= minConfidence) + .Where(x => x.RuleDirection == AnomalyRuleDirection.Beneficial.ToString() || x.RuleDirection == AnomalyRuleDirection.Harmful.ToString()) + .ToListAsync(ct); + + return rules + .Where(_movement.IsContextualQuantizedRule) + .ToList(); + } + + public async Task> LoadExistingRuleSuppressionKeysAsync(CancellationToken ct) + { + await using var db = new MagicQuantContext(); + var scope = await ResolveScopeAsync(db, ct); + + var rows = await db.AnomalyInteractionRules + .AsNoTracking() + .Where(x => x.ArchitectureFamilyId == scope.ArchitectureFamilyId) + .Where(x => x.TensorGroupProfileId == scope.TensorGroupProfileId) + .Where(x => x.AiModelHashId == scope.AiModelHashId) + .Where(x => x.ImatrixDefinitionId == scope.ImatrixDefinitionId) + .Where(x => x.BenchmarkCategory == (byte)BenchmarkCategory.General) + .Where(x => x.RuleStatus != AnomalyRuleStatus.Retired.ToString()) + .Select(x => new { x.ReferenceContextKey, x.GroupSetHash }) + .ToListAsync(ct); + + return rows + .Select(x => BuildRuleSuppressionKey(x.ReferenceContextKey, x.GroupSetHash)) + .ToHashSet(StringComparer.Ordinal); + } + + public string BuildRuleSuppressionKey(TensorConfig reference, IReadOnlyList groups) + => BuildRuleSuppressionKey(_movement.ReferenceContextKey(reference), _movement.BuildChangedGroupHash(groups)); + + public async Task HasSuppressionOrRuleAsync( + TensorConfig reference, + IReadOnlyList groups, + CancellationToken ct) + { + var keys = await LoadExistingRuleSuppressionKeysAsync(ct); + return keys.Contains(BuildRuleSuppressionKey(reference, groups)); + } + + private static string BuildRuleSuppressionKey(string referenceContextKey, string groupSetHash) + => $"context={referenceContextKey}|groups={groupSetHash}"; + + private async Task ResolveScopeAsync(MagicQuantContext db, CancellationToken ct) + { + uint aiModelHashId = await ArchitectureFamilyService.ResolveScopedAiModelHashIdAsync(db, ct); + int? imatrixId = await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, aiModelHashId, createIfMissing: false, ct); + return new AnomalyScope( + TensorGroupProfileService.RequireCurrentArchitectureFamilyId(), + TensorGroupProfileService.RequireCurrentProfileId(), + aiModelHashId, + imatrixId); + } + + private static async Task EnsureTensorComboAsync(MagicQuantContext db, TensorConfig config, CancellationToken ct) + { + var combo = await db.TensorCombos.FirstOrDefaultAsync(x => + x.BaseQuant == config.BaseQuant && + x.Embeddings == config.Embeddings && + x.LmHead == config.LmHead && + x.AttnQ == config.AttnQ && + x.AttnKV == config.AttnKV && + x.AttnOutput == config.AttnOutput && + x.FfnUpGate == config.FfnUpGate && + x.FfnDown == config.FfnDown && + x.MoeExperts == config.MoeExperts && + x.MoeRouter == config.MoeRouter, + ct); + + if (combo != null) + return combo; + + combo = new TensorCombo(config); + db.TensorCombos.Add(combo); + await db.SaveChangesAsync(ct); + return combo; + } + + private static object ToGroupLog(AnomalyChangedGroup x) + { + return new + { + groupId = x.Group.UniqueId, + group = x.Group.Name, + shortCode = x.Group.ShortCode, + candidateQuantId = x.CandidateQuantId, + candidateQuant = BaselineQuants.FromId(x.CandidateQuantId).Names[0], + referenceQuantId = x.ReferenceQuantId, + referenceQuant = BaselineQuants.FromId(x.ReferenceQuantId).Names[0], + movement = x.Movement.ToString() + }; + } + + private static ulong ComputeSizeSavings(BenchmarkSnapshotRecord? reference, BenchmarkSnapshotRecord? probe) + { + if (reference == null || probe == null || reference.SizeBytes <= probe.SizeBytes) + return 0UL; + + return reference.SizeBytes - probe.SizeBytes; + } + + private static string BuildRuleKey(AnomalyProbeResult result) + { + var groups = result.Plan.ProbeGroups + .OrderBy(x => x.Group.UniqueId) + .Select(x => $"{x.Group.UniqueId}:{x.ReferenceQuantId}->{x.CandidateQuantId}"); + + return $"{result.RuleDirection}|ref={TensorConfigIdentity.ToKey(result.Plan.ReferenceConfig)}|probe={TensorConfigIdentity.ToKey(result.Plan.ProbeConfig)}|{string.Join("|", groups)}"; + } + + private static string ResolveRuleType(IReadOnlyList rows) + { + var first = rows[0]; + if (first.RuleDirection == AnomalyRuleDirection.SuppressionOnly) + return "SuppressionOnly"; + + return first.Plan.ProbeType switch + { + "single" => "SingleGroupInversion", + "pair" => "PairSynergy", + "composition" => rows.Any(x => x.RuleDirection == AnomalyRuleDirection.Harmful) ? "HarmfulInterferenceComposition" : "CounterfactualSynergyComposition", + "context-transfer" => rows.Any(x => x.RuleDirection == AnomalyRuleDirection.Harmful) ? "HarmfulContextTransfer" : "ContextTransfer", + "context-rank-pair" => rows.Any(x => x.RuleDirection == AnomalyRuleDirection.Harmful) ? "HarmfulContextRankReversal" : "ContextRankPair", + "confirmed-neighborhood" => rows.Any(x => x.Classification == AnomalyProbeClassification.ContaminatingPassenger) ? "ContaminatingPassenger" : "ConfirmedAnomalyNeighborhood", + "full" => rows.Any(x => x.Plan.ProbeGroups.Count >= 3) ? "HigherOrderSynergy" : "PairSynergy", + "leave-one-out" => "HigherOrderSynergy", + _ => rows.Any(x => x.Classification == AnomalyProbeClassification.ContaminatingPassenger) ? "ContaminatingPassenger" : "ContextOnly" + }; + } + + private static double ComputeConfidence(IReadOnlyList rows) + { + if (rows.Count == 0) + return 0d; + + double accepted = rows.Count(x => x.Accepted) / (double)rows.Count; + double gain = Math.Clamp(rows.Max(x => Math.Abs(x.ActualGainVsTwin)) / Math.Max(Config.AnomalyDetection.MinActualGainVsTwinKld, 1e-9), 0d, 2d) / 2d; + return Math.Clamp((accepted * 0.70d) + (gain * 0.30d), 0d, 1d); + } + + private static double ComputePredictionAdjustment(AnomalyProbeResult result, double confidence) + { + var cfg = Config.AnomalyDetection; + var synergy = Config.SynergyDetection; + + // Prediction KLD is rank-relative. Even when real probes confirm a beneficial + // counterfactual effect, the adjustment is sized by how far the candidate must + // move in prediction space to sit below its virtual/same-context twin. Actual + // KLD affects confidence/classification, not raw numeric subtraction. + double baseGap = result.Plan.Seed.PredictionSpaceGapVsTwin; + double required = result.RuleDirection switch + { + AnomalyRuleDirection.Beneficial => -(Math.Max(0d, baseGap) + cfg.PredictionSpaceViolationMargin), + AnomalyRuleDirection.Harmful => Math.Max(cfg.PredictionSpaceViolationMargin, Math.Abs(baseGap) + cfg.PredictionSpaceViolationMargin), + _ => 0d + }; + + double multiplier = result.Plan.SeedClass switch + { + AnomalySeedClass.SynergyCompositionProbe => synergy.SameSelectedGroupsConfidenceMultiplier, + AnomalySeedClass.SynergyTransferProbe => synergy.SameSelectedGroupsConfidenceMultiplier, + AnomalySeedClass.ConfirmedAnomalyNeighborhoodProbe => synergy.SameSelectedGroupsConfidenceMultiplier, + _ => synergy.ExactContextConfidenceMultiplier + }; + + if (result.Classification == AnomalyProbeClassification.ContaminatingPassenger) + multiplier *= synergy.ContaminationPenaltyConfidenceMultiplier; + + double adjusted = required * confidence * cfg.AnomalyAdjustmentShrinkFactor * multiplier; + if (adjusted < 0d) + return Math.Max(adjusted, -Math.Min(cfg.MaxNegativeAdjustmentKld, synergy.MaxNegativeAdjustmentKld)); + + return Math.Min(adjusted, cfg.MaxPositiveAdjustmentKld); + } + + private readonly record struct AnomalyScope(int ArchitectureFamilyId, int TensorGroupProfileId, uint AiModelHashId, int? ImatrixDefinitionId); +} diff --git a/src/MagicQuant/Services/AnomalyWorkflowService.cs b/src/MagicQuant/Services/AnomalyWorkflowService.cs new file mode 100644 index 0000000..1470a3f --- /dev/null +++ b/src/MagicQuant/Services/AnomalyWorkflowService.cs @@ -0,0 +1,2918 @@ +using System.Diagnostics; +using System.Globalization; +using System.Numerics; +using System.Text.Json; +using DuckDB.NET.Data; +using MagicQuant.Configuration; +using MagicQuant.Helpers; +using MagicQuant.Models; +using Microsoft.EntityFrameworkCore; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class AnomalyWorkflowService +{ + private const int DuckSmokeScanLimit = 0; // 0 means scan all predicted DuckDB rows; anomaly smoke must not be top-rank truncated. + + private static readonly JsonSerializerOptions JsonOptions = new() + { + WriteIndented = true + }; + + private readonly RemainingCombinationStore _store; + private readonly HybridBenchmarkRepository _repository; + private readonly QuantizationService _quantizationService; + private readonly QuantFidelityComparerService _movement; + private readonly AnomalyRuleRepository _rules; + private readonly AnomalyAdjustedPredictionService _adjuster; + private AnomalySmokeScanDiagnostics? _lastDuckSmokeDiagnostics; + + public AnomalyWorkflowService( + RemainingCombinationStore store, + HybridBenchmarkRepository repository, + QuantizationService quantizationService) + { + _store = store; + _repository = repository; + _quantizationService = quantizationService; + _movement = new QuantFidelityComparerService(); + _rules = new AnomalyRuleRepository(_movement); + _adjuster = new AnomalyAdjustedPredictionService(store); + } + + public async Task RunAsync( + IReadOnlyCollection pureBaselineSnapshots, + CancellationToken ct = default) + { + if (!Config.AnomalyDetection.Enabled || Config.AnomalyDetection.MaxAnomalyRefinementRounds <= 0) + { + AnsiConsole.MarkupLine("[grey]Anomaly detection disabled by config.[/]"); + return new AnomalyRunResult(); + } + + AnsiConsole.Write(new Rule("[yellow]Counterfactual Anomaly Smoke / Probe Pass[/]") { Justification = Justify.Left }); + + var session = await _rules.StartSessionAsync("prediction-guided-selection", ct); + try + { + var historical = await DetectHistoricalSmokeAsync(ct); + var duck = await DetectDuckSmokeAsync(ct); + var smoke = historical + .Concat(duck) + .GroupBy(x => TensorConfigIdentity.ToKey(x.CandidateConfig), StringComparer.Ordinal) + .Select(g => g.OrderByDescending(x => x.IsConfirmedFromHistory).ThenByDescending(x => x.SmokeScore).First()) + .OrderByDescending(x => x.IsConfirmedFromHistory) + .ThenByDescending(x => x.SmokeScore) + .Take(Config.AnomalyDetection.MaxSmokeCandidatesPerReferenceZone * Math.Max(1, RuntimeSearchSpace.GetActiveCombinationBaselines().Count)) + .ToList(); + + WriteSmokeConsoleSummary(historical.Count, duck.Count, smoke); + await WriteJsonAsync("magicquant-anomaly-smoke-scan.json", new + { + generatedAtUtc = DateTime.UtcNow, + historicalCount = historical.Count, + duckPredictionSpaceCount = duck.Count, + selectedSmokeCount = smoke.Count, + duckDiagnostics = _lastDuckSmokeDiagnostics, + smoke = smoke.Select(ToSmokeLog).ToList() + }, ct); + + await WriteJsonAsync("magicquant-anomaly-seeds.json", smoke.Select(ToSmokeLog).ToList(), ct); + await WriteJsonAsync("magicquant-synergy-smoke-scan.json", new + { + generatedAtUtc = DateTime.UtcNow, + terminology = "CounterfactualSynergy smoke. Anomaly names are retained as backward-compatible aliases.", + historicalCount = historical.Count, + duckPredictionSpaceCount = duck.Count, + selectedSmokeCount = smoke.Count, + duckDiagnostics = _lastDuckSmokeDiagnostics, + smoke = smoke.Select(ToSmokeLog).ToList() + }, ct); + + var planningDiagnostics = new ProbePlanningDiagnostics(); + var probes = await PlanProbesAsync(smoke, planningDiagnostics, ct); + + var results = await ValidateProbesAsync(probes, ct); + + var exploratoryPairProbes = await PlanExploratoryContextPairProbesAsync(planningDiagnostics, ct); + if (exploratoryPairProbes.Count > 0) + { + probes = probes.Concat(exploratoryPairProbes).ToList(); + var exploratoryPairResults = await ValidateProbesAsync(exploratoryPairProbes, ct); + results = results.Concat(exploratoryPairResults).ToList(); + } + + int remainingTransferBudget = Math.Max( + 0, + Config.SynergyDetection.MaxTotalTransferProbesPerRun - exploratoryPairProbes.Count); + var transferProbes = await PlanSynergyTransferProbesAsync(results, planningDiagnostics, remainingTransferBudget, ct); + if (transferProbes.Count > 0) + { + probes = probes.Concat(transferProbes).ToList(); + var transferResults = await ValidateProbesAsync(transferProbes, ct); + results = results.Concat(transferResults).ToList(); + } + + var expansionProbes = await PlanConfirmedAnomalyExpansionProbesAsync(results, planningDiagnostics, ct); + if (expansionProbes.Count > 0) + { + probes = probes.Concat(expansionProbes).ToList(); + var expansionResults = await ValidateProbesAsync(expansionProbes, ct); + results = results.Concat(expansionResults).ToList(); + } + + var compositionDiagnostics = new List(); + var compositionProbes = await PlanSynergyCompositionProbesAsync(results, planningDiagnostics, compositionDiagnostics, ct); + if (compositionProbes.Count > 0) + { + probes = probes.Concat(compositionProbes).ToList(); + var compositionResults = await ValidateProbesAsync(compositionProbes, ct); + results = results.Concat(compositionResults).ToList(); + compositionDiagnostics = BuildCompositionDiagnostics(compositionProbes, compositionResults); + } + + await WriteJsonAsync("magicquant-synergy-composition-probes.json", new + { + generatedAtUtc = DateTime.UtcNow, + compositionProbes = compositionDiagnostics + }, ct); + + await WriteJsonAsync("magicquant-anomaly-probes.json", new + { + generatedAtUtc = DateTime.UtcNow, + planningDiagnostics, + probes = probes.Select(ToProbeLog).ToList() + }, ct); + + await _rules.PersistProbeResultsAsync(session.Id, results, ct); + var upsertedRules = await _rules.UpsertRulesFromResultsAsync(results, ct); + var applicableRules = await _rules.LoadApplicableRulesAsync(ct); + var bestAnomaly = BuildBestConfirmedAnomalyReconciliation(results, Array.Empty()); + WriteBestAnomalyConsoleLog(bestAnomaly); + var adjustment = await _adjuster.ApplyAsync(applicableRules, ct); + + await WriteJsonAsync("magicquant-anomaly-rules.json", new + { + generatedAtUtc = DateTime.UtcNow, + upserted = upsertedRules.Select(ToRuleLog).ToList(), + applicable = applicableRules.Select(ToRuleLog).ToList() + }, ct); + await WriteJsonAsync("magicquant-synergy-templates.json", new + { + generatedAtUtc = DateTime.UtcNow, + templates = applicableRules.Select(ToSynergyTemplateLog).ToList() + }, ct); + await WriteJsonAsync("magicquant-synergy-probes.json", results.Select(ToResultLog).ToList(), ct); + await WriteJsonAsync("magicquant-synergy-transfer-probes.json", results + .Where(x => x.Plan.SeedClass == AnomalySeedClass.SynergyTransferProbe || x.Plan.SeedClass == AnomalySeedClass.ConfirmedAnomalyNeighborhoodProbe) + .Select(ToResultLog).ToList(), ct); + + var wingSummary = BuildSynergyWingSummary(smoke, results, adjustment); + WriteSynergyWingConsoleSummary(wingSummary); + await WriteJsonAsync("magicquant-synergy-wing-summary.json", wingSummary, ct); + + await WriteJsonAsync("magicquant-anomaly-adjusted-predictions-summary.json", adjustment, ct); + await WriteJsonAsync("magicquant-synergy-adjusted-predictions-summary.json", adjustment, ct); + await WriteFinalManifestAsync("magicquant.anomalies.json", new + { + generatedAtUtc = DateTime.UtcNow, + smoke = smoke.Select(ToSmokeLog).ToList(), + probes = probes.Select(ToProbeLog).ToList(), + results = results.Select(ToResultLog).ToList(), + rules = applicableRules.Select(ToRuleLog).ToList(), + bestConfirmedAnomaly = bestAnomaly, + adjustment + }, ct); + await WriteFinalManifestAsync("magicquant.prediction-audit.json", new + { + generatedAtUtc = DateTime.UtcNow, + note = "BaseRankSafeKld is normal PAVA gravity. FinalPredictedKld is BaseRankSafeKld plus scoped anomaly adjustments. Global PAVA is not rerun after anomaly exceptions.", + adjustment + }, ct); + + return new AnomalyRunResult + { + SmokeCandidates = smoke, + ProbePlans = probes, + ProbeResults = results, + AdjustmentSummary = adjustment, + BestAnomalyReconciliation = bestAnomaly + }; + } + finally + { + await _rules.CompleteSessionAsync(session.Id, ct); + } + } + + + private async Task> DetectHistoricalSmokeAsync(CancellationToken ct) + { + var snapshots = await LoadAllCurrentBenchmarkSnapshotsAsync(ct); + var byKey = snapshots.ToDictionary(x => TensorConfigIdentity.ToKey(x.Config), StringComparer.Ordinal); + await EmitQ8ContextReferenceDriftDiagnosticsAsync(byKey, ct); + var predictionLookup = await LoadPredictionLookupAsync(ct); + var smoke = new List(); + int skippedIsolation = 0; + int skippedSparse = 0; + int skippedNonContextualTwin = 0; + int skippedMixed = 0; + int contextualScanned = 0; + + foreach (var candidate in snapshots.Where(x => !TensorConfigIdentity.IsPureBaseline(x.Config))) + { + if (ShouldSkipInvalidContextualAnomalyConfig(candidate.Config, "history", out var skipReason)) + { + if (skipReason.Contains("SparseActiveGroup", StringComparison.OrdinalIgnoreCase)) + { + skippedSparse++; + LogHistoricalSparseCandidateIgnored(candidate.Config, skipReason, skippedSparse); + } + else + { + skippedIsolation++; + LogSkippedInvalidContextualAnomalyConfig("history", candidate.Config, skipReason, skippedIsolation); + } + continue; + } + + contextualScanned++; + var twinConfig = _movement.BuildBaseContextTwin(candidate.Config); + if (TensorConfigIdentity.ToKey(twinConfig) == TensorConfigIdentity.ToKey(candidate.Config)) + continue; + + if (ShouldSkipInvalidContextualAnomalyConfig(twinConfig, "history-twin", out var twinSkipReason)) + { + skippedNonContextualTwin++; + LogSkippedInvalidContextualAnomalyConfig("history-twin", twinConfig, twinSkipReason, skippedNonContextualTwin); + continue; + } + + var movement = _movement.Analyze(twinConfig, candidate.Config); + if (movement.Classification != AnomalyMovementClassification.MonotoneDowngrade) + { + if (movement.Classification == AnomalyMovementClassification.MixedTrade) + { + skippedMixed++; + if (Config.AnomalyDetection.VerboseAnomalyLogging && skippedMixed <= 12) + { + AnsiConsole.MarkupLine("[grey]Ignored anomaly smoke:[/] classification=MixedTrade reason=normal protect/compress frontier behavior"); + } + } + continue; + } + + if (movement.DowngradeCount > Config.AnomalyDetection.MaxProbeGroupCount) + continue; + + string explicitTwinKey = TensorConfigIdentity.ToKey(twinConfig); + var sparseTwinConfig = BuildSparsePureContext(twinConfig.BaseQuant); + string sparseTwinKey = TensorConfigIdentity.ToKey(sparseTwinConfig); + bool searchedExplicit = true; + bool searchedSparse = true; + byKey.TryGetValue(explicitTwinKey, out var explicitTwin); + byKey.TryGetValue(sparseTwinKey, out var sparseTwin); + + var twin = explicitTwin ?? sparseTwin; + string twinLookupMode = explicitTwin != null + ? (sparseTwin != null ? "explicit-context-preferred; sparse-pure-also-found" : "explicit-context-found") + : sparseTwin != null + ? "sparse-pure-fallback-found" + : "missing; searched-explicit-context-and-sparse-pure"; + + if (Config.AnomalyDetection.VerboseAnomalyLogging) + { + AnsiConsole.MarkupLine($"[grey]Historical twin lookup:[/] candidate={Markup.Escape(candidate.DisplayName)} searchedExplicitContext={searchedExplicit} searchedSparsePure={searchedSparse} mode={Markup.Escape(twinLookupMode)}"); + } + + if (twin != null && ShouldSkipInvalidContextualAnomalyConfig(twin.Config, "history-existing-twin", out var existingTwinSkipReason)) + { + skippedNonContextualTwin++; + LogSkippedInvalidContextualAnomalyConfig("history-existing-twin", twin.Config, existingTwinSkipReason, skippedNonContextualTwin); + continue; + } + + predictionLookup.TryGetValue(TensorConfigIdentity.ToKey(candidate.Config), out var candidatePrediction); + predictionLookup.TryGetValue(explicitTwinKey, out var twinPrediction); + if (twinPrediction == null) + predictionLookup.TryGetValue(sparseTwinKey, out twinPrediction); + + bool confirmed = twin != null && + candidate.SizeBytes <= twin.SizeBytes && + twin.Kld - candidate.Kld >= Config.AnomalyDetection.MinActualGainVsTwinKld; + + if (!confirmed && twin != null && twin.Kld <= candidate.Kld) + continue; + + ulong? predictedCandidateSize = candidatePrediction?.PredictedSizeBytes; + ulong? predictedTwinSize = twinPrediction?.PredictedSizeBytes; + ulong? predictedSavings = predictedCandidateSize.HasValue && predictedTwinSize.HasValue && predictedTwinSize.Value >= predictedCandidateSize.Value + ? predictedTwinSize.Value - predictedCandidateSize.Value + : null; + ulong? actualSavings = twin != null && twin.SizeBytes >= candidate.SizeBytes ? twin.SizeBytes - candidate.SizeBytes : null; + + smoke.Add(new AnomalySmokeCandidate + { + Source = "history", + CandidateConfig = candidate.Config, + TwinConfig = twinConfig, + Movement = movement, + CandidatePredictedKld = candidatePrediction?.BaseRankSafeKld ?? candidatePrediction?.FinalPredictedKld ?? candidate.Kld, + TwinPredictedKld = twinPrediction?.BaseRankSafeKld ?? twinPrediction?.FinalPredictedKld ?? twin?.Kld ?? candidate.Kld, + CandidatePredictedSizeBytes = predictedCandidateSize, + TwinPredictedSizeBytes = predictedTwinSize, + PredictedSizeSavingsBytes = predictedSavings, + ActualSizeSavingsBytes = actualSavings, + PlannedProbeWillMeasureSize = twin == null || actualSavings == null, + TwinLookupMode = twinLookupMode, + TwinFoundInLookupDictionary = twinPrediction != null, + PredictionSpaceGapVsTwin = (candidatePrediction?.BaseRankSafeKld ?? candidate.Kld) - (twinPrediction?.BaseRankSafeKld ?? twin?.Kld ?? candidate.Kld), + CandidatePredictionRank = candidatePrediction?.PredictionRank, + TwinPredictionRank = twinPrediction?.PredictionRank, + SmokeScore = confirmed ? 1_000_000d : 100d, + SmokeStrength = confirmed ? "ConfirmedHistory" : "HistoricalMissingTwin", + SeedClass = confirmed ? AnomalySeedClass.ConfirmedHistoricalCounterfactual : AnomalySeedClass.HistoricalMissingTwin, + HasActualTwin = twin != null, + CandidateActualKld = candidate.Kld, + TwinActualKld = twin?.Kld, + CandidateActualSizeBytes = candidate.SizeBytes, + TwinActualSizeBytes = twin?.SizeBytes, + IsConfirmedFromHistory = confirmed, + Message = confirmed + ? "Existing explicit contextual quantized benchmark history contains a monotone downgrade candidate that beats its higher-bit twin." + : "Existing explicit contextual quantized benchmark history has monotone downgrade smoke but the exact twin is missing." + }); + } + + if (skippedIsolation > 0 || skippedNonContextualTwin > 0 || skippedSparse > 0) + { + AnsiConsole.MarkupLine( + $"[yellow]Historical contextual anomaly scan:[/] contextualScanned={contextualScanned:N0} skippedIsolation={skippedIsolation:N0} sparseIgnored={skippedSparse:N0} skippedTwins={skippedNonContextualTwin:N0}"); + } + + return smoke; + } + + + private async Task> DetectDuckSmokeAsync(CancellationToken ct) + { + var totalClock = Stopwatch.StartNew(); + var loadClock = Stopwatch.StartNew(); + var rows = await LoadPredictionRowsAsync(DuckSmokeScanLimit, ct); + loadClock.Stop(); + + var lookupClock = Stopwatch.StartNew(); + var lookupRows = new Dictionary(StringComparer.Ordinal); + var candidateRows = new Dictionary(StringComparer.Ordinal); + var result = new List(); + var rejected = new List(); + var closestGapFailures = new List(); + var existingKeys = await _rules.LoadExistingRuleSuppressionKeysAsync(ct); + + int skippedIsolation = 0; + int skippedSparse = 0; + int normalizedSparse = 0; + int skippedPure = 0; + int skippedMixed = 0; + int skippedMovement = 0; + int skippedNoTwin = 0; + int skippedSavings = 0; + int skippedGap = 0; + int contextualScanned = 0; + int twinLookupCount = 0; + int dictionaryTwinHits = 0; + int fallbackDbTwinLookups = 0; + + foreach (var row in rows) + { + if (!_movement.TryNormalizeSparseDuckRowToActivatedContext(row.Config, out var activated, out var wasSparse, out var normalizeReason)) + { + if (normalizeReason.Contains("SparseActiveGroup", StringComparison.OrdinalIgnoreCase)) + skippedSparse++; + else + skippedIsolation++; + + LogSkippedInvalidContextualAnomalyConfig("duckdb", row.Config, normalizeReason, skippedIsolation + skippedSparse); + continue; + } + + if (wasSparse) + normalizedSparse++; + + var normalizedRow = row with { Config = activated }; + AddOrPreferBetterPredictionRow(lookupRows, normalizedRow); + + // Keep the sparse pure carrier in the lookup as an optional prediction source, + // but never let it become a contextual anomaly identity or probe/rule row. + if (TensorConfigIdentity.IsPureBaseline(row.Config)) + AddOrPreferBetterPredictionRow(lookupRows, row); + + if (TensorConfigIdentity.ToKey(activated) == TensorConfigIdentity.ToKey(_movement.BuildBaseContextTwin(activated))) + { + skippedPure++; + continue; + } + + AddOrPreferBetterPredictionRow(candidateRows, normalizedRow); + } + lookupClock.Stop(); + + var scanClock = Stopwatch.StartNew(); + foreach (var row in candidateRows.Values) + { + contextualScanned++; + var twin = _movement.BuildBaseContextTwin(row.Config); + string candidateKey = TensorConfigIdentity.ToKey(row.Config); + string twinKey = TensorConfigIdentity.ToKey(twin); + + if (ShouldSkipInvalidContextualAnomalyConfig(row.Config, "duckdb-normalized-candidate", out var candidateSkipReason)) + { + skippedIsolation++; + AddRejectedPreview(rejected, row.Config, twin, null, null, null, null, null, "InvalidCandidate: " + candidateSkipReason, false, false); + LogSkippedInvalidContextualAnomalyConfig("duckdb-normalized-candidate", row.Config, candidateSkipReason, skippedIsolation); + continue; + } + + if (ShouldSkipInvalidContextualAnomalyConfig(twin, "duckdb-twin", out var twinSkipReason)) + { + skippedIsolation++; + AddRejectedPreview(rejected, row.Config, twin, null, null, null, null, null, "InvalidTwin: " + twinSkipReason, false, false); + LogSkippedInvalidContextualAnomalyConfig("duckdb-twin", twin, twinSkipReason, skippedIsolation); + continue; + } + + var movement = _movement.Analyze(twin, row.Config); + bool matchedConfirmedPattern = existingKeys.Contains(_rules.BuildRuleSuppressionKey(twin, movement.ChangedGroups)); + + if (movement.Classification == AnomalyMovementClassification.MixedTrade) + { + skippedMixed++; + AddRejectedPreview(rejected, row.Config, twin, movement, row, null, null, null, "MixedTrade", matchedConfirmedPattern, false); + if (Config.AnomalyDetection.VerboseAnomalyLogging && skippedMixed <= 12) + { + AnsiConsole.MarkupLine("[grey]Ignored anomaly smoke:[/] classification=MixedTrade reason=normal protect/compress frontier behavior"); + } + continue; + } + + if (movement.Classification != AnomalyMovementClassification.MonotoneDowngrade) + { + skippedMovement++; + AddRejectedPreview(rejected, row.Config, twin, movement, row, null, null, null, "MovementNotMonotoneDowngrade", matchedConfirmedPattern, false); + continue; + } + + if (movement.DowngradeCount <= 0 || movement.DowngradeCount > Config.AnomalyDetection.MaxProbeGroupCount) + { + skippedMovement++; + AddRejectedPreview(rejected, row.Config, twin, movement, row, null, null, null, "ChangedGroupBudgetExceeded", matchedConfirmedPattern, false); + continue; + } + + twinLookupCount++; + var twinRow = ResolveTwinFromLookupOnly(twin, lookupRows, out var twinLookupMode, out var twinFoundInDictionary); + if (twinRow == null) + { + skippedNoTwin++; + AddRejectedPreview(rejected, row.Config, twin, movement, row, null, null, null, "MissingTwinInPreloadedDictionary", matchedConfirmedPattern, false); + continue; + } + + if (twinFoundInDictionary) + dictionaryTwinHits++; + + if (twinRow.PredictedSizeBytes <= row.PredictedSizeBytes) + { + skippedSavings++; + AddRejectedPreview(rejected, row.Config, twin, movement, row, twinRow, null, null, "PredictedSizeSavingsNotPositive", matchedConfirmedPattern, true); + continue; + } + + ulong savingsBytes = twinRow.PredictedSizeBytes - row.PredictedSizeBytes; + double savingsPercent = savingsBytes * 100d / Math.Max(1d, twinRow.PredictedSizeBytes); + if (savingsPercent < Config.AnomalyDetection.MinPredictedSizeSavingsVsTwinPercent) + { + skippedSavings++; + AddRejectedPreview(rejected, row.Config, twin, movement, row, twinRow, savingsBytes, null, "PredictedSizeSavingsBelowThreshold", matchedConfirmedPattern, true); + continue; + } + + double gap = row.BaseRankSafeKld - twinRow.BaseRankSafeKld; + double score = ComputeSmokeScore(gap, savingsPercent, movement.DowngradeCount, row.PredictionRank, twinRow.PredictionRank); + bool gapCatastrophic = gap > Math.Max(Config.AnomalyDetection.MaxPredictionSpaceGapVsTwinKld, Config.SynergyDetection.MaxSmokeGapKld); + bool scoreTooLow = score < Config.SynergyDetection.MinSmokeScore && !matchedConfirmedPattern; + if (gapCatastrophic || scoreTooLow) + { + skippedGap++; + string reason = gapCatastrophic ? "PredictionSpaceGapTooLarge" : "BelowMinSmokeScore"; + var preview = AddRejectedPreview(rejected, row.Config, twin, movement, row, twinRow, savingsBytes, gap, reason, matchedConfirmedPattern, true); + closestGapFailures.Add(preview); + continue; + } + + + result.Add(new AnomalySmokeCandidate + { + Source = "duckdb-prediction-space", + CandidateConfig = row.Config, + TwinConfig = twin, + Movement = movement, + CandidatePredictedKld = row.BaseRankSafeKld, + TwinPredictedKld = twinRow.BaseRankSafeKld, + CandidatePredictedSizeBytes = row.PredictedSizeBytes, + TwinPredictedSizeBytes = twinRow.PredictedSizeBytes, + PredictedSizeSavingsBytes = savingsBytes, + PlannedProbeWillMeasureSize = true, + TwinLookupMode = twinLookupMode, + TwinFoundInLookupDictionary = twinFoundInDictionary, + PredictionSpaceGapVsTwin = gap, + CandidatePredictionRank = row.PredictionRank, + TwinPredictionRank = twinRow.PredictionRank, + SmokeScore = score, + SmokeStrength = gap <= 0d ? "Strong" : "Close", + SeedClass = AnomalySeedClass.PredictionSpaceSmoke, + MatchedConfirmedAnomalyPattern = matchedConfirmedPattern, + Message = "Prediction-space contextual monotone downgrade candidate is close enough to its higher-bit quantized twin to justify probes. Twin lookup was dictionary-only from the preloaded DuckDB row set." + }); + } + scanClock.Stop(); + totalClock.Stop(); + + var diagnostics = new AnomalySmokeScanDiagnostics + { + PredictedRowsScanned = rows.Count, + SparseRowsNormalized = normalizedSparse, + SparseRowsSkipped = skippedSparse, + Bf16ExactRowsSkipped = skippedIsolation, + PureReferenceRowsSkipped = skippedPure, + ContextualRowsScanned = contextualScanned, + TwinLookupCount = twinLookupCount, + DictionaryTwinHits = dictionaryTwinHits, + MissingTwins = skippedNoTwin, + FallbackDbTwinLookups = fallbackDbTwinLookups, + MovementNotMonotoneDowngrade = skippedMovement, + MixedTradeIgnored = skippedMixed, + SizeSavingsBelowThreshold = skippedSavings, + PredictionSpaceGapTooLarge = skippedGap, + QueuedSmokeCandidates = result.Count, + LoadPredictedRowsMs = loadClock.ElapsedMilliseconds, + BuildLookupDictionaryMs = lookupClock.ElapsedMilliseconds, + ScanRowsMs = scanClock.ElapsedMilliseconds, + RejectedPreview = rejected + .OrderBy(x => x.SortOrder) + .Take(25) + .Select(x => x.ToLog()) + .ToList(), + ClosestGapFailures = closestGapFailures + .OrderBy(x => x.PredictionSpaceGap ?? double.MaxValue) + .ThenByDescending(x => x.PredictedSizeSavingsBytes ?? 0UL) + .Take(10) + .Select(x => x.ToLog()) + .ToList() + }; + + _lastDuckSmokeDiagnostics = diagnostics; + + AnsiConsole.MarkupLine("[yellow]DuckDB contextual smoke scan:[/]"); + AnsiConsole.MarkupLine($"[grey] predicted rows scanned=[/] [cyan]{rows.Count:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] load predicted rows ms=[/] [cyan]{diagnostics.LoadPredictedRowsMs:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] build lookup dictionary ms=[/] [cyan]{diagnostics.BuildLookupDictionaryMs:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] scan rows ms=[/] [cyan]{diagnostics.ScanRowsMs:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] sparse rows normalized to explicit context=[/] [cyan]{normalizedSparse:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] sparse rows skipped=[/] [cyan]{skippedSparse:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] BF16/exact rows skipped=[/] [cyan]{skippedIsolation:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] pure/logical reference rows kept for lookup but skipped as smoke=[/] [cyan]{skippedPure:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] contextual quantized rows scanned=[/] [cyan]{contextualScanned:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] twin lookup count=[/] [cyan]{twinLookupCount:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] dictionary twin hits=[/] [cyan]{dictionaryTwinHits:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] missing twins=[/] [cyan]{skippedNoTwin:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] fallback DB twin lookups=[/] [cyan]{fallbackDbTwinLookups:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] movement not monotone downgrade=[/] [cyan]{skippedMovement:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] mixed trade ignored=[/] [cyan]{skippedMixed:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] size savings below threshold=[/] [cyan]{skippedSavings:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] prediction-space gap too large=[/] [cyan]{skippedGap:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] queued smoke candidates=[/] [cyan]{result.Count:N0}[/]"); + AnsiConsole.MarkupLine("[yellow]DuckDB synergy smoke timings:[/]"); + AnsiConsole.MarkupLine($"[grey] predicted row scan ms=[/] [cyan]{diagnostics.LoadPredictedRowsMs:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] virtual twin construction ms=[/] [cyan]0[/] [grey](computed inline while scanning)[/]"); + AnsiConsole.MarkupLine($"[grey] twin lookup ms=[/] [cyan]{diagnostics.ScanRowsMs:N0}[/] [grey](dictionary-only in normal operation)[/]"); + AnsiConsole.MarkupLine($"[grey] scoring ms=[/] [cyan]{diagnostics.ScanRowsMs:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] diagnostic formatting ms=[/] [cyan]deferred[/]"); + + if (result.Count == 0 && rows.Count > 0) + { + AnsiConsole.MarkupLine("[yellow]DuckDB contextual smoke scan produced zero candidates.[/] Top rejected-smoke previews and closest gap failures were written to magicquant-anomaly-smoke-scan-duckdb-diagnostics.json."); + foreach (var preview in closestGapFailures.OrderBy(x => x.PredictionSpaceGap ?? double.MaxValue).Take(10)) + { + AnsiConsole.MarkupLine($"[grey] rejected monotone gap:[/] candidate={Markup.Escape(preview.CandidateName)} twin={Markup.Escape(preview.TwinName)} gap={FmtNullable(preview.PredictionSpaceGap)} savings={FmtNullable(preview.PredictedSizeSavingsBytes)} reason={Markup.Escape(preview.RejectionReason)} matchedRule={preview.MatchedConfirmedAnomalyPattern}"); + } + } + + await WriteJsonAsync("magicquant-anomaly-smoke-scan-duckdb-diagnostics.json", new + { + generatedAtUtc = DateTime.UtcNow, + diagnostics, + queued = result.Select(ToSmokeLog).ToList() + }, ct); + + return result + .GroupBy(x => x.TwinConfig.BaseQuant) + .SelectMany(g => g.OrderByDescending(x => x.SmokeScore).Take(Config.AnomalyDetection.MaxSmokeCandidatesPerReferenceZone)) + .ToList(); + } + + + private async Task> PlanProbesAsync( + IReadOnlyList seeds, + ProbePlanningDiagnostics diagnostics, + CancellationToken ct) + { + var plans = new List(); + var seen = new HashSet(StringComparer.Ordinal); + var existingRuleKeys = await _rules.LoadExistingRuleSuppressionKeysAsync(ct); + diagnostics.ExistingRuleKeysLoaded = existingRuleKeys.Count; + + foreach (var seed in seeds) + { + bool invalidSeedCandidate = ShouldSkipInvalidContextualAnomalyConfig(seed.CandidateConfig, "probe-seed-candidate", out var seedCandidateReason); + bool invalidSeedTwin = ShouldSkipInvalidContextualAnomalyConfig(seed.TwinConfig, "probe-seed-twin", out var seedTwinReason); + if (invalidSeedCandidate || invalidSeedTwin) + { + diagnostics.SkippedInvalidMovement++; + string reason = invalidSeedCandidate ? seedCandidateReason : seedTwinReason; + AnsiConsole.MarkupLine($"[yellow]SkippedInvalidContextualAnomalyProbe:[/] source=probe-seed reason={Markup.Escape(reason)} candidate={Markup.Escape(HybridBenchmarkRepository.BuildDisplayName(seed.CandidateQuant))} twin={Markup.Escape(HybridBenchmarkRepository.BuildDisplayName(seed.TwinQuant))}"); + continue; + } + + var reference = _movement.CreateActivatedContextBlanket(seed.TwinConfig.BaseQuant); + _movement.EnsureAllActiveGroupsExplicit(reference, "probe-plan-reference"); + + var changed = _movement.Analyze(reference, seed.CandidateConfig) + .ChangedGroups + .Where(x => x.Movement == QuantMovementKind.Downgrade) + .OrderBy(x => x.Group.UniqueId) + .Take(Config.AnomalyDetection.MaxProbeGroupCount) + .ToList(); + + if (changed.Count == 0) + { + diagnostics.SkippedInvalidMovement++; + continue; + } + + var subsets = BuildProbeSubsets(changed); + int perSeed = 0; + foreach (var subset in subsets) + { + if (perSeed >= Config.AnomalyDetection.MaxProbesPerSeed || plans.Count >= Config.AnomalyDetection.MaxTotalProbesPerRun) + { + diagnostics.SkippedBudget++; + break; + } + + var probeConfig = reference; + foreach (var g in subset) + probeConfig = _movement.WithStoredSlot(probeConfig, g.Group, g.CandidateStoredSlot); + + if (ShouldSkipInvalidContextualAnomalyConfig(probeConfig, "probe-plan", out var probeSkipReason)) + { + diagnostics.SkippedInvalidMovement++; + AnsiConsole.MarkupLine($"[yellow]SkippedInvalidContextualAnomalyProbe:[/] source=probe-plan reason={Markup.Escape(probeSkipReason)} probe={Markup.Escape(HybridBenchmarkRepository.BuildDisplayName((HybridQuant)probeConfig))}"); + continue; + } + + _movement.EnsureAllActiveGroupsExplicit(probeConfig, "probe-plan-probe"); + + if (existingRuleKeys.Contains(_rules.BuildRuleSuppressionKey(reference, subset))) + { + diagnostics.SkippedExistingRuleOrSuppression++; + continue; + } + + string key = TensorConfigIdentity.ToKey(reference) + "=>" + TensorConfigIdentity.ToKey(probeConfig); + if (!seen.Add(key)) + { + diagnostics.SkippedDuplicate++; + continue; + } + + string probeType = ResolveProbeType(subset.Count, changed.Count); + var plan = new AnomalyProbePlan + { + Seed = seed, + ReferenceConfig = reference, + ProbeConfig = probeConfig, + ProbeGroups = subset, + ProbeType = probeType, + HypothesisLabel = _movement.DescribeGroups(subset), + SeedClass = seed.SeedClass, + ProbePriorityClass = probeType == "single" + ? AnomalySeedClass.ExploratorySingle + : probeType == "pair" + ? AnomalySeedClass.ExploratoryPair + : seed.SeedClass + }; + plans.Add(plan); + perSeed++; + diagnostics.ProbesQueued++; + } + + AnsiConsole.MarkupLine( + $"[yellow]Potential anomaly smoke:[/] seedClass={seed.SeedClass} classification={seed.Movement.Classification} candidate={Markup.Escape(HybridBenchmarkRepository.BuildDisplayName(seed.CandidateQuant))} " + + $"higher-bit twin={Markup.Escape(HybridBenchmarkRepository.BuildDisplayName(seed.TwinQuant))} changed groups={Markup.Escape(_movement.DescribeGroups(changed))} " + + $"upgradeCount={seed.Movement.UpgradeCount} downgradeCount={seed.Movement.DowngradeCount} " + + $"prediction-space gap={seed.PredictionSpaceGapVsTwin:0.000000} predicted size savings={FmtNullable(seed.PredictedSizeSavingsBytes)} actual size savings={FmtNullable(seed.ActualSizeSavingsBytes)} plannedProbeWillMeasureSize={seed.PlannedProbeWillMeasureSize} probes queued={perSeed:N0}"); + } + + AnsiConsole.MarkupLine( + $"[grey]Anomaly probe planning diagnostics:[/] existingRuleKeysLoaded={diagnostics.ExistingRuleKeysLoaded:N0} skippedExistingRuleOrSuppression={diagnostics.SkippedExistingRuleOrSuppression:N0} skippedDuplicate={diagnostics.SkippedDuplicate:N0} skippedInvalidMovement={diagnostics.SkippedInvalidMovement:N0} skippedBudget={diagnostics.SkippedBudget:N0} probesQueued={diagnostics.ProbesQueued:N0}"); + + return plans; + } + + + private async Task> PlanExploratoryContextPairProbesAsync( + ProbePlanningDiagnostics diagnostics, + CancellationToken ct) + { + var cfg = Config.SynergyDetection; + int limit = Math.Min( + Math.Max(0, cfg.MaxExploratoryContextPairsPerRun), + Math.Max(0, cfg.MaxTotalTransferProbesPerRun)); + if (!cfg.Enabled || !cfg.TransferProbeEnabled || !cfg.ExploratoryContextPairEnabled || limit <= 0) + return new List(); + + var requestedStrata = cfg.ExploratoryPairContextStrata.ToHashSet(StringComparer.OrdinalIgnoreCase); + var contexts = ResolveTransferTargetContexts(cfg.TransferProbeContextStrata) + .Where(x => requestedStrata.Contains(x.Stratum)) + .ToList(); + if (contexts.Count == 0 || cfg.ExploratoryPairBitRanges.Count == 0) + return new List(); + + var isolationPairs = await LoadExploratoryIsolationPairsAsync( + cfg.ExploratoryPairBitRanges.ToHashSet(), + ct); + if (isolationPairs.Count == 0) + { + AnsiConsole.MarkupLine("[grey]Exploratory context-rank pairs:[/] no non-equivalent same-bit isolation winner/size-matched-contender pairs were available."); + return new List(); + } + + var existingRuleKeys = await _rules.LoadExistingRuleSuppressionKeysAsync(ct); + var candidatesByContext = contexts.ToDictionary( + x => x, + x => isolationPairs + .Select(pair => TryBuildExploratoryContextPairPlan(x, pair, existingRuleKeys, diagnostics)) + .Where(x => x != null) + .Select(x => x!) + .OrderBy(x => x.Pair.Group.UniqueId) + .ThenByDescending(x => x.Pair.IsolationGap) + .ThenBy(x => x.IdentityKey, StringComparer.Ordinal) + .ToList()); + + var cursors = contexts.ToDictionary(x => x, _ => 0); + var plans = new List(); + var identities = new HashSet(StringComparer.Ordinal); + while (plans.Count < limit) + { + bool added = false; + foreach (var context in contexts) + { + var candidates = candidatesByContext[context]; + while (cursors[context] < candidates.Count) + { + var candidate = candidates[cursors[context]++]; + if (!identities.Add(candidate.IdentityKey)) + continue; + + plans.Add(candidate.Plan); + diagnostics.ProbesQueued++; + diagnostics.TransferProbesQueued++; + diagnostics.ExploratoryPairProbesQueued++; + added = true; + break; + } + + if (plans.Count >= limit) + break; + } + + if (!added) + break; + } + + int candidateCount = candidatesByContext.Values.Sum(x => x.Count); + diagnostics.SkippedBudget += Math.Max(0, candidateCount - plans.Count); + AnsiConsole.MarkupLine( + $"[yellow]Exploratory context-rank pairs:[/] isolationPairs=[cyan]{isolationPairs.Count:N0}[/] " + + $"strata=[cyan]{contexts.Count:N0}[/] candidates=[cyan]{candidateCount:N0}[/] " + + $"queued=[cyan]{plans.Count:N0}[/] budget=[cyan]{limit:N0}[/]"); + + foreach (var context in contexts) + { + int queued = plans.Count(x => x.ReferenceConfig.BaseQuant == context.QuantId); + AnsiConsole.MarkupLine( + $"[grey] rank-pair stratum={Markup.Escape(context.Stratum)} reference={Markup.Escape(SafeName(context.QuantId))} " + + $"candidates={candidatesByContext[context].Count:N0} queued={queued:N0}[/]"); + } + + return plans; + } + + private async Task> LoadExploratoryIsolationPairsAsync( + IReadOnlySet bitRanges, + CancellationToken ct) + { + var pairs = new List(); + var nativeExactScheme = TensorWeightScheme.GetCurrentNativePrecisionScheme(); + + foreach (var group in _movement.ActiveGroups.OrderBy(x => x.UniqueId)) + { + var observations = new List(); + // Deliberately include isolation-pruned candidates here. A candidate that loses in + // native/F16 surroundings is exactly the candidate that may reverse rank in a Q3 + // context; requiring it to survive that earlier pruning would make this probe blind. + // Invalid/unlearnable candidates still fall out because they have no isolation snapshot. + foreach (var baseline in RuntimeSearchSpace.GetRealExplicitCombinationCandidatesForGroup(group) + .Where(x => bitRanges.Contains(x.BitRange)) + .OrderBy(x => x.UniqueId)) + { + var isolation = HybridQuant.CreateExactBlanket( + BaselineQuants.Q8_0, + _movement.ActiveGroups, + nativeExactScheme); + isolation.SetLearnedCandidateOverride(group, baseline); + var snapshot = await _repository.LoadBenchmarkSnapshotAsync((TensorConfig)isolation, ct); + if (snapshot != null) + observations.Add(new ExploratoryIsolationObservation(group, baseline, snapshot)); + } + + var ordered = observations + .OrderBy(x => x.Snapshot.Kld) + .ThenBy(x => x.Snapshot.SizeBytes) + .ThenBy(x => x.Baseline.UniqueId) + .ToList(); + if (ordered.Count < 2) + continue; + + var winner = ordered[0]; + var contender = ordered + .Skip(1) + .Where(x => !IsolationOutcomesEquivalent(winner, x)) + .OrderBy(x => Math.Abs((double)x.Snapshot.SizeBytes - winner.Snapshot.SizeBytes)) + .ThenBy(x => x.Snapshot.Kld) + .ThenBy(x => x.Baseline.UniqueId) + .FirstOrDefault(); + if (contender == null) + continue; + + pairs.Add(new ExploratoryIsolationPair( + group, + winner, + contender, + Math.Max(0d, contender.Snapshot.Kld - winner.Snapshot.Kld))); + } + + return pairs; + } + + private static bool IsolationOutcomesEquivalent( + ExploratoryIsolationObservation left, + ExploratoryIsolationObservation right) + { + return left.Snapshot.SizeBytes == right.Snapshot.SizeBytes && + Math.Abs(left.Snapshot.Kld - right.Snapshot.Kld) <= 1e-12d; + } + + private ExploratoryContextPairCandidate? TryBuildExploratoryContextPairPlan( + SynergyTransferContext context, + ExploratoryIsolationPair pair, + IReadOnlySet existingRuleKeys, + ProbePlanningDiagnostics diagnostics) + { + if (!TryBuildControlledRankPairConfig( + context.QuantId, + pair.Group, + pair.Contender.Baseline.UniqueId, + pair.Winner.Baseline.UniqueId, + out var reference, + out var probe, + out var changed)) + { + diagnostics.SkippedInvalidMovement++; + return null; + } + + if (ShouldSkipInvalidContextualAnomalyConfig(reference, "exploratory-context-rank-reference", out _) || + ShouldSkipInvalidContextualAnomalyConfig(probe, "exploratory-context-rank-probe", out _)) + { + diagnostics.SkippedInvalidMovement++; + return null; + } + + if (existingRuleKeys.Contains(_rules.BuildRuleSuppressionKey(reference, changed))) + { + diagnostics.SkippedExistingRuleOrSuppression++; + return null; + } + + string identityKey = TensorConfigIdentity.ToKey(reference) + "=>" + TensorConfigIdentity.ToKey(probe); + var movement = _movement.Analyze(reference, probe); + var seed = new AnomalySmokeCandidate + { + Source = "exploratory-isolation-rank-transfer", + CandidateConfig = probe, + TwinConfig = reference, + Movement = movement, + CandidatePredictedKld = pair.Winner.Snapshot.Kld, + TwinPredictedKld = pair.Contender.Snapshot.Kld, + CandidatePredictedSizeBytes = pair.Winner.Snapshot.SizeBytes, + TwinPredictedSizeBytes = pair.Contender.Snapshot.SizeBytes, + PredictionSpaceGapVsTwin = pair.Winner.Snapshot.Kld - pair.Contender.Snapshot.Kld, + SmokeScore = 2_000_000d + pair.IsolationGap, + SmokeStrength = $"IsolationRankPair:{context.Stratum}", + SeedClass = AnomalySeedClass.SynergyTransferProbe, + MatchedConfirmedAnomalyPattern = false, + PlannedProbeWillMeasureSize = true, + Message = "Remeasures a same-bit native-isolation winner and its closest-size non-equivalent contender head-to-head inside a controlled surrounding-fidelity context." + }; + var plan = new AnomalyProbePlan + { + Seed = seed, + ReferenceConfig = reference, + ProbeConfig = probe, + ProbeGroups = changed, + ProbeType = "context-rank-pair", + HypothesisLabel = $"{pair.Group.Name}: isolation winner {pair.Winner.Baseline.Names[0]} vs closest-size contender {pair.Contender.Baseline.Names[0]} in {context.Stratum} {SafeName(context.QuantId)} blanket", + SeedClass = AnomalySeedClass.SynergyTransferProbe, + ProbePriorityClass = AnomalySeedClass.SynergyTransferProbe + }; + + return new ExploratoryContextPairCandidate(identityKey, pair, plan); + } + + internal static bool TryBuildControlledRankPairConfig( + byte targetContextQuantId, + TensorGroup group, + byte referenceCandidateQuantId, + byte probeCandidateQuantId, + out TensorConfig reference, + out TensorConfig probe, + out IReadOnlyList changedGroups) + { + var movementService = new QuantFidelityComparerService(); + reference = movementService.CreateActivatedContextBlanket(targetContextQuantId); + probe = reference; + changedGroups = Array.Empty(); + + if (!movementService.ActiveGroups.Any(x => x.UniqueId == group.UniqueId) || + referenceCandidateQuantId == probeCandidateQuantId || + BaselineQuants.IsNativeExactAlias(referenceCandidateQuantId) || + BaselineQuants.IsNativeExactAlias(probeCandidateQuantId)) + { + return false; + } + + byte referenceStored = BaselineQuants.EncodeTensorConfigGroupSlotBaselineId(referenceCandidateQuantId); + byte probeStored = BaselineQuants.EncodeTensorConfigGroupSlotBaselineId(probeCandidateQuantId); + reference = movementService.WithStoredSlot(reference, group, referenceStored); + probe = movementService.WithStoredSlot(probe, group, probeStored); + changedGroups = + [ + new AnomalyChangedGroup + { + Group = group, + ReferenceQuantId = referenceCandidateQuantId, + CandidateQuantId = probeCandidateQuantId, + ReferenceStoredSlot = referenceStored, + CandidateStoredSlot = probeStored, + Movement = movementService.Compare(referenceCandidateQuantId, probeCandidateQuantId) + } + ]; + return changedGroups[0].Movement != QuantMovementKind.Unknown; + } + + + private async Task> PlanSynergyTransferProbesAsync( + IReadOnlyList currentResults, + ProbePlanningDiagnostics diagnostics, + int availableBudget, + CancellationToken ct) + { + var cfg = Config.SynergyDetection; + if (!cfg.Enabled || !cfg.TransferProbeEnabled || cfg.MaxTotalTransferProbesPerRun <= 0 || availableBudget <= 0) + return new List(); + + var contexts = ResolveTransferTargetContexts(cfg.TransferProbeContextStrata); + if (contexts.Count == 0) + { + AnsiConsole.MarkupLine("[yellow]Synergy transfer probes skipped:[/] no configured context-stratum quant names resolved to active baselines."); + return new List(); + } + + var historicalRules = await _rules.LoadApplicableRulesAsync(ct); + var templates = BuildTransferTemplates(historicalRules, currentResults) + .Where(x => x.Confidence >= cfg.MinConfidenceToScheduleTransferProbe) + .GroupBy(x => x.Key, StringComparer.Ordinal) + .Select(g => g + .OrderByDescending(x => x.Confidence) + .ThenByDescending(x => x.ActualEffectMagnitude) + .First()) + .ToList(); + + if (templates.Count == 0) + { + AnsiConsole.MarkupLine("[grey]Synergy transfer probes:[/] no confirmed templates met the transfer confidence threshold."); + return new List(); + } + + var existingRuleKeys = await _rules.LoadExistingRuleSuppressionKeysAsync(ct); + var candidatesByContext = contexts.ToDictionary( + x => x, + x => BuildTransferCandidatesForContext(x, templates, existingRuleKeys, diagnostics)); + + int globalLimit = Math.Min(Math.Max(0, cfg.MaxTotalTransferProbesPerRun), Math.Max(0, availableBudget)); + int perTemplateLimit = Math.Max(1, cfg.MaxTransferProbesPerTemplate); + var cursors = contexts.ToDictionary(x => x, _ => 0); + var perTemplateCounts = new Dictionary(StringComparer.Ordinal); + var selectedKeys = new HashSet(StringComparer.Ordinal); + var plans = new List(); + + while (plans.Count < globalLimit) + { + bool addedInRound = false; + foreach (var context in contexts) + { + var candidates = candidatesByContext[context]; + while (cursors[context] < candidates.Count) + { + var candidate = candidates[cursors[context]++]; + int used = perTemplateCounts.GetValueOrDefault(candidate.TemplateKey); + if (used >= perTemplateLimit || !selectedKeys.Add(candidate.IdentityKey)) + continue; + + plans.Add(candidate.Plan); + perTemplateCounts[candidate.TemplateKey] = used + 1; + diagnostics.ProbesQueued++; + diagnostics.TransferProbesQueued++; + addedInRound = true; + break; + } + + if (plans.Count >= globalLimit) + break; + } + + if (!addedInRound) + break; + } + + int candidateCount = candidatesByContext.Values.Sum(x => x.Count); + diagnostics.SkippedBudget += Math.Max(0, candidateCount - plans.Count); + AnsiConsole.MarkupLine( + $"[yellow]Synergy context-transfer probes:[/] templates=[cyan]{templates.Count:N0}[/] " + + $"strata=[cyan]{contexts.Count:N0}[/] candidates=[cyan]{candidateCount:N0}[/] " + + $"queued=[cyan]{plans.Count:N0}[/] globalLimit=[cyan]{globalLimit:N0}[/] perTemplateLimit=[cyan]{perTemplateLimit:N0}[/]"); + + foreach (var context in contexts) + { + int queued = plans.Count(x => x.ReferenceConfig.BaseQuant == context.QuantId); + AnsiConsole.MarkupLine( + $"[grey] transfer stratum={Markup.Escape(context.Stratum)} reference={Markup.Escape(SafeName(context.QuantId))} " + + $"candidates={candidatesByContext[context].Count:N0} queued={queued:N0}[/]"); + } + + return plans; + } + + private List BuildTransferCandidatesForContext( + SynergyTransferContext context, + IReadOnlyList templates, + IReadOnlySet existingRuleKeys, + ProbePlanningDiagnostics diagnostics) + { + var candidates = new List(); + int targetTier = _movement.EffectiveTier(context.QuantId); + + foreach (var template in templates) + { + if (template.SourceReferenceQuantId == context.QuantId) + continue; + + if (!TryBuildControlledTransferConfig( + context.QuantId, + template.Groups.Select(x => (x.Group.UniqueId, x.CandidateQuantId)).ToList(), + out var reference, + out var probe, + out var changed) || + changed.Count > Config.AnomalyDetection.MaxProbeGroupCount) + { + continue; + } + + if (ShouldSkipInvalidContextualAnomalyConfig(probe, "synergy-context-transfer", out _)) + { + diagnostics.SkippedInvalidMovement++; + continue; + } + + if (existingRuleKeys.Contains(_rules.BuildRuleSuppressionKey(reference, changed))) + { + diagnostics.SkippedExistingRuleOrSuppression++; + continue; + } + + string identityKey = TensorConfigIdentity.ToKey(reference) + "=>" + TensorConfigIdentity.ToKey(probe); + var analyzedMovement = _movement.Analyze(reference, probe); + var seed = new AnomalySmokeCandidate + { + Source = $"synergy-template-transfer:{template.Source}", + CandidateConfig = probe, + TwinConfig = reference, + Movement = analyzedMovement, + SmokeScore = 1_000_000d + template.Confidence + template.ActualEffectMagnitude, + SmokeStrength = $"ControlledContextTransfer:{context.Stratum}", + SeedClass = AnomalySeedClass.SynergyTransferProbe, + MatchedConfirmedAnomalyPattern = true, + PlannedProbeWillMeasureSize = true, + Message = "Controlled blanket transfer probe: remeasures a confirmed tensor template under a different surrounding-fidelity context." + }; + + var plan = new AnomalyProbePlan + { + Seed = seed, + ReferenceConfig = reference, + ProbeConfig = probe, + ProbeGroups = changed, + ProbeType = "context-transfer", + HypothesisLabel = $"{_movement.DescribeGroups(changed)} in {context.Stratum} {SafeName(context.QuantId)} blanket", + SeedClass = AnomalySeedClass.SynergyTransferProbe, + ProbePriorityClass = AnomalySeedClass.SynergyTransferProbe + }; + + double candidateTierDistance = changed + .Select(x => Math.Abs(_movement.EffectiveTier(x.CandidateQuantId) - targetTier)) + .DefaultIfEmpty(int.MaxValue) + .Average(); + candidates.Add(new SynergyTransferCandidate( + template.Key, + identityKey, + plan, + changed.Count, + candidateTierDistance, + template.Confidence, + template.ActualEffectMagnitude)); + } + + return candidates + .OrderBy(x => x.GroupCount) + .ThenBy(x => x.CandidateTierDistance) + .ThenByDescending(x => x.Confidence) + .ThenByDescending(x => x.ActualEffectMagnitude) + .ThenBy(x => x.IdentityKey, StringComparer.Ordinal) + .ToList(); + } + + internal static bool TryBuildControlledTransferConfig( + byte targetContextQuantId, + IReadOnlyList<(byte TensorGroupId, byte CandidateQuantId)> templateGroups, + out TensorConfig reference, + out TensorConfig probe, + out IReadOnlyList changedGroups) + { + var movementService = new QuantFidelityComparerService(); + reference = movementService.CreateActivatedContextBlanket(targetContextQuantId); + probe = reference; + var changed = new List(); + var activeGroupsById = movementService.ActiveGroups.ToDictionary(x => x.UniqueId); + + foreach (var state in templateGroups.OrderBy(x => x.TensorGroupId)) + { + if (state.CandidateQuantId == targetContextQuantId || + !activeGroupsById.TryGetValue(state.TensorGroupId, out var group)) + { + continue; + } + + var movement = movementService.Compare(targetContextQuantId, state.CandidateQuantId); + if (movement == QuantMovementKind.Unknown || BaselineQuants.IsNativeExactAlias(state.CandidateQuantId)) + continue; + + byte candidateStored = BaselineQuants.EncodeTensorConfigGroupSlotBaselineId(state.CandidateQuantId); + changed.Add(new AnomalyChangedGroup + { + Group = group, + ReferenceQuantId = targetContextQuantId, + CandidateQuantId = state.CandidateQuantId, + ReferenceStoredSlot = BaselineQuants.EncodeTensorConfigGroupSlotBaselineId(targetContextQuantId), + CandidateStoredSlot = candidateStored, + Movement = movement + }); + probe = movementService.WithStoredSlot(probe, group, candidateStored); + } + + changedGroups = changed; + return changed.Count > 0; + } + + private List BuildTransferTemplates( + IReadOnlyList historicalRules, + IReadOnlyList currentResults) + { + var activeGroupsById = _movement.ActiveGroups.ToDictionary(x => x.UniqueId); + var knownQuantIds = BaselineQuants.All.Select(x => x.UniqueId).ToHashSet(); + var templates = new List(); + + foreach (var rule in historicalRules) + { + var groups = rule.GroupStates + .Where(x => activeGroupsById.ContainsKey(x.TensorGroupId)) + .Where(x => knownQuantIds.Contains(x.CandidateQuantId) && !BaselineQuants.IsNativeExactAlias(x.CandidateQuantId)) + .OrderBy(x => x.TensorGroupId) + .Select(x => new SynergyTransferTemplateGroup(activeGroupsById[x.TensorGroupId], x.CandidateQuantId)) + .ToList(); + if (groups.Count == 0) + continue; + + templates.Add(new SynergyTransferTemplate( + BuildTransferTemplateKey(groups), + $"historical-rule:{rule.Id:N}:{rule.RuleDirection}", + rule.ReferenceQuantId, + rule.Confidence, + Math.Abs(rule.MeanActualGainVsTwin), + groups)); + } + + double effectScale = Math.Max(Config.AnomalyDetection.MinActualGainVsTwinKld, 1e-9d); + foreach (var result in currentResults + .Where(x => x.Accepted && x.RuleDirection != AnomalyRuleDirection.SuppressionOnly) + .Where(x => x.ReferenceSnapshot != null && x.ProbeSnapshot != null)) + { + var groups = result.Plan.ProbeGroups + .Where(x => activeGroupsById.ContainsKey(x.Group.UniqueId)) + .Where(x => knownQuantIds.Contains(x.CandidateQuantId) && !BaselineQuants.IsNativeExactAlias(x.CandidateQuantId)) + .OrderBy(x => x.Group.UniqueId) + .Select(x => new SynergyTransferTemplateGroup(activeGroupsById[x.Group.UniqueId], x.CandidateQuantId)) + .ToList(); + if (groups.Count == 0) + continue; + + double confidence = Math.Clamp(0.70d + (0.15d * Math.Clamp(Math.Abs(result.ActualGainVsTwin) / effectScale, 0d, 2d)), 0d, 1d); + templates.Add(new SynergyTransferTemplate( + BuildTransferTemplateKey(groups), + $"current-probe:{result.Plan.ProbeType}:{result.RuleDirection}", + result.Plan.ReferenceConfig.BaseQuant, + confidence, + Math.Abs(result.ActualGainVsTwin), + groups)); + } + + return templates; + } + + private static string BuildTransferTemplateKey(IReadOnlyList groups) + => string.Join("|", groups.OrderBy(x => x.Group.UniqueId).Select(x => $"{x.Group.UniqueId}:{x.CandidateQuantId}")); + + private static List ResolveTransferTargetContexts(RuntimeSynergyTransferProbeContextStrataConfig strata) + { + var learnedContextIds = BaselineQuants + .GetLearningBaselines(RuntimeSearchSpace.HasUsableImatrix()) + .Select(x => x.UniqueId) + .ToHashSet(); + var byName = BaselineQuants.All + .SelectMany(x => x.Names.Select(name => (Name: name, Quant: x))) + .GroupBy(x => x.Name, StringComparer.OrdinalIgnoreCase) + .ToDictionary(x => x.Key, x => x.First().Quant, StringComparer.OrdinalIgnoreCase); + var contexts = new List(); + var skippedWithoutLearnedCarrier = new List(); + + void add(IEnumerable names, string stratum) + { + foreach (string name in names.Where(x => !string.IsNullOrWhiteSpace(x))) + { + if (!byName.TryGetValue(name.Trim(), out var quant) || BaselineQuants.IsNativeExactAlias(quant.UniqueId)) + continue; + + // Context blankets need a complete learned tensor map for their base so + // base-quant exception tensors can be reconstructed alongside explicit + // group overrides. In selected-baseline mode, a recognized built-in name + // is not necessarily enabled as a learning baseline. + if (!learnedContextIds.Contains(quant.UniqueId)) + { + skippedWithoutLearnedCarrier.Add($"{name.Trim()} ({stratum})"); + continue; + } + + if (contexts.All(x => x.QuantId != quant.UniqueId)) + contexts.Add(new SynergyTransferContext(quant.UniqueId, stratum)); + } + } + + add(strata.HighFidelityReferenceQuants, "high-fidelity"); + add(strata.MidFidelityReferenceQuants, "mid-fidelity"); + if (strata.LowFidelityEnabled) + add(strata.LowFidelityReferenceQuants, "low-fidelity"); + + if (skippedWithoutLearnedCarrier.Count > 0) + { + AnsiConsole.MarkupLine( + $"[yellow]Controlled context skipped:[/] no learned base-carrier mapping is configured for " + + $"{Markup.Escape(string.Join(", ", skippedWithoutLearnedCarrier.Distinct(StringComparer.OrdinalIgnoreCase)))}. " + + "Enable these names in baselines.enabled_standard_learning_baselines or configure learned external baseline names."); + } + + return contexts; + } + + + + private async Task> PlanConfirmedAnomalyExpansionProbesAsync( + IReadOnlyList initialResults, + ProbePlanningDiagnostics diagnostics, + CancellationToken ct) + { + var cfg = Config.AnomalyDetection.ConfirmedAnomalyExpansion; + if (!cfg.Enabled || cfg.MaxTotalExpansionProbes <= 0) + return new List(); + + var allowedReference = ResolveQuantNames(cfg.AllowedReferenceQuants).ToHashSet(); + var allowedCandidate = ResolveQuantNames(cfg.AllowedCandidateQuants).ToHashSet(); + var existingRuleKeys = await _rules.LoadExistingRuleSuppressionKeysAsync(ct); + var seen = new HashSet(StringComparer.Ordinal); + var plans = new List(); + + foreach (var result in initialResults + .Where(x => x.RuleDirection == AnomalyRuleDirection.Beneficial) + .Where(x => x.ReferenceSnapshot != null && x.ProbeSnapshot != null) + .OrderByDescending(x => x.ActualGainVsTwin)) + { + if (plans.Count >= cfg.MaxTotalExpansionProbes) + break; + + if (allowedReference.Count > 0 && !allowedReference.Contains(result.Plan.ReferenceConfig.BaseQuant)) + continue; + + var seedGroups = result.Plan.ProbeGroups.OrderBy(x => x.Group.UniqueId).ToList(); + if (seedGroups.Count == 0) + continue; + + byte primaryCandidateQuant = seedGroups[0].CandidateQuantId; + if (allowedCandidate.Count > 0 && !allowedCandidate.Contains(primaryCandidateQuant)) + continue; + + var reference = _movement.CreateActivatedContextBlanket(result.Plan.ReferenceConfig.BaseQuant); + _movement.EnsureAllActiveGroupsExplicit(reference, "confirmed-anomaly-expansion-reference"); + + int perRule = 0; + foreach (var neighbor in _movement.ActiveGroups.Where(g => seedGroups.All(s => s.Group.UniqueId != g.UniqueId)).OrderBy(g => g.UniqueId)) + { + if (perRule >= cfg.MaxNeighborsPerConfirmedRule || plans.Count >= cfg.MaxTotalExpansionProbes) + break; + + var groups = seedGroups + .Concat(new[] + { + new AnomalyChangedGroup + { + Group = neighbor, + ReferenceQuantId = reference.BaseQuant, + CandidateQuantId = primaryCandidateQuant, + ReferenceStoredSlot = BaselineQuants.EncodeTensorConfigGroupSlotBaselineId(reference.BaseQuant), + CandidateStoredSlot = BaselineQuants.EncodeTensorConfigGroupSlotBaselineId(primaryCandidateQuant), + Movement = QuantMovementKind.Downgrade + } + }) + .OrderBy(x => x.Group.UniqueId) + .ToList(); + + if (groups.Count > Config.AnomalyDetection.MaxProbeGroupCount) + continue; + + if (existingRuleKeys.Contains(_rules.BuildRuleSuppressionKey(reference, groups))) + { + diagnostics.SkippedExistingRuleOrSuppression++; + continue; + } + + var probe = reference; + foreach (var group in groups) + probe = _movement.WithStoredSlot(probe, group.Group, group.CandidateStoredSlot); + + if (ShouldSkipInvalidContextualAnomalyConfig(probe, "confirmed-anomaly-expansion", out _)) + { + diagnostics.SkippedInvalidMovement++; + continue; + } + + string key = TensorConfigIdentity.ToKey(reference) + "=>" + TensorConfigIdentity.ToKey(probe); + if (!seen.Add(key)) + { + diagnostics.SkippedDuplicate++; + continue; + } + + var seed = new AnomalySmokeCandidate + { + Source = "confirmed-anomaly-neighborhood", + CandidateConfig = probe, + TwinConfig = reference, + Movement = _movement.Analyze(reference, probe), + CandidatePredictedKld = result.Plan.Seed.CandidatePredictedKld, + TwinPredictedKld = result.Plan.Seed.TwinPredictedKld, + PredictionSpaceGapVsTwin = result.Plan.Seed.PredictionSpaceGapVsTwin, + SmokeScore = 900_000d + Math.Max(0d, result.ActualGainVsTwin), + SmokeStrength = "ConfirmedAnomalyNeighborhood", + SeedClass = AnomalySeedClass.ConfirmedAnomalyNeighborhoodProbe, + MatchedConfirmedAnomalyPattern = true, + PlannedProbeWillMeasureSize = true, + Message = "Bounded neighborhood probe generated from a confirmed beneficial contextual anomaly." + }; + + plans.Add(new AnomalyProbePlan + { + Seed = seed, + ReferenceConfig = reference, + ProbeConfig = probe, + ProbeGroups = groups, + ProbeType = "confirmed-neighborhood", + HypothesisLabel = _movement.DescribeGroups(groups), + SeedClass = AnomalySeedClass.ConfirmedAnomalyNeighborhoodProbe, + ProbePriorityClass = groups.Count == 1 ? AnomalySeedClass.ExploratorySingle : AnomalySeedClass.ExploratoryPair + }); + + perRule++; + diagnostics.ProbesQueued++; + diagnostics.ExpansionProbesQueued++; + } + } + + if (plans.Count > 0) + { + AnsiConsole.MarkupLine($"[yellow]Confirmed anomaly neighborhood probes:[/] queued={plans.Count:N0} maxTotal={cfg.MaxTotalExpansionProbes:N0}"); + } + + return plans; + } + + + + private async Task> PlanSynergyCompositionProbesAsync( + IReadOnlyList results, + ProbePlanningDiagnostics diagnostics, + List previewRecords, + CancellationToken ct) + { + var cfg = Config.SynergyDetection; + var plans = new List(); + if (!cfg.Enabled || !cfg.CompositionProbeEnabled || cfg.MaxCompositionProbesPerRun <= 0) + return plans; + + var templates = results + .Where(x => x.RuleDirection == AnomalyRuleDirection.Beneficial) + .Where(x => x.Accepted && x.ReferenceSnapshot != null && x.ProbeSnapshot != null) + .Where(x => x.Plan.ProbeGroups.Count > 0) + .Select(x => new + { + Result = x, + Confidence = Math.Clamp(Math.Abs(x.ActualGainVsTwin) / Math.Max(Config.AnomalyDetection.MinActualGainVsTwinKld, 1e-9), 0d, 1d), + Key = string.Join(",", x.Plan.ProbeGroups.OrderBy(g => g.Group.UniqueId).Select(g => $"{g.Group.UniqueId}:{g.CandidateQuantId}")) + }) + .Where(x => x.Confidence >= cfg.MinTemplateConfidenceForComposition) + .GroupBy(x => x.Key, StringComparer.Ordinal) + .Select(g => g.OrderByDescending(x => x.Result.ActualGainVsTwin).First()) + .OrderByDescending(x => x.Result.ActualGainVsTwin) + .Take(cfg.MaxTemplatesToCompose) + .ToList(); + + int considered = templates.Count; + int candidates = 0; + int reused = 0; + var seen = new HashSet(StringComparer.Ordinal); + var existingRuleKeys = await _rules.LoadExistingRuleSuppressionKeysAsync(ct); + + for (int i = 0; i < templates.Count; i++) + { + for (int j = i + 1; j < templates.Count; j++) + { + if (plans.Count >= cfg.MaxCompositionProbesPerRun) + break; + + var a = templates[i].Result; + var b = templates[j].Result; + if (a.Plan.ReferenceConfig.BaseQuant != b.Plan.ReferenceConfig.BaseQuant) + continue; + + var merged = a.Plan.ProbeGroups + .Concat(b.Plan.ProbeGroups) + .GroupBy(g => g.Group.UniqueId) + .Select(g => g.OrderBy(x => x.CandidateQuantId).First()) + .OrderBy(g => g.Group.UniqueId) + .ToList(); + + if (merged.Count <= Math.Max(a.Plan.ProbeGroups.Count, b.Plan.ProbeGroups.Count)) + continue; + if (merged.Count > cfg.MaxTemplateCompositionGroupCount) + continue; + + candidates++; + var reference = _movement.CreateActivatedContextBlanket(a.Plan.ReferenceConfig.BaseQuant); + var probe = reference; + foreach (var group in merged) + probe = _movement.WithStoredSlot(probe, group.Group, group.CandidateStoredSlot); + + if (ShouldSkipInvalidContextualAnomalyConfig(probe, "synergy-composition", out var skipReason)) + { + diagnostics.SkippedInvalidMovement++; + previewRecords.Add(BuildCompositionPreview(a, b, reference, probe, "SkippedInvalidContextualProbe", skipReason)); + continue; + } + + if (existingRuleKeys.Contains(_rules.BuildRuleSuppressionKey(reference, merged))) + { + diagnostics.SkippedExistingRuleOrSuppression++; + reused++; + previewRecords.Add(BuildCompositionPreview(a, b, reference, probe, "ExistingRuleOrSuppression", "A matching composition rule/suppression already exists.")); + continue; + } + + string key = TensorConfigIdentity.ToKey(reference) + "=>" + TensorConfigIdentity.ToKey(probe); + if (!seen.Add(key)) + { + diagnostics.SkippedDuplicate++; + reused++; + continue; + } + + var movement = _movement.Analyze(reference, probe); + var seed = new AnomalySmokeCandidate + { + Source = "counterfactual-synergy-composition", + CandidateConfig = probe, + TwinConfig = reference, + Movement = movement, + SmokeScore = 950_000d + Math.Max(0d, a.ActualGainVsTwin) + Math.Max(0d, b.ActualGainVsTwin), + SmokeStrength = "CompositionProbe", + SeedClass = AnomalySeedClass.SynergyCompositionProbe, + MatchedConfirmedAnomalyPattern = true, + PlannedProbeWillMeasureSize = true, + Message = "Tiny composition probe: tests whether two confirmed synergy templates cooperate, add, overlap redundantly, or interfere." + }; + + plans.Add(new AnomalyProbePlan + { + Seed = seed, + ReferenceConfig = reference, + ProbeConfig = probe, + ProbeGroups = merged, + ProbeType = "composition", + HypothesisLabel = _movement.DescribeGroups(merged), + SeedClass = AnomalySeedClass.SynergyCompositionProbe, + ProbePriorityClass = AnomalySeedClass.SynergyCompositionProbe + }); + diagnostics.ProbesQueued++; + diagnostics.CompositionProbesQueued++; + previewRecords.Add(BuildCompositionPreview(a, b, reference, probe, "Queued", "Composition probe queued for real benchmark validation.")); + } + } + + AnsiConsole.MarkupLine($"[yellow]Synergy composition probes:[/] source templates considered=[cyan]{considered:N0}[/] composition candidates=[cyan]{candidates:N0}[/] queued=[cyan]{plans.Count:N0}[/] reused/skipped existing=[cyan]{reused:N0}[/]"); + return plans; + } + + private SynergyCompositionProbeRecord BuildCompositionPreview( + AnomalyProbeResult a, + AnomalyProbeResult b, + TensorConfig reference, + TensorConfig probe, + string classification, + string note) + { + var movement = _movement.Analyze(reference, probe); + return new SynergyCompositionProbeRecord + { + CompositionId = TensorConfigIdentity.ToKey(reference) + "=>" + TensorConfigIdentity.ToKey(probe), + SourceTemplateIds = new[] + { + TensorConfigIdentity.ToKey(a.Plan.ProbeConfig), + TensorConfigIdentity.ToKey(b.Plan.ProbeConfig) + }, + SourceTemplateLabels = new[] { a.Plan.HypothesisLabel, b.Plan.HypothesisLabel }, + CandidateEffectiveGroups = _movement.BuildEffectiveGroupVector(probe), + TwinEffectiveGroups = _movement.BuildEffectiveGroupVector(reference), + CombinedGroupCount = movement.DowngradeCount, + Classification = classification, + ActualCandidateKld = null, + ActualTwinKld = null, + ActualGainVsTwin = null, + PredictedCandidateKld = null, + PredictedTwinKld = null, + PredictionSpaceGap = null, + Notes = new[] { note } + }; + } + + private List BuildCompositionDiagnostics( + IReadOnlyList compositionPlans, + IReadOnlyList compositionResults) + { + var records = new List(); + foreach (var result in compositionResults.Where(x => x.Plan.SeedClass == AnomalySeedClass.SynergyCompositionProbe)) + { + string classification = result.Classification switch + { + AnomalyProbeClassification.HarmfulInteraction => "HarmfulInterference", + AnomalyProbeClassification.ContaminatingPassenger => "HarmfulInterference", + AnomalyProbeClassification.NormalGravity => "RedundantComposition", + AnomalyProbeClassification.SuppressionOnly => "CompositionRejected", + _ when result.RuleDirection == AnomalyRuleDirection.Beneficial && result.ActualGainVsTwin >= Config.AnomalyDetection.MinActualGainVsTwinKld * 2d => "SuperSynergy", + _ when result.RuleDirection == AnomalyRuleDirection.Beneficial => "AdditiveComposition", + _ => "CompositionRejected" + }; + + records.Add(new SynergyCompositionProbeRecord + { + CompositionId = TensorConfigIdentity.ToKey(result.Plan.ReferenceConfig) + "=>" + TensorConfigIdentity.ToKey(result.Plan.ProbeConfig), + SourceTemplateIds = result.Plan.ProbeGroups.Select(g => $"{g.Group.UniqueId}:{g.ReferenceQuantId}->{g.CandidateQuantId}").ToList(), + SourceTemplateLabels = new[] { result.Plan.HypothesisLabel }, + CandidateEffectiveGroups = _movement.BuildEffectiveGroupVector(result.Plan.ProbeConfig), + TwinEffectiveGroups = _movement.BuildEffectiveGroupVector(result.Plan.ReferenceConfig), + CombinedGroupCount = result.Plan.ProbeGroups.Count, + Classification = classification, + ActualCandidateKld = result.ProbeSnapshot?.Kld, + ActualTwinKld = result.ReferenceSnapshot?.Kld, + ActualGainVsTwin = result.ActualGainVsTwin, + PredictedCandidateKld = result.Plan.Seed.CandidatePredictedKld, + PredictedTwinKld = result.Plan.Seed.TwinPredictedKld, + PredictionSpaceGap = result.Plan.Seed.PredictionSpaceGapVsTwin, + Notes = new[] { result.Message } + }); + } + + int super = records.Count(x => x.Classification == "SuperSynergy"); + int additive = records.Count(x => x.Classification == "AdditiveComposition"); + int harmful = records.Count(x => x.Classification == "HarmfulInterference"); + int rejected = records.Count(x => x.Classification == "CompositionRejected" || x.Classification == "RedundantComposition"); + AnsiConsole.MarkupLine($"[yellow]Synergy composition probes:[/] confirmed super-synergy=[cyan]{super:N0}[/] additive=[cyan]{additive:N0}[/] harmful/interference=[cyan]{harmful:N0}[/] rejected/redundant=[cyan]{rejected:N0}[/]"); + return records; + } + + private async Task> ValidateProbesAsync(IReadOnlyList probes, CancellationToken ct) + { + if (probes.Count == 0) + return new List(); + + var results = new List(); + var safeProbes = new List(); + + foreach (var plan in probes) + { + string referenceReason = string.Empty; + string probeReason = string.Empty; + bool invalidReference = ShouldSkipInvalidContextualAnomalyConfig(plan.ReferenceConfig, "validate-reference", out referenceReason); + bool invalidProbe = ShouldSkipInvalidContextualAnomalyConfig(plan.ProbeConfig, "validate-probe", out probeReason); + if (invalidReference || invalidProbe) + { + string reason = invalidReference ? referenceReason : probeReason; + AnsiConsole.MarkupLine($"[yellow]SkippedInvalidContextualAnomalyProbe:[/] source=validate reason={Markup.Escape(reason)} reference={Markup.Escape(HybridBenchmarkRepository.BuildDisplayName((HybridQuant)plan.ReferenceConfig))} probe={Markup.Escape(HybridBenchmarkRepository.BuildDisplayName((HybridQuant)plan.ProbeConfig))}"); + results.Add(new AnomalyProbeResult + { + Plan = plan, + Classification = AnomalyProbeClassification.SuppressionOnly, + RuleDirection = AnomalyRuleDirection.SuppressionOnly, + Accepted = false, + FailureCode = reason.Contains("SparseActiveGroup", StringComparison.OrdinalIgnoreCase) ? "SKIPPED_SPARSE_CONTEXTUAL_PROBE" : "SKIPPED_ISOLATION_SAMPLE", + Message = reason.Contains("SparseActiveGroup", StringComparison.OrdinalIgnoreCase) + ? "SkippedInvalidContextualAnomalyProbe: active groups must be explicitly stored for contextual anomaly probes." + : "SkippedIsolationSample: BF16/exact isolation rows are not contextual anomaly probes." + }); + continue; + } + + _movement.EnsureAllActiveGroupsExplicit(plan.ReferenceConfig, "pre-quantization-reference"); + _movement.EnsureAllActiveGroupsExplicit(plan.ProbeConfig, "pre-quantization-probe"); + safeProbes.Add(plan); + } + + if (safeProbes.Count == 0) + return results; + + foreach (var plan in safeProbes) + WriteContextualProbeConsoleLog(plan); + + var quants = safeProbes + .SelectMany(x => new[] { x.ReferenceConfig, x.ProbeConfig }) + .DistinctBy(TensorConfigIdentity.ToKey) + .Select(x => (HybridQuant)x) + .ToList(); + + AnsiConsole.MarkupLine($"[grey]Validating anomaly probes:[/] unique quant builds/benchmarks=[cyan]{quants.Count:N0}[/] probe plans=[cyan]{safeProbes.Count:N0}[/]"); + await _quantizationService.ProcessHybridBatchAsync(quants, ct); + + foreach (var plan in safeProbes) + { + var reference = await _repository.LoadBenchmarkSnapshotAsync(plan.ReferenceConfig, ct); + var probe = await _repository.LoadBenchmarkSnapshotAsync(plan.ProbeConfig, ct); + var result = ClassifyProbe(plan, reference, probe); + results.Add(result); + + WriteProbeOutcome(result); + } + + return results; + } + + private AnomalyProbeResult ClassifyProbe( + AnomalyProbePlan plan, + BenchmarkSnapshotRecord? reference, + BenchmarkSnapshotRecord? probe) + { + if (reference == null) + { + return new AnomalyProbeResult + { + Plan = plan, + ReferenceSnapshot = null, + ProbeSnapshot = probe, + Classification = AnomalyProbeClassification.MissingTwin, + RuleDirection = AnomalyRuleDirection.SuppressionOnly, + FailureCode = "REFERENCE_TWIN_MISSING", + Message = "Reference/higher-bit twin benchmark was missing after anomaly probe validation." + }; + } + + if (probe == null) + { + return new AnomalyProbeResult + { + Plan = plan, + ReferenceSnapshot = reference, + ProbeSnapshot = null, + Classification = AnomalyProbeClassification.MissingProbeBenchmark, + RuleDirection = AnomalyRuleDirection.SuppressionOnly, + FailureCode = "PROBE_BENCHMARK_MISSING", + Message = "Probe benchmark was missing after anomaly probe validation." + }; + } + + if (plan.ProbeType == "context-rank-pair") + return ClassifyContextRankPair(plan, reference, probe); + + double gain = reference.Kld - probe.Kld; + bool sameOrSmaller = probe.SizeBytes <= reference.SizeBytes; + if (sameOrSmaller && gain >= Config.AnomalyDetection.MinActualGainVsTwinKld) + { + return new AnomalyProbeResult + { + Plan = plan, + ReferenceSnapshot = reference, + ProbeSnapshot = probe, + Classification = plan.ProbeType switch + { + "single" => AnomalyProbeClassification.SingleGroupInversion, + "pair" => AnomalyProbeClassification.PairSynergy, + "composition" when gain >= Config.AnomalyDetection.MinActualGainVsTwinKld * 2d => AnomalyProbeClassification.SuperSynergy, + "composition" => AnomalyProbeClassification.AdditiveComposition, + "full" when plan.ProbeGroups.Count >= 3 => AnomalyProbeClassification.HigherOrderSynergy, + _ => AnomalyProbeClassification.CounterfactualMdaViolation + }, + RuleDirection = AnomalyRuleDirection.Beneficial, + Accepted = true, + ActualGainVsTwin = gain, + Message = plan.ProbeType == "composition" + ? "Composed counterfactual synergy template beat its higher-fidelity same-context twin." + : "Lower-fidelity monotone probe beat its higher-fidelity same-context twin." + }; + } + + double harmfulMargin = Math.Max(Config.AnomalyDetection.MinActualGainVsTwinKld, Config.SynergyDetection.MinFailureMarginForContaminationKld); + if (probe.Kld - reference.Kld >= harmfulMargin) + { + bool contaminatingPassenger = Config.SynergyDetection.ContaminatingPassengerDetectionEnabled && + (plan.SeedClass == AnomalySeedClass.ConfirmedAnomalyNeighborhoodProbe || + plan.SeedClass == AnomalySeedClass.SynergyTransferProbe || + plan.SeedClass == AnomalySeedClass.SynergyCompositionProbe) && + plan.ProbeGroups.Count > 1; + + return new AnomalyProbeResult + { + Plan = plan, + ReferenceSnapshot = reference, + ProbeSnapshot = probe, + Classification = contaminatingPassenger + ? AnomalyProbeClassification.ContaminatingPassenger + : plan.ProbeType == "composition" + ? AnomalyProbeClassification.HarmfulInterference + : AnomalyProbeClassification.HarmfulInteraction, + RuleDirection = AnomalyRuleDirection.Harmful, + Accepted = true, + ActualGainVsTwin = gain, + Message = contaminatingPassenger + ? "Probe was meaningfully worse than its twin; persisted as scoped contaminating-passenger negative evidence." + : "Probe was meaningfully worse than its higher-fidelity twin; persisted as harmful interaction." + }; + } + + return new AnomalyProbeResult + { + Plan = plan, + ReferenceSnapshot = reference, + ProbeSnapshot = probe, + Classification = AnomalyProbeClassification.NormalGravity, + RuleDirection = AnomalyRuleDirection.SuppressionOnly, + Accepted = false, + ActualGainVsTwin = gain, + FailureCode = "NORMAL_GRAVITY", + Message = "Smoke rejected / normal MDA gravity confirmed." + }; + } + + private static AnomalyProbeResult ClassifyContextRankPair( + AnomalyProbePlan plan, + BenchmarkSnapshotRecord reference, + BenchmarkSnapshotRecord probe) + { + double gain = reference.Kld - probe.Kld; + double threshold = Math.Max(Config.AnomalyDetection.MinActualGainVsTwinKld, 1e-12d); + if (gain >= threshold) + { + return new AnomalyProbeResult + { + Plan = plan, + ReferenceSnapshot = reference, + ProbeSnapshot = probe, + Classification = AnomalyProbeClassification.ContextOnly, + RuleDirection = AnomalyRuleDirection.Beneficial, + Accepted = true, + ActualGainVsTwin = gain, + Message = "The native-isolation winner retained a material KLD advantage over its closest-size same-bit contender in this measured context." + }; + } + + if (-gain >= threshold) + { + return new AnomalyProbeResult + { + Plan = plan, + ReferenceSnapshot = reference, + ProbeSnapshot = probe, + Classification = AnomalyProbeClassification.HarmfulInteraction, + RuleDirection = AnomalyRuleDirection.Harmful, + Accepted = true, + ActualGainVsTwin = gain, + Message = "Context rank reversal: the native-isolation winner became materially worse than its closest-size same-bit contender in this measured context." + }; + } + + return new AnomalyProbeResult + { + Plan = plan, + ReferenceSnapshot = reference, + ProbeSnapshot = probe, + Classification = AnomalyProbeClassification.NormalGravity, + RuleDirection = AnomalyRuleDirection.SuppressionOnly, + Accepted = false, + ActualGainVsTwin = gain, + FailureCode = "CONTEXT_RANK_PAIR_INCONCLUSIVE", + Message = "The same-bit context pair did not separate beyond the configured KLD evidence threshold." + }; + } + + + private async Task EmitQ8ContextReferenceDriftDiagnosticsAsync( + IReadOnlyDictionary byKey, + CancellationToken ct) + { + byte q8 = BaselineQuants.Q8_0.UniqueId; + var sparse = BuildSparsePureContext(q8); + TensorConfig explicitContext; + try + { + explicitContext = _movement.CreateActivatedContextBlanket(q8); + } + catch + { + return; + } + + byKey.TryGetValue(TensorConfigIdentity.ToKey(sparse), out var sparseSnapshot); + byKey.TryGetValue(TensorConfigIdentity.ToKey(explicitContext), out var explicitSnapshot); + if (sparseSnapshot == null || explicitSnapshot == null) + return; + + double kldDelta = explicitSnapshot.Kld - sparseSnapshot.Kld; + long sizeDelta = unchecked((long)explicitSnapshot.SizeBytes - (long)sparseSnapshot.SizeBytes); + bool material = Math.Abs(kldDelta) >= Config.AnomalyDetection.MinActualGainVsTwinKld || Math.Abs(sizeDelta) > 0; + + if (material) + AnsiConsole.MarkupLine("[yellow]Q8_CONTEXT_REFERENCE_DRIFT[/]"); + else if (Config.AnomalyDetection.VerboseAnomalyLogging) + AnsiConsole.MarkupLine("[grey]Q8 contextual reference drift check:[/]"); + + if (material || Config.AnomalyDetection.VerboseAnomalyLogging) + { + AnsiConsole.MarkupLine($"[grey] pureQ8Kld=[/] [cyan]{sparseSnapshot.Kld:0.000000}[/] [grey]explicitContextQ8Kld=[/] [cyan]{explicitSnapshot.Kld:0.000000}[/] [grey]kldDelta=[/] [cyan]{kldDelta:0.000000}[/]"); + AnsiConsole.MarkupLine($"[grey] pureQ8SizeBytes=[/] [cyan]{sparseSnapshot.SizeBytes:N0}[/] [grey]explicitContextQ8SizeBytes=[/] [cyan]{explicitSnapshot.SizeBytes:N0}[/] [grey]sizeDeltaBytes=[/] [cyan]{sizeDelta:N0}[/]"); + } + + await WriteJsonAsync("magicquant-anomaly-q8-reference-drift.json", new + { + generatedAtUtc = DateTime.UtcNow, + driftCode = material ? "Q8_CONTEXT_REFERENCE_DRIFT" : "none", + pureQ8 = new { key = TensorConfigIdentity.ToKey(sparseSnapshot.Config), sparseSnapshot.DisplayName, sparseSnapshot.Kld, sparseSnapshot.SizeBytes }, + explicitContextQ8 = new { key = TensorConfigIdentity.ToKey(explicitSnapshot.Config), explicitSnapshot.DisplayName, explicitSnapshot.Kld, explicitSnapshot.SizeBytes }, + kldDelta, + sizeDeltaBytes = sizeDelta, + note = "Anomaly probes prefer the explicit all-active contextual Q8 twin. Sparse pure Q8 remains a useful baseline anchor but may not be identical if benchmark execution settings drifted. Compare NGL/benchmark run metadata in SQLite BenchmarkRuns if material drift appears." + }, ct); + } + + private async Task> LoadAllCurrentBenchmarkSnapshotsAsync(CancellationToken ct) + { + await using var db = new MagicQuantContext(); + var modelHashId = await ArchitectureFamilyService.ResolveScopedAiModelHashIdOrNullAsync(db, ct); + if (modelHashId == null) + return new List(); + + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + int? imatrixId = await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, modelHashId.Value, createIfMissing: false, ct); + + var rows = await db.AiBenchmarks + .AsNoTracking() + .Include(x => x.TensorCombo) + .Include(x => x.CategorBenchmarks) + .Where(x => x.ArchitectureFamilyId == architectureFamilyId) + .Where(x => x.TensorGroupProfileId == tensorGroupProfileId) + .Where(x => x.AiModelHashId == modelHashId.Value) + .Where(x => x.ImatrixDefinitionId == imatrixId) + .ToListAsync(ct); + + var list = new List(); + foreach (var row in rows) + { + var general = row.CategorBenchmarks.FirstOrDefault(x => x.Category == (byte)BenchmarkCategory.General) + ?? row.CategorBenchmarks.OrderBy(x => x.Category).FirstOrDefault(); + if (general == null) + continue; + + var config = new TensorConfig(row.TensorCombo.BaseQuant, row.TensorCombo.Embeddings, row.TensorCombo.LmHead, row.TensorCombo.AttnQ, row.TensorCombo.AttnKV, row.TensorCombo.AttnOutput, row.TensorCombo.FfnUpGate, row.TensorCombo.FfnDown, row.TensorCombo.MoeExperts, row.TensorCombo.MoeRouter); + var quant = (HybridQuant)config; + list.Add(new BenchmarkSnapshotRecord + { + Config = config, + Quant = quant, + DisplayName = HybridBenchmarkRepository.BuildDisplayName(quant), + ProviderName = HybridBenchmarkRepository.ResolveProviderName(quant, exportNaming: false), + BaselineFamily = HybridBenchmarkRepository.ResolveBaselineFamily(quant), + IsHybrid = HybridBenchmarkRepository.IsTrueMagicQuantHybrid(quant), + IsExternalRebuiltBaseline = HybridBenchmarkRepository.IsExternalRebuiltBaseline(quant), + IsMaterializedTensorMapped = quant.Tensors.Count > 0, + SizeBytes = row.SizeBytes, + Kld = general.Kld, + Ppl = general.Ppl + }); + } + + return list; + } + + private async Task> LoadPredictionLookupAsync(CancellationToken ct) + { + var rows = await LoadPredictionRowsAsync(DuckSmokeScanLimit, ct); + return rows.ToDictionary(x => TensorConfigIdentity.ToKey(x.Config), StringComparer.Ordinal); + } + + private async Task> LoadPredictionRowsAsync(int limit, CancellationToken ct) + { + using var c = new DuckDBConnection($"Data Source={_store.GetDatabaseFilePath()}"); + await c.OpenAsync(ct); + await ConfigureDuckAsync(c, ct); + + using var cmd = c.CreateCommand(); + string limitSql = limit > 0 ? "\nLIMIT ?" : string.Empty; + cmd.CommandText = $@" +SELECT {CombinationDuckDbSchema.SlotColumnList}, + COALESCE(BaseRankSafeKld, PredictedKld) AS BaseRankSafeKld, + COALESCE(FinalPredictedKld, PredictedKld) AS FinalPredictedKld, + PredictedSizeBytes, + PredictionConfidence, + PredictionRank +FROM {CombinationDuckDbSchema.TableName} +WHERE COALESCE(BaseRankSafeKld, PredictedKld) IS NOT NULL + AND PredictedSizeBytes IS NOT NULL + AND PredictionRank IS NOT NULL +ORDER BY PredictionRank ASC{limitSql};"; + if (limit > 0) + cmd.Parameters.Add(new DuckDBParameter { Value = limit }); + + var rows = new List(); + using var r = await cmd.ExecuteReaderAsync(ct); + while (await r.ReadAsync(ct)) + rows.Add(ReadPredictionDuckRow(r)); + + return rows; + } + + private async Task LoadSinglePredictionRowAsync(TensorConfig config, CancellationToken ct) + { + using var c = new DuckDBConnection($"Data Source={_store.GetDatabaseFilePath()}"); + await c.OpenAsync(ct); + await ConfigureDuckAsync(c, ct); + + using var cmd = c.CreateCommand(); + cmd.CommandText = $@" +SELECT {CombinationDuckDbSchema.SlotColumnList}, + COALESCE(BaseRankSafeKld, PredictedKld) AS BaseRankSafeKld, + COALESCE(FinalPredictedKld, PredictedKld) AS FinalPredictedKld, + PredictedSizeBytes, + PredictionConfidence, + PredictionRank +FROM {CombinationDuckDbSchema.TableName} +WHERE BaseQuant = ? + AND Embeddings = ? + AND LmHead = ? + AND AttnQ = ? + AND AttnKV = ? + AND AttnOutput = ? + AND FfnUpGate = ? + AND FfnDown = ? + AND MoeExperts = ? + AND MoeRouter = ? +LIMIT 1;"; + AddConfigParameters(cmd, config); + + using var r = await cmd.ExecuteReaderAsync(ct); + if (!await r.ReadAsync(ct)) + return null; + + return ReadPredictionDuckRow(r); + } + + + private async Task LoadContextualTwinPredictionRowAsync( + TensorConfig explicitTwin, + IReadOnlyDictionary explicitRows, + CancellationToken ct) + { + string explicitKey = TensorConfigIdentity.ToKey(explicitTwin); + if (explicitRows.TryGetValue(explicitKey, out var inMemoryExplicit)) + return inMemoryExplicit; + + var explicitRow = await LoadSinglePredictionRowAsync(explicitTwin, ct); + if (explicitRow != null) + return explicitRow with { Config = explicitTwin }; + + // The normal generator may only contain the pure sparse carrier for an all-Q8/all-Q6 + // reference. For smoke scoring, that sparse carrier is allowed as a prediction source + // only; the anomaly seed/probe/twin identity remains the explicit activated blanket. + var sparsePure = new TensorConfig( + explicitTwin.BaseQuant, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue); + + var sparseRow = await LoadSinglePredictionRowAsync(sparsePure, ct); + return sparseRow == null ? null : sparseRow with { Config = sparsePure }; + } + + + private static void AddOrPreferBetterPredictionRow(IDictionary rows, PredictionDuckRow row) + { + string key = TensorConfigIdentity.ToKey(row.Config); + if (!rows.TryGetValue(key, out var existing) || + row.BaseRankSafeKld < existing.BaseRankSafeKld || + (Math.Abs(row.BaseRankSafeKld - existing.BaseRankSafeKld) < 1e-12 && row.PredictedSizeBytes < existing.PredictedSizeBytes)) + { + rows[key] = row; + } + } + + private static TensorConfig BuildSparsePureContext(byte baseQuantId) => new( + baseQuantId, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue, + BaselineQuants.TensorConfigNullSlotValue); + + private PredictionDuckRow? ResolveTwinFromLookupOnly( + TensorConfig explicitTwin, + IReadOnlyDictionary lookup, + out string lookupMode, + out bool foundInDictionary) + { + string explicitKey = TensorConfigIdentity.ToKey(explicitTwin); + if (lookup.TryGetValue(explicitKey, out var explicitRow)) + { + lookupMode = "explicit-context-found"; + foundInDictionary = true; + return explicitRow with { Config = explicitTwin }; + } + + var sparsePure = BuildSparsePureContext(explicitTwin.BaseQuant); + string sparseKey = TensorConfigIdentity.ToKey(sparsePure); + if (lookup.TryGetValue(sparseKey, out var sparseRow)) + { + lookupMode = "sparse-pure-prediction-fallback; explicit-context-identity-preserved"; + foundInDictionary = true; + return sparseRow with { Config = sparsePure }; + } + + lookupMode = "missing; dictionary-only lookup searched explicit-context and sparse-pure"; + foundInDictionary = false; + return null; + } + + private RejectedSmokePreview AddRejectedPreview( + List previews, + TensorConfig candidate, + TensorConfig twin, + AnomalyMovementAnalysis? movement, + PredictionDuckRow? candidateRow, + PredictionDuckRow? twinRow, + ulong? predictedSizeSavingsBytes, + double? predictionSpaceGap, + string rejectionReason, + bool matchedConfirmedAnomalyPattern, + bool twinFoundInLookup) + { + var preview = new RejectedSmokePreview( + previews.Count, + candidate, + twin, + movement, + candidateRow, + twinRow, + predictedSizeSavingsBytes, + predictionSpaceGap, + rejectionReason, + matchedConfirmedAnomalyPattern, + twinFoundInLookup); + + if (previews.Count < 500 || rejectionReason.Contains("PredictionSpaceGap", StringComparison.OrdinalIgnoreCase) || matchedConfirmedAnomalyPattern) + previews.Add(preview); + + return preview; + } + + private static HashSet ResolveQuantNames(IEnumerable names) + { + var map = BaselineQuants.GetAllRecognizedBaselines() + .SelectMany(q => q.Names.Select(n => (Name: n, Quant: q))) + .GroupBy(x => x.Name, StringComparer.OrdinalIgnoreCase) + .ToDictionary(g => g.Key, g => g.First().Quant.UniqueId, StringComparer.OrdinalIgnoreCase); + + var result = new HashSet(); + foreach (var name in names ?? Array.Empty()) + { + if (map.TryGetValue(name, out var id)) + result.Add(id); + } + + return result; + } + + private object? BuildBestConfirmedAnomalyReconciliation( + IReadOnlyList results, + IReadOnlyCollection selectedSurvivors) + { + var best = results + .Where(x => x.RuleDirection == AnomalyRuleDirection.Beneficial) + .Where(x => x.ReferenceSnapshot != null && x.ProbeSnapshot != null) + .OrderByDescending(x => x.ActualGainVsTwin) + .ThenBy(x => x.ProbeSnapshot!.Kld) + .ThenBy(x => x.ProbeSnapshot!.SizeBytes) + .FirstOrDefault(); + + if (best == null || best.ReferenceSnapshot == null || best.ProbeSnapshot == null) + return null; + + string key = TensorConfigIdentity.ToKey(best.ProbeSnapshot.Config); + bool selected = selectedSurvivors.Any(x => TensorConfigIdentity.ToKey(x.Config) == key); + string reasonNotSelected = selected + ? string.Empty + : selectedSurvivors.Count == 0 + ? "final selection has not run yet" + : BuildReasonBestAnomalyNotSelected(best.ProbeSnapshot, selectedSurvivors); + + return new + { + candidate = TensorConfigIdentity.ToKey(best.ProbeSnapshot.Config), + candidateName = best.ProbeSnapshot.DisplayName, + twin = TensorConfigIdentity.ToKey(best.ReferenceSnapshot.Config), + twinName = best.ReferenceSnapshot.DisplayName, + actualCandidateKld = best.ProbeSnapshot.Kld, + actualTwinKld = best.ReferenceSnapshot.Kld, + actualGain = best.ActualGainVsTwin, + actualCandidateSizeBytes = best.ProbeSnapshot.SizeBytes, + actualTwinSizeBytes = best.ReferenceSnapshot.SizeBytes, + actualSizeSavingsBytes = best.ReferenceSnapshot.SizeBytes >= best.ProbeSnapshot.SizeBytes ? best.ReferenceSnapshot.SizeBytes - best.ProbeSnapshot.SizeBytes : 0UL, + classification = best.Classification.ToString(), + selectedAsSurvivor = selected, + reasonNotSelected + }; + } + + private static string BuildReasonBestAnomalyNotSelected( + BenchmarkSnapshotRecord anomaly, + IReadOnlyCollection selectedSurvivors) + { + var dominator = selectedSurvivors.FirstOrDefault(x => x.SizeBytes <= anomaly.SizeBytes && x.Kld <= anomaly.Kld && (x.SizeBytes < anomaly.SizeBytes || x.Kld < anomaly.Kld)); + if (dominator != null) + return $"dominated by selected survivor {dominator.DisplayName} (kld={dominator.Kld:0.000000}, size={dominator.SizeBytes})"; + + var lowerKld = selectedSurvivors.OrderBy(x => x.Kld).ThenBy(x => x.SizeBytes).FirstOrDefault(); + if (lowerKld != null && lowerKld.Kld < anomaly.Kld) + return $"selected frontier contains lower-KLD survivor {lowerKld.DisplayName}; anomaly was not a final dominance/spacing winner"; + + return "not present in selected survivor set; no dominance reason was found in current reconciliation data"; + } + + private static void WriteBestAnomalyConsoleLog(object? reconciliation) + { + if (reconciliation == null) + { + AnsiConsole.MarkupLine("[grey]Best confirmed beneficial anomaly:[/] none"); + return; + } + + string json = JsonSerializer.Serialize(reconciliation, JsonOptions); + using var doc = JsonDocument.Parse(json); + var root = doc.RootElement; + AnsiConsole.MarkupLine("[yellow]Best confirmed beneficial anomaly:[/]"); + AnsiConsole.MarkupLine($"[grey] candidate=[/] [cyan]{Markup.Escape(root.GetProperty("candidateName").GetString() ?? "unknown")}[/]"); + AnsiConsole.MarkupLine($"[grey] twin=[/] [cyan]{Markup.Escape(root.GetProperty("twinName").GetString() ?? "unknown")}[/]"); + AnsiConsole.MarkupLine($"[grey] actualCandidateKld=[/] [cyan]{root.GetProperty("actualCandidateKld").GetDouble():0.000000}[/]"); + AnsiConsole.MarkupLine($"[grey] actualTwinKld=[/] [cyan]{root.GetProperty("actualTwinKld").GetDouble():0.000000}[/]"); + AnsiConsole.MarkupLine($"[grey] actualGain=[/] [cyan]{root.GetProperty("actualGain").GetDouble():0.000000}[/]"); + AnsiConsole.MarkupLine($"[grey] selectedAsSurvivor=[/] [cyan]{root.GetProperty("selectedAsSurvivor").GetBoolean()}[/]"); + string reason = root.TryGetProperty("reasonNotSelected", out var r) ? r.GetString() ?? string.Empty : string.Empty; + if (!string.IsNullOrWhiteSpace(reason)) + AnsiConsole.MarkupLine($"[grey] reasonNotSelected=[/] [yellow]{Markup.Escape(reason)}[/]"); + } + + private static string FmtNullable(double? value) => value.HasValue ? value.Value.ToString("0.000000", CultureInfo.InvariantCulture) : "n/a"; + private static string FmtNullable(ulong? value) => value.HasValue ? value.Value.ToString("N0", CultureInfo.InvariantCulture) : "n/a"; + + private static PredictionDuckRow ReadPredictionDuckRow(System.Data.Common.DbDataReader r) + { + return new PredictionDuckRow( + new TensorConfig( + ToByte(r.GetValue(0)), + ToByte(r.GetValue(1)), + ToByte(r.GetValue(2)), + ToByte(r.GetValue(3)), + ToByte(r.GetValue(4)), + ToByte(r.GetValue(5)), + ToByte(r.GetValue(6)), + ToByte(r.GetValue(7)), + ToByte(r.GetValue(8)), + ToByte(r.GetValue(9))), + ToDouble(r.GetValue(10)), + ToDouble(r.GetValue(11)), + ToUInt64(r.GetValue(12)), + ToDouble(r.GetValue(13)), + ToUInt64(r.GetValue(14))); + } + + + private bool ShouldSkipInvalidContextualAnomalyConfig(TensorConfig config, string source, out string reason) + { + if (_movement.TryValidateContextualAnomalyConfig(config, out var validationReason)) + { + reason = string.Empty; + return false; + } + + reason = $"{source}: {validationReason}"; + return true; + } + + private static void LogSkippedInvalidContextualAnomalyConfig(string source, TensorConfig config, string reason, int count) + { + if (!Config.AnomalyDetection.VerboseAnomalyLogging && count > 1) + return; + + if (count > 12) + return; + + string label = reason.Contains("SparseActiveGroup", StringComparison.OrdinalIgnoreCase) + ? "SkippedInvalidContextualAnomalyProbe" + : "SkippedIsolationSample"; + + string canonicalMessage = reason.Contains("SparseActiveGroup", StringComparison.OrdinalIgnoreCase) + ? "active groups must be explicit for contextual anomaly probes" + : "BF16/exact isolation rows are not contextual anomaly probes"; + + AnsiConsole.MarkupLine( + $"[yellow]{label}:[/] {Markup.Escape(canonicalMessage)}. source={Markup.Escape(source)} reason={Markup.Escape(reason)} config={Markup.Escape(TensorConfigIdentity.ToKey(config))} name={Markup.Escape(HybridBenchmarkRepository.BuildDisplayName((HybridQuant)config))}"); + } + + private static void LogHistoricalSparseCandidateIgnored(TensorConfig config, string reason, int count) + { + if (!Config.AnomalyDetection.VerboseAnomalyLogging && count > 1) + return; + + if (count > 12) + return; + + AnsiConsole.MarkupLine( + $"[yellow]HistoricalSparseCandidateIgnored:[/] reason=CannotTrustTensorComboIdentityForAnomalyRule detail={Markup.Escape(reason)} config={Markup.Escape(TensorConfigIdentity.ToKey(config))} name={Markup.Escape(HybridBenchmarkRepository.BuildDisplayName((HybridQuant)config))}"); + } + + private void WriteContextualProbeConsoleLog(AnomalyProbePlan plan) + { + var movement = _movement.Analyze(plan.ReferenceConfig, plan.ProbeConfig); + var referenceGroups = _movement.BuildEffectiveGroupVector(plan.ReferenceConfig); + var probeGroups = _movement.BuildEffectiveGroupVector(plan.ProbeConfig); + var inactive = _movement.BuildInactiveGroupList(); + + AnsiConsole.MarkupLine("[yellow]Contextual anomaly probe:[/]"); + AnsiConsole.MarkupLine($"[grey] kind=[/] [cyan]{Markup.Escape(plan.ProbeType)}[/]"); + AnsiConsole.MarkupLine($"[grey] seedClass=[/] [cyan]{Markup.Escape(plan.SeedClass.ToString())}[/] [grey]priority=[/] [cyan]{Markup.Escape(plan.ProbePriorityClass.ToString())}[/]"); + AnsiConsole.MarkupLine($"[grey] referenceQuant=[/] [cyan]{Markup.Escape(SafeName(plan.ReferenceConfig.BaseQuant))}[/]"); + AnsiConsole.MarkupLine($"[grey] base=[/] [cyan]{Markup.Escape(SafeName(plan.ProbeConfig.BaseQuant))}[/]"); + AnsiConsole.MarkupLine("[grey] effective groups:[/]"); + foreach (var item in probeGroups) + AnsiConsole.MarkupLine($"[grey] {Markup.Escape(item.Key)}=[/] [cyan]{Markup.Escape(item.Value)}[/]"); + + if (inactive.Count > 0) + { + AnsiConsole.MarkupLine("[grey] inactive groups:[/]"); + foreach (var group in inactive) + AnsiConsole.MarkupLine($"[grey] {Markup.Escape(group)}=NULL[/]"); + } + + AnsiConsole.MarkupLine($"[grey] old BF16 isolation=[/] [cyan]false[/]"); + AnsiConsole.MarkupLine($"[grey] movement=[/] [cyan]{movement.Classification}[/] [grey]up={movement.UpgradeCount} down={movement.DowngradeCount} same={movement.SameCount} unknown={movement.UnknownCount}[/]"); + } + + private static IReadOnlyList> BuildProbeSubsets(IReadOnlyList changed) + { + var result = new List>(); + + foreach (var item in changed) + result.Add(new[] { item }); + + for (int i = 0; i < changed.Count; i++) + { + for (int j = i + 1; j < changed.Count; j++) + result.Add(new[] { changed[i], changed[j] }); + } + + result.Add(changed.ToList()); + + if (changed.Count >= 3) + { + for (int i = 0; i < changed.Count; i++) + result.Add(changed.Where((_, index) => index != i).ToList()); + } + + return result + .GroupBy(x => string.Join(",", x.OrderBy(g => g.Group.UniqueId).Select(g => g.Group.UniqueId)), StringComparer.Ordinal) + .Select(g => g.First()) + .ToList(); + } + + private static string ResolveProbeType(int subsetCount, int fullCount) + { + if (subsetCount == 1) return "single"; + if (subsetCount == 2) return "pair"; + if (subsetCount == fullCount) return "full"; + return "leave-one-out"; + } + + private static double ComputeSmokeScore(double gap, double savingsPercent, int changedGroupCount, ulong? candidateRank, ulong? twinRank) + { + double maxGap = Math.Max(Config.AnomalyDetection.MaxPredictionSpaceGapVsTwinKld, Config.SynergyDetection.MaxSmokeGapKld); + double closenessScore = maxGap <= 0d ? 0d : Math.Clamp((maxGap - Math.Max(0d, gap)) / maxGap, 0d, 1d); + double savingsScore = Math.Clamp(savingsPercent / Math.Max(Config.AnomalyDetection.MinPredictedSizeSavingsVsTwinPercent, 0.01d), 0d, 2d) / 2d; + double groupPenalty = 1d / Math.Max(1, changedGroupCount); + double rankBonus = 0d; + if (candidateRank.HasValue && twinRank.HasValue) + rankBonus = Math.Clamp((double)twinRank.Value - candidateRank.Value, -10_000d, 10_000d) / 20_000d; + + return Math.Clamp((closenessScore * 0.55d) + (savingsScore * 0.30d) + (groupPenalty * 0.10d) + rankBonus, 0d, 1d); + } + + private static void WriteSmokeConsoleSummary(int historicalCount, int duckCount, IReadOnlyList selected) + { + int monotone = selected.Count(x => x.Movement.Classification == AnomalyMovementClassification.MonotoneDowngrade); + int existingTwins = selected.Count(x => x.HasActualTwin); + int missingTwins = selected.Count - existingTwins; + + AnsiConsole.MarkupLine("[yellow]Anomaly smoke scan:[/]"); + AnsiConsole.MarkupLine($"[grey] historical benchmarks scanned smoke=[/] [cyan]{historicalCount:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] DuckDB prediction-space smoke=[/] [cyan]{duckCount:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] monotone downgrade smoke candidates=[/] [cyan]{monotone:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] existing twins found=[/] [cyan]{existingTwins:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] missing twins queued=[/] [cyan]{missingTwins:N0}[/]"); + } + + private static void WriteProbeOutcome(AnomalyProbeResult result) + { + if (result.RuleDirection == AnomalyRuleDirection.Beneficial && result.ProbeSnapshot != null && result.ReferenceSnapshot != null) + { + AnsiConsole.MarkupLine( + $"[green]Counterfactual MDA violation confirmed:[/] candidate={Markup.Escape(HybridBenchmarkRepository.BuildDisplayName(result.ProbeSnapshot.Quant))} " + + $"twin={Markup.Escape(HybridBenchmarkRepository.BuildDisplayName(result.ReferenceSnapshot.Quant))} " + + $"actual candidate KLD={result.ProbeSnapshot.Kld:0.000000} actual twin KLD={result.ReferenceSnapshot.Kld:0.000000} " + + $"gain={result.ActualGainVsTwin:0.000000} classification={result.Classification}"); + return; + } + + if (result.RuleDirection == AnomalyRuleDirection.SuppressionOnly && result.ProbeSnapshot != null && result.ReferenceSnapshot != null) + { + AnsiConsole.MarkupLine( + $"[grey]Smoke rejected / normal gravity confirmed:[/] candidate={Markup.Escape(HybridBenchmarkRepository.BuildDisplayName(result.ProbeSnapshot.Quant))} " + + $"twin={Markup.Escape(HybridBenchmarkRepository.BuildDisplayName(result.ReferenceSnapshot.Quant))} " + + $"actual candidate KLD={result.ProbeSnapshot.Kld:0.000000} actual twin KLD={result.ReferenceSnapshot.Kld:0.000000} persisted suppression={Config.AnomalyDetection.PersistSuppressionResults}"); + return; + } + + AnsiConsole.MarkupLine($"[yellow]Anomaly probe outcome:[/] {Markup.Escape(result.Classification.ToString())} {Markup.Escape(result.Message)}"); + } + + private static async Task ConfigureDuckAsync(DuckDBConnection c, CancellationToken ct) + { + await ExecuteDuckAsync(c, "SET preserve_insertion_order = false;", ct); + await ExecuteDuckAsync(c, $"SET threads = {Math.Max(1, Environment.ProcessorCount)};", ct); + } + + private static async Task ExecuteDuckAsync(DuckDBConnection c, string sql, CancellationToken ct) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = sql; + await cmd.ExecuteNonQueryAsync(ct); + } + + private static void AddConfigParameters(DuckDBCommand cmd, TensorConfig c) + { + cmd.Parameters.Add(new DuckDBParameter { Value = c.BaseQuant }); + cmd.Parameters.Add(new DuckDBParameter { Value = c.Embeddings }); + cmd.Parameters.Add(new DuckDBParameter { Value = c.LmHead }); + cmd.Parameters.Add(new DuckDBParameter { Value = c.AttnQ }); + cmd.Parameters.Add(new DuckDBParameter { Value = c.AttnKV }); + cmd.Parameters.Add(new DuckDBParameter { Value = c.AttnOutput }); + cmd.Parameters.Add(new DuckDBParameter { Value = c.FfnUpGate }); + cmd.Parameters.Add(new DuckDBParameter { Value = c.FfnDown }); + cmd.Parameters.Add(new DuckDBParameter { Value = c.MoeExperts }); + cmd.Parameters.Add(new DuckDBParameter { Value = c.MoeRouter }); + } + + private static async Task WriteJsonAsync(string fileName, object payload, CancellationToken ct) + { + string dir = Cache.ModelMagicQuantDirectory ?? Cache.MagicQuantDirectory ?? Directory.GetCurrentDirectory(); + Directory.CreateDirectory(dir); + string path = Path.Combine(dir, fileName); + await File.WriteAllTextAsync(path, JsonSerializer.Serialize(payload, JsonOptions), ct); + } + + private static async Task WriteFinalManifestAsync(string fileName, object payload, CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(Cache.OutputDirectory)) + return; + + string manifestDir = Path.Combine(Cache.OutputDirectory!, "magicquant-manifest"); + Directory.CreateDirectory(manifestDir); + string path = Path.Combine(manifestDir, fileName); + await File.WriteAllTextAsync(path, JsonSerializer.Serialize(payload, JsonOptions), ct); + } + + + + + private IReadOnlyList BuildSynergyWingSummary( + IReadOnlyList smoke, + IReadOnlyList results, + AnomalyAdjustmentSummary adjustment) + { + var zones = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["Q8→Q6"] = new SynergyWingSummary { Zone = "Q8→Q6" }, + ["Q6→Q5/Q4"] = new SynergyWingSummary { Zone = "Q6→Q5/Q4" }, + ["Q5→IQ4/IQ3"] = new SynergyWingSummary { Zone = "Q5→IQ4/IQ3" }, + ["Other"] = new SynergyWingSummary { Zone = "Other" } + }; + + foreach (var item in smoke) + zones[ResolveWingZone(item.TwinConfig.BaseQuant, item.Movement.ChangedGroups.Select(x => x.CandidateQuantId))].SmokeCount++; + + foreach (var result in results) + { + var zone = zones[ResolveWingZone(result.Plan.ReferenceConfig.BaseQuant, result.Plan.ProbeGroups.Select(x => x.CandidateQuantId))]; + if (result.RuleDirection == AnomalyRuleDirection.Beneficial) + { + zone.ConfirmedBeneficialTemplates++; + zone.ValidationSuccessCount++; + } + else if (result.RuleDirection == AnomalyRuleDirection.Harmful) + { + zone.HarmfulTemplates++; + zone.ValidationFailureCount++; + } + else + { + zone.SuppressionOnlyTemplates++; + zone.ValidationFailureCount++; + } + } + + foreach (var match in adjustment.RuleMatches) + { + string text = JsonSerializer.Serialize(match); + var zone = text.Contains("Q6_", StringComparison.OrdinalIgnoreCase) && text.Contains("Q8_0", StringComparison.OrdinalIgnoreCase) + ? zones["Q8→Q6"] + : text.Contains("Q5", StringComparison.OrdinalIgnoreCase) || text.Contains("Q4", StringComparison.OrdinalIgnoreCase) + ? zones["Q6→Q5/Q4"] + : zones["Other"]; + + if (text.Contains("Harmful", StringComparison.OrdinalIgnoreCase) || text.Contains("Contaminating", StringComparison.OrdinalIgnoreCase)) + zone.CandidateRowsDemoted++; + else + zone.CandidateRowsAdjustedPositively++; + } + + foreach (var zone in zones.Values) + { + if (zone.ConfirmedBeneficialTemplates == 0 && (zone.HarmfulTemplates > 0 || zone.SuppressionOnlyTemplates > 0)) + zone.Explanation = "No nonlinear wing survived because harmful/suppression evidence dominated or no adjusted candidate beat the frontier line."; + else if (zone.ConfirmedBeneficialTemplates == 0 && zone.SmokeCount == 0) + zone.Explanation = "No contextual synergy smoke was strong enough to probe in this fidelity zone."; + else if (zone.ConfirmedBeneficialTemplates > 0) + zone.Explanation = "Confirmed beneficial counterfactual synergy evidence exists in this fidelity zone."; + else + zone.Explanation = "Smoke existed but did not produce confirmed beneficial evidence."; + } + + return zones.Values.ToList(); + } + + private static string ResolveWingZone(byte referenceQuantId, IEnumerable candidateQuantIds) + { + string reference = SafeName(referenceQuantId); + var candidates = candidateQuantIds.Select(SafeName).ToList(); + if (reference.StartsWith("Q8", StringComparison.OrdinalIgnoreCase) && candidates.Any(x => x.Contains("Q6", StringComparison.OrdinalIgnoreCase))) + return "Q8→Q6"; + if (reference.Contains("Q6", StringComparison.OrdinalIgnoreCase) && candidates.Any(x => x.Contains("Q5", StringComparison.OrdinalIgnoreCase) || x.Contains("Q4", StringComparison.OrdinalIgnoreCase) || x.Contains("IQ4", StringComparison.OrdinalIgnoreCase))) + return "Q6→Q5/Q4"; + if ((reference.Contains("Q5", StringComparison.OrdinalIgnoreCase) || reference.Contains("Q4", StringComparison.OrdinalIgnoreCase)) && candidates.Any(x => x.Contains("IQ4", StringComparison.OrdinalIgnoreCase) || x.Contains("IQ3", StringComparison.OrdinalIgnoreCase) || x.Contains("Q3", StringComparison.OrdinalIgnoreCase))) + return "Q5→IQ4/IQ3"; + return "Other"; + } + + private static void WriteSynergyWingConsoleSummary(IReadOnlyList summaries) + { + AnsiConsole.MarkupLine("[yellow]Synergy wing summary:[/]"); + foreach (var zone in summaries) + { + AnsiConsole.MarkupLine($"[grey] {Markup.Escape(zone.Zone)}:[/] beneficial=[cyan]{zone.ConfirmedBeneficialTemplates:N0}[/], harmful=[cyan]{zone.HarmfulTemplates:N0}[/], suppressed=[cyan]{zone.SuppressionOnlyTemplates:N0}[/], adjusted=[cyan]{zone.CandidateRowsAdjustedPositively:N0}[/], demoted=[cyan]{zone.CandidateRowsDemoted:N0}[/]"); + } + } + + private object ToSynergyTemplateLog(AnomalyInteractionRule x) + { + Dictionary selected = new(StringComparer.OrdinalIgnoreCase); + Dictionary raised = new(StringComparer.OrdinalIgnoreCase); + foreach (var state in x.GroupStates.OrderBy(g => g.SortOrder)) + { + string groupName = ResolveGroupName(state.TensorGroupId); + selected[groupName] = SafeName(state.CandidateQuantId); + raised[groupName] = SafeName(state.ReferenceQuantId); + } + + return new + { + templateType = x.RuleType, + referenceQuant = SafeName(x.ReferenceQuantId), + selectedGroupStates = selected, + raisedCounterfactualStates = raised, + discoveryContext = TryDeserializeDictionary(x.ReferenceEffectiveGroupsJson), + candidateContext = TryDeserializeDictionary(x.CandidateEffectiveGroupsJson), + actualGainKld = x.BestActualGainVsTwin, + sizeSavingsBytes = x.MetadataJson?.Contains("sizeSavingsBytes", StringComparison.OrdinalIgnoreCase) == true ? (object?)"see metadataJson" : null, + x.Confidence, + generalizationPolicy = "ExactStrong_TransferWeak", + metadataJson = x.MetadataJson + }; + } + + private static Dictionary TryDeserializeDictionary(string? json) + { + if (string.IsNullOrWhiteSpace(json)) + return new Dictionary(StringComparer.OrdinalIgnoreCase); + try + { + return JsonSerializer.Deserialize>(json) ?? new Dictionary(StringComparer.OrdinalIgnoreCase); + } + catch + { + return new Dictionary(StringComparer.OrdinalIgnoreCase); + } + } + + private static string ResolveGroupName(byte groupId) + { + var group = TReg.All.FirstOrDefault(x => x.UniqueId == groupId); + return group?.Name ?? $"group:{groupId}"; + } + + private object ToSmokeLog(AnomalySmokeCandidate x) + { + return new + { + x.Source, + isContextualAnomalySmoke = true, + oldBf16Isolation = false, + allActiveGroupsExplicit = _movement.HasAllActiveGroupsExplicit(x.CandidateConfig) && _movement.HasAllActiveGroupsExplicit(x.TwinConfig), + candidate = TensorConfigIdentity.ToKey(x.CandidateConfig), + twin = TensorConfigIdentity.ToKey(x.TwinConfig), + candidateName = HybridBenchmarkRepository.BuildDisplayName(x.CandidateQuant), + twinName = HybridBenchmarkRepository.BuildDisplayName(x.TwinQuant), + referenceEffectiveGroups = _movement.BuildEffectiveGroupVector(x.TwinConfig), + candidateEffectiveGroups = _movement.BuildEffectiveGroupVector(x.CandidateConfig), + inactiveGroups = _movement.BuildInactiveGroupList(), + movement = x.Movement.Classification.ToString(), + x.Movement.UpgradeCount, + x.Movement.DowngradeCount, + x.Movement.SameCount, + x.Movement.UnknownCount, + changedGroups = x.Movement.ChangedGroups.Select(g => new + { + group = g.Group.Name, + candidate = SafeName(g.CandidateQuantId), + reference = SafeName(g.ReferenceQuantId), + movement = g.Movement.ToString() + }).ToList(), + x.CandidatePredictedKld, + x.TwinPredictedKld, + x.PredictionSpaceGapVsTwin, + x.CandidatePredictedSizeBytes, + x.TwinPredictedSizeBytes, + x.PredictedSizeSavingsBytes, + x.CandidateActualSizeBytes, + x.TwinActualSizeBytes, + x.ActualSizeSavingsBytes, + x.PlannedProbeWillMeasureSize, + x.TwinLookupMode, + x.RejectionReason, + x.MatchedConfirmedAnomalyPattern, + x.TwinFoundInLookupDictionary, + seedClass = x.SeedClass.ToString(), + x.CandidatePredictionRank, + x.TwinPredictionRank, + x.SmokeScore, + x.SmokeStrength, + x.HasActualTwin, + x.CandidateActualKld, + x.TwinActualKld, + x.IsConfirmedFromHistory, + x.Message + }; + } + + private object ToProbeLog(AnomalyProbePlan x) + { + var movement = _movement.Analyze(x.ReferenceConfig, x.ProbeConfig); + return new + { + isContextualAnomalyProbe = true, + oldBf16Isolation = false, + allActiveGroupsExplicit = _movement.HasAllActiveGroupsExplicit(x.ReferenceConfig) && _movement.HasAllActiveGroupsExplicit(x.ProbeConfig), + reference = TensorConfigIdentity.ToKey(x.ReferenceConfig), + probe = TensorConfigIdentity.ToKey(x.ProbeConfig), + referenceName = HybridBenchmarkRepository.BuildDisplayName((HybridQuant)x.ReferenceConfig), + probeName = HybridBenchmarkRepository.BuildDisplayName((HybridQuant)x.ProbeConfig), + referenceEffectiveGroups = _movement.BuildEffectiveGroupVector(x.ReferenceConfig), + candidateEffectiveGroups = _movement.BuildEffectiveGroupVector(x.ProbeConfig), + inactiveGroups = _movement.BuildInactiveGroupList(), + movementClassification = movement.Classification.ToString(), + movement.UpgradeCount, + movement.DowngradeCount, + movement.SameCount, + movement.UnknownCount, + x.ProbeType, + x.HypothesisLabel, + seedClass = x.SeedClass.ToString(), + probePriorityClass = x.ProbePriorityClass.ToString(), + groups = x.ProbeGroups.Select(g => new + { + group = g.Group.Name, + candidate = SafeName(g.CandidateQuantId), + reference = SafeName(g.ReferenceQuantId) + }).ToList() + }; + } + + private object ToResultLog(AnomalyProbeResult x) + { + return new + { + probe = ToProbeLog(x.Plan), + classification = x.Classification.ToString(), + direction = x.RuleDirection.ToString(), + x.Accepted, + x.ActualGainVsTwin, + referenceKld = x.ReferenceSnapshot?.Kld, + probeKld = x.ProbeSnapshot?.Kld, + referenceSizeBytes = x.ReferenceSnapshot?.SizeBytes, + probeSizeBytes = x.ProbeSnapshot?.SizeBytes, + x.FailureCode, + x.Message + }; + } + + private static object ToRuleLog(AnomalyInteractionRule x) + { + return new + { + x.Id, + x.RuleType, + x.RuleDirection, + x.RuleStatus, + x.ReferenceQuantId, + referenceQuant = SafeName(x.ReferenceQuantId), + x.ReferenceContextKey, + x.ReferenceEffectiveGroupsJson, + x.CandidateEffectiveGroupsJson, + x.InactiveGroupsJson, + x.FullTensorConfigKey, + x.GroupSetHash, + x.GroupCount, + x.MeanActualGainVsTwin, + x.BestActualGainVsTwin, + x.MeanPredictionSpaceGap, + x.BestPredictionSpaceGap, + x.AppliedPredictionSpaceAdjustmentKld, + x.EvidenceCount, + x.Confidence, + groups = x.GroupStates.OrderBy(g => g.SortOrder).Select(g => new + { + g.TensorGroupId, + candidate = SafeName(g.CandidateQuantId), + reference = SafeName(g.ReferenceQuantId), + g.Movement + }).ToList() + }; + } + + private static byte ToByte(object? value) + { + if (value is null || value is DBNull) return 0; + if (value is BigInteger big) return (byte)big; + return Convert.ToByte(value, CultureInfo.InvariantCulture); + } + + private static double ToDouble(object? value) + { + if (value is null || value is DBNull) return 0d; + if (value is BigInteger big) return (double)big; + return Convert.ToDouble(value, CultureInfo.InvariantCulture); + } + + private static ulong ToUInt64(object? value) + { + if (value is null || value is DBNull) return 0UL; + if (value is BigInteger big) return (ulong)big; + return Convert.ToUInt64(value, CultureInfo.InvariantCulture); + } + + private static string SafeName(byte quantId) + { + try + { + return BaselineQuants.FromId(quantId).Names[0]; + } + catch + { + return $"id:{quantId}"; + } + } + + + private sealed record SynergyTransferContext(byte QuantId, string Stratum); + + private sealed record SynergyTransferTemplateGroup(TensorGroup Group, byte CandidateQuantId); + + private sealed record SynergyTransferTemplate( + string Key, + string Source, + byte SourceReferenceQuantId, + double Confidence, + double ActualEffectMagnitude, + IReadOnlyList Groups); + + private sealed record SynergyTransferCandidate( + string TemplateKey, + string IdentityKey, + AnomalyProbePlan Plan, + int GroupCount, + double CandidateTierDistance, + double Confidence, + double ActualEffectMagnitude); + + private sealed record ExploratoryIsolationObservation( + TensorGroup Group, + BaselineQuants Baseline, + BenchmarkSnapshotRecord Snapshot); + + private sealed record ExploratoryIsolationPair( + TensorGroup Group, + ExploratoryIsolationObservation Winner, + ExploratoryIsolationObservation Contender, + double IsolationGap); + + private sealed record ExploratoryContextPairCandidate( + string IdentityKey, + ExploratoryIsolationPair Pair, + AnomalyProbePlan Plan); + + + private sealed class RejectedSmokePreview + { + public RejectedSmokePreview( + int sortOrder, + TensorConfig candidate, + TensorConfig twin, + AnomalyMovementAnalysis? movement, + PredictionDuckRow? candidateRow, + PredictionDuckRow? twinRow, + ulong? predictedSizeSavingsBytes, + double? predictionSpaceGap, + string rejectionReason, + bool matchedConfirmedAnomalyPattern, + bool twinFoundInLookup) + { + SortOrder = sortOrder; + Candidate = candidate; + Twin = twin; + Movement = movement; + CandidateRow = candidateRow; + TwinRow = twinRow; + PredictedSizeSavingsBytes = predictedSizeSavingsBytes; + PredictionSpaceGap = predictionSpaceGap; + RejectionReason = rejectionReason; + MatchedConfirmedAnomalyPattern = matchedConfirmedAnomalyPattern; + TwinFoundInLookup = twinFoundInLookup; + } + + public int SortOrder { get; } + public TensorConfig Candidate { get; } + public TensorConfig Twin { get; } + public AnomalyMovementAnalysis? Movement { get; } + public PredictionDuckRow? CandidateRow { get; } + public PredictionDuckRow? TwinRow { get; } + public ulong? PredictedSizeSavingsBytes { get; } + public double? PredictionSpaceGap { get; } + public string RejectionReason { get; } + public bool MatchedConfirmedAnomalyPattern { get; } + public bool TwinFoundInLookup { get; } + public string CandidateName => HybridBenchmarkRepository.BuildDisplayName((HybridQuant)Candidate); + public string TwinName => HybridBenchmarkRepository.BuildDisplayName((HybridQuant)Twin); + + public object ToLog() => new + { + candidate = TensorConfigIdentity.ToKey(Candidate), + twin = TensorConfigIdentity.ToKey(Twin), + candidateName = CandidateName, + twinName = TwinName, + movement = Movement?.Classification.ToString() ?? "Unknown", + changedGroups = Movement?.ChangedGroups.Select(g => $"{g.Group.ShortCode}={AnomalyWorkflowService.SafeName(g.CandidateQuantId)}").ToList() ?? new List(), + smokeScore = Movement == null || CandidateRow == null || TwinRow == null || !PredictionSpaceGap.HasValue || !PredictedSizeSavingsBytes.HasValue + ? (double?)null + : AnomalyWorkflowService.ComputeSmokeScore(PredictionSpaceGap.Value, PredictedSizeSavingsBytes.Value * 100d / Math.Max(1d, TwinRow.PredictedSizeBytes), Movement.DowngradeCount, CandidateRow.PredictionRank, TwinRow.PredictionRank), + predictedCandidateKld = CandidateRow?.BaseRankSafeKld, + predictedTwinKld = TwinRow?.BaseRankSafeKld, + predictionSpaceGap = PredictionSpaceGap, + predictedCandidateSizeBytes = CandidateRow?.PredictedSizeBytes, + predictedTwinSizeBytes = TwinRow?.PredictedSizeBytes, + predictedSizeSavingsBytes = PredictedSizeSavingsBytes, + rejectionReason = RejectionReason, + matchedConfirmedAnomalyPattern = MatchedConfirmedAnomalyPattern, + twinExistedInLookupDictionary = TwinFoundInLookup + }; + } + + private sealed record PredictionDuckRow( + TensorConfig Config, + double BaseRankSafeKld, + double FinalPredictedKld, + ulong PredictedSizeBytes, + double PredictionConfidence, + ulong PredictionRank); +} diff --git a/src/MagicQuant/Services/ArchitectureFamilyService.cs b/src/MagicQuant/Services/ArchitectureFamilyService.cs new file mode 100644 index 0000000..a6f18f1 --- /dev/null +++ b/src/MagicQuant/Services/ArchitectureFamilyService.cs @@ -0,0 +1,249 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using MagicQuant.Helpers; +using Microsoft.EntityFrameworkCore; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models.DbModels; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class ArchitectureFamilyService +{ + private readonly PythonManager _python; + + public ArchitectureFamilyService(PythonManager python) + { + _python = python; + } + + public async Task EnsureCurrentArchitectureFamilyAsync(string bf16GgufPath, CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(Cache.CurrentArchitectureFamilyName)) + throw new InvalidOperationException("Architecture family is required. Provide --architecture-family or set identity.architecture_family_name in YAML."); + + if (string.IsNullOrWhiteSpace(Cache.CurrentModelId)) + throw new InvalidOperationException("Cache.CurrentModelId is not set."); + + var tensorNames = await ReadTensorNamesFromGgufAsync(bf16GgufPath, ct); + if (tensorNames.Count == 0) + throw new InvalidOperationException("Architecture-family validation could not read any tensor names from the BF16 GGUF."); + + string signatureHash = ComputeTensorSignatureHash(tensorNames); + int tensorCount = tensorNames.Count; + string normalized = NormalizeFamilyName(Cache.CurrentArchitectureFamilyName); + + await using var db = new MagicQuantContext(); + + var aiModelHash = await db.AiModelHashes.FirstOrDefaultAsync(x => x.UniqueHash == Cache.CurrentModelId, ct); + if (aiModelHash == null) + { + aiModelHash = new AiModelHash { UniqueHash = Cache.CurrentModelId }; + db.AiModelHashes.Add(aiModelHash); + await db.SaveChangesAsync(ct); + } + + var existingMapping = await db.Set() + .Include(x => x.ArchitectureFamily) + .FirstOrDefaultAsync(x => x.AiModelHashId == aiModelHash.Id, ct); + + if (existingMapping != null) + { + if (!string.Equals(existingMapping.ArchitectureFamily.NormalizedName, normalized, StringComparison.Ordinal)) + { + throw new InvalidOperationException($"Current model hash is already mapped to architecture family '{existingMapping.ArchitectureFamily.DisplayName}'."); + } + + Cache.CurrentArchitectureFamilyId = existingMapping.ArchitectureFamilyId; + Cache.CurrentArchitectureFamilyName = existingMapping.ArchitectureFamily.DisplayName; + return; + } + + var matchingName = await db.Set() + .FirstOrDefaultAsync(x => x.NormalizedName == normalized, ct); + + if (matchingName != null) + { + if (matchingName.TensorCount != tensorCount || !string.Equals(matchingName.TensorSignatureHash, signatureHash, StringComparison.Ordinal)) + { + throw new InvalidOperationException($"Architecture family '{matchingName.DisplayName}' already exists, but the current model tensor names/count do not match the previously registered architecture. Expected count={matchingName.TensorCount}, actual count={tensorCount}."); + } + + bool familyAlreadyHasHashes = await db.Set() + .AnyAsync(x => x.ArchitectureFamilyId == matchingName.Id, ct); + + if (familyAlreadyHasHashes && !Cache.AllowArchitectureFamilyAliasOverride) + { + throw new InvalidOperationException( + $"Architecture family '{matchingName.DisplayName}' already has one or more model hashes attached. " + + "Adding the current hash means you are manually asserting these different model hashes share the same tensor architecture/truth. " + + "Rerun with --allow-architecture-family-alias-override only if you intentionally approve this shared-family linkage."); + } + + db.Add(new ArchitectureFamilyModelHash + { + ArchitectureFamilyId = matchingName.Id, + AiModelHashId = aiModelHash.Id, + IsCanonical = false + }); + await db.SaveChangesAsync(ct); + + Cache.CurrentArchitectureFamilyId = matchingName.Id; + Cache.CurrentArchitectureFamilyName = matchingName.DisplayName; + AnsiConsole.MarkupLine($"[green]Architecture family linked:[/] [cyan]{Markup.Escape(matchingName.DisplayName)}[/] -> model hash [grey]{Markup.Escape(Cache.CurrentModelId)}[/]"); + return; + } + + var sameSignatureFamilies = await db.Set() + .Where(x => x.TensorCount == tensorCount && x.TensorSignatureHash == signatureHash) + .OrderBy(x => x.DisplayName) + .ToListAsync(ct); + + if (sameSignatureFamilies.Count > 0 && !Cache.AllowArchitectureFamilyAliasOverride) + { + throw new InvalidOperationException( + $"The provided architecture family '{Cache.CurrentArchitectureFamilyName}' matches an existing architecture signature already registered under: {string.Join(", ", sameSignatureFamilies.Select(x => x.DisplayName))}. Use one of those names or rerun with --allow-architecture-family-alias-override if you intentionally want a separate family namespace."); + } + + var family = new ArchitectureFamily + { + NormalizedName = normalized, + DisplayName = Cache.CurrentArchitectureFamilyName.Trim(), + TensorSignatureHash = signatureHash, + TensorCount = tensorCount, + CreatedUtc = DateTime.UtcNow + }; + db.Add(family); + await db.SaveChangesAsync(ct); + + db.Add(new ArchitectureFamilyModelHash + { + ArchitectureFamilyId = family.Id, + AiModelHashId = aiModelHash.Id, + IsCanonical = true + }); + await db.SaveChangesAsync(ct); + + Cache.CurrentArchitectureFamilyId = family.Id; + Cache.CurrentArchitectureFamilyName = family.DisplayName; + AnsiConsole.MarkupLine($"[green]Architecture family created:[/] [cyan]{Markup.Escape(family.DisplayName)}[/] tensors={tensorCount:N0}"); + } + + public static async Task ResolveExactCurrentAiModelHashIdOrNullAsync(MagicQuantContext db, CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(Cache.CurrentModelId)) + return null; + + return await db.AiModelHashes + .AsNoTracking() + .Where(x => x.UniqueHash == Cache.CurrentModelId) + .Select(x => (uint?)x.Id) + .FirstOrDefaultAsync(ct); + } + + public static async Task ResolveExactCurrentAiModelHashIdAsync(MagicQuantContext db, CancellationToken ct = default) + { + var id = await ResolveExactCurrentAiModelHashIdOrNullAsync(db, ct); + if (id == null) + throw new InvalidOperationException("Unable to resolve the exact current AiModelHashId."); + return id.Value; + } + + public static async Task ResolveScopedAiModelHashIdOrNullAsync(MagicQuantContext db, CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(Cache.CurrentModelId)) + return null; + + var current = await db.AiModelHashes.AsNoTracking().FirstOrDefaultAsync(x => x.UniqueHash == Cache.CurrentModelId, ct); + if (current == null) + return null; + + if (Cache.CurrentArchitectureFamilyId == null) + return current.Id; + + var canonical = await db.Set() + .AsNoTracking() + .Where(x => x.ArchitectureFamilyId == Cache.CurrentArchitectureFamilyId.Value) + .OrderByDescending(x => x.IsCanonical) + .ThenBy(x => x.AiModelHashId) + .Select(x => (uint?)x.AiModelHashId) + .FirstOrDefaultAsync(ct); + + return canonical ?? current.Id; + } + + public static async Task ResolveScopedAiModelHashIdAsync(MagicQuantContext db, CancellationToken ct = default) + { + var id = await ResolveScopedAiModelHashIdOrNullAsync(db, ct); + if (id == null) + throw new InvalidOperationException("Unable to resolve the current scoped AiModelHashId."); + return id.Value; + } + + private static string NormalizeFamilyName(string value) => value.Trim().ToLowerInvariant(); + + private static string ComputeTensorSignatureHash(IReadOnlyCollection tensorNames) + { + using var sha = SHA256.Create(); + var payload = string.Join("\n", tensorNames.OrderBy(x => x, StringComparer.Ordinal)); + var hash = sha.ComputeHash(Encoding.UTF8.GetBytes(payload)); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private async Task> ReadTensorNamesFromGgufAsync(string ggufPath, CancellationToken ct) + { + string workingDir = Path.Combine(Cache.ModelMagicQuantDirectory ?? Path.GetTempPath(), "_architecture_family"); + Directory.CreateDirectory(workingDir); + string unique = Guid.NewGuid().ToString("N"); + string payloadPath = Path.Combine(workingDir, $"arch_payload_{unique}.json"); + string resultPath = Path.Combine(workingDir, $"arch_result_{unique}.json"); + string scriptPath = Path.Combine(workingDir, $"arch_script_{unique}.py"); + + try + { + await File.WriteAllTextAsync(payloadPath, JsonSerializer.Serialize(new { gguf_path = ggufPath, output_path = resultPath }), ct); + const string py = """ +import json +import sys +payload_path = sys.argv[1] +with open(payload_path, 'r', encoding='utf-8') as f: + payload = json.load(f) +try: + import gguf + reader = gguf.GGUFReader(payload['gguf_path']) + names = [t.name for t in reader.tensors] + result = {'TensorNames': names, 'Error': None} +except Exception as e: + result = {'TensorNames': [], 'Error': str(e)} +with open(payload['output_path'], 'w', encoding='utf-8') as f: + json.dump(result, f, indent=2) +"""; + await File.WriteAllTextAsync(scriptPath, py, ct); + await _python.RunPythonScriptAsync(scriptPath, $"\"{payloadPath}\""); + using var stream = File.OpenRead(resultPath); + using var doc = await JsonDocument.ParseAsync(stream, cancellationToken: ct); + var root = doc.RootElement; + var err = root.TryGetProperty("Error", out var e) && e.ValueKind != JsonValueKind.Null ? e.GetString() : null; + if (!string.IsNullOrWhiteSpace(err)) + throw new InvalidOperationException($"Failed to read GGUF tensor names for architecture family validation: {err}"); + return root.GetProperty("TensorNames").EnumerateArray() + .Select(x => x.GetString() ?? string.Empty) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .OrderBy(x => x, StringComparer.Ordinal) + .ToList(); + } + finally + { + TryDelete(payloadPath); + TryDelete(resultPath); + TryDelete(scriptPath); + } + } + + private static void TryDelete(string path) + { + try { if (File.Exists(path)) File.Delete(path); } catch { } + } +} \ No newline at end of file diff --git a/src/MagicQuant/Services/BaselineDefinitionResolver.cs b/src/MagicQuant/Services/BaselineDefinitionResolver.cs new file mode 100644 index 0000000..845371a --- /dev/null +++ b/src/MagicQuant/Services/BaselineDefinitionResolver.cs @@ -0,0 +1,111 @@ +using Microsoft.EntityFrameworkCore; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; + +namespace MagicQuant.Services; + +public static class BaselineDefinitionResolver +{ + public static string NormalizeRepoId(string value) => (value ?? string.Empty).Trim().ToLowerInvariant(); + + public static string NormalizeFileName(string value) => (value ?? string.Empty).Trim().Replace('\\', '/').ToLowerInvariant(); + + public static string NormalizeCanonicalKey(string value) => (value ?? string.Empty).Trim().ToLowerInvariant(); + + public static string BuildCustomCanonicalKey(string architectureFamilyName, string repoId, string fileName) => + $"custom:{NormalizeRepoId(architectureFamilyName)}:{NormalizeRepoId(repoId)}:{NormalizeFileName(fileName)}"; + + public static async Task ResolveRequiredDefinitionAsync( + MagicQuantContext db, + BaselineQuants baseline, + CancellationToken ct = default) + { + int? familyId = baseline.IsCustomBaseline + ? TensorGroupProfileService.RequireCurrentArchitectureFamilyId() + : null; + + string normalizedKey = NormalizeCanonicalKey(baseline.CanonicalKey); + + var definition = await db.BaselineQuantDefinitions + .AsNoTracking() + .FirstOrDefaultAsync(x => + x.ArchitectureFamilyId == familyId && + x.RuntimeBaselineId == baseline.UniqueId, + ct) + ?? await db.BaselineQuantDefinitions + .AsNoTracking() + .FirstOrDefaultAsync(x => + x.ArchitectureFamilyId == familyId && + x.NormalizedCanonicalKey == normalizedKey, + ct); + + if (definition == null) + { + throw new InvalidOperationException( + $"Baseline definition was not found in SQLite for runtime id {baseline.UniqueId} / key '{baseline.CanonicalKey}'. " + + "Run custom baseline precheck/sync before using learned or historical tensor-combo truth."); + } + + return definition; + } + + public static async Task TryResolveDefinitionByRuntimeIdAsync( + MagicQuantContext db, + byte runtimeBaselineId, + int? architectureFamilyId, + CancellationToken ct = default) + { + if (architectureFamilyId != null) + { + var custom = await db.BaselineQuantDefinitions + .AsNoTracking() + .FirstOrDefaultAsync(x => x.ArchitectureFamilyId == architectureFamilyId.Value && x.RuntimeBaselineId == runtimeBaselineId, ct); + + if (custom != null) + return custom; + } + + return await db.BaselineQuantDefinitions + .AsNoTracking() + .FirstOrDefaultAsync(x => x.ArchitectureFamilyId == null && x.RuntimeBaselineId == runtimeBaselineId, ct); + } + + public static BaselineQuants ToRuntimeBaseline(BaselineQuantDefinition definition, bool forceInactiveRegistration = false) + { + if (!definition.IsCustomBaseline) + return BaselineQuants.FromId(definition.RuntimeBaselineId); + + var scheme = TensorWeightScheme.FromId(definition.DefaultTensorSchemeId); + var baseline = BaselineQuants.CreateDynamicCustomBaseline( + uniqueId: definition.RuntimeBaselineId, + displayName: string.IsNullOrWhiteSpace(definition.DisplayName) ? definition.BaselineName : definition.DisplayName, + quantizeBaseArgumentName: definition.QuantizeBaseArgumentName, + sourceRepository: definition.SourceRepository ?? string.Empty, + sourceFileName: definition.SourceFileName ?? string.Empty, + shortSourceName: definition.ShortSourceName ?? "External", + sourceOwner: definition.SourceOwner ?? string.Empty, + sourceKind: definition.SourceKind, + canonicalKey: definition.CanonicalKey, + primaryTensorWeightScheme: scheme, + learnedMatchTensorWeightSchemes: [scheme], + bannedGroupIds: Array.Empty(), + requiresImatrix: definition.RequiresImatrix, + isLearningBaseline: definition.IsActiveInCurrentConfig && definition.IsLearningBaseline, + isCombinationCarrierCandidate: definition.IsActiveInCurrentConfig && definition.IsCombinationCarrierCandidate, + isExplicitGroupCombinationCandidate: definition.IsActiveInCurrentConfig && definition.IsExplicitGroupCombinationCandidate, + bitRange: definition.BitRange, + explicitCandidateSortOrder: definition.ExplicitCandidateSortOrder); + + if (definition.IsActiveInCurrentConfig || forceInactiveRegistration) + return baseline; + + return baseline with + { + IsLearningBaseline = false, + IsCombinationCarrierCandidate = false, + IsExplicitGroupCombinationCandidate = false + }; + } +} diff --git a/src/MagicQuant/Services/BenchmarkCommands.cs b/src/MagicQuant/Services/BenchmarkCommands.cs new file mode 100644 index 0000000..4905548 --- /dev/null +++ b/src/MagicQuant/Services/BenchmarkCommands.cs @@ -0,0 +1,34 @@ +using System.Globalization; +using MagicQuant.Runtime; + +namespace MagicQuant.Services; + +/// Native benchmark arguments, separated from scheduling and measured-truth persistence. +internal static class BenchmarkCommands +{ + public static NativeCommand Bench(string executable, string model, bool gpu, int ngl, string tensorSplit) + { + List args = ["-m", model, "-p", "8", "-t", "16"]; + if (gpu) args.AddRange(["-ngl", ngl.ToString(CultureInfo.InvariantCulture), .. SplitArgs(tensorSplit)]); + else args.AddRange(["-ngl", "0"]); + args.AddRange(["-o", "md"]); + return new NativeCommand(executable, args); + } + + public static NativeCommand Perplexity(string executable, string model, string corpus, bool gpu, + int ngl, string tensorSplit, string? logitsFile = null, bool compareLogits = false) + { + List args = ["-m", model, "-ngl", (gpu ? ngl : 0).ToString(CultureInfo.InvariantCulture)]; + if (gpu) args.AddRange(SplitArgs(tensorSplit)); + args.AddRange(["-t", "4", "-c", "2048", "--file", corpus]); + if (logitsFile != null) + { + args.AddRange(["--kl-divergence-base", logitsFile]); + if (compareLogits) args.Add("--kl-divergence"); + } + return new NativeCommand(executable, args); + } + + // This fragment is emitted only by LlamaGpuArgumentBuilder (flag + numeric vector). + private static string[] SplitArgs(string tensorSplit) => tensorSplit.Split(' ', StringSplitOptions.RemoveEmptyEntries); +} diff --git a/src/MagicQuant/Services/BenchmarkGpuPlanning.cs b/src/MagicQuant/Services/BenchmarkGpuPlanning.cs new file mode 100644 index 0000000..12bb1b6 --- /dev/null +++ b/src/MagicQuant/Services/BenchmarkGpuPlanning.cs @@ -0,0 +1,461 @@ +using System.Text.Json; + +namespace MagicQuant.Services; + +internal sealed record GpuProbeSample( + int Ngl, + bool Success, + double SecondsPerPass, + double ElapsedSeconds); + +internal sealed record BenchmarkSlot( + int SlotId, + string ProfileName, + int[] DeviceIndices, + int Q8StableNgl, + IReadOnlyList ProbeSamples) +{ + public BenchmarkSlot(int slotId, int[] deviceIndices) + : this(slotId, "default", deviceIndices, 0, []) + { + } + + public bool UsesGpu => DeviceIndices.Length > 0; + public int DeviceCount => DeviceIndices.Length; + + public string DisplayName => + UsesGpu + ? $"{ProfileName}:GPU[{string.Join(",", DeviceIndices)}]" + : "CPU"; + + public IReadOnlyDictionary? BuildProcessEnv() + { + if (!UsesGpu) + return null; + + string visible = string.Join(",", DeviceIndices); + return new Dictionary(StringComparer.OrdinalIgnoreCase) + { + ["CUDA_VISIBLE_DEVICES"] = visible, + ["HIP_VISIBLE_DEVICES"] = visible, + ["ROCR_VISIBLE_DEVICES"] = visible + }; + } +} + +internal sealed record BenchmarkTopologyProfile( + string Name, + IReadOnlyList Slots, + double MeasuredJobsPerSecond, + double MeasuredSecondsPerPass); + +internal sealed class BenchmarkTopologyCacheEnvelope +{ + public int Version { get; set; } = 1; + public int MaxOffloadNgl { get; set; } + public ulong IndependentMaxModelSizeBytes { get; set; } + public BenchmarkTopologyCacheProfile Shared { get; set; } = new(); + public BenchmarkTopologyCacheProfile Independent { get; set; } = new(); +} + +internal sealed class BenchmarkTopologyCacheProfile +{ + public string Name { get; set; } = string.Empty; + public double MeasuredJobsPerSecond { get; set; } + public double MeasuredSecondsPerPass { get; set; } + public List Slots { get; set; } = new(); +} + +internal sealed class BenchmarkTopologyCacheSlot +{ + public int SlotId { get; set; } + public int[] DeviceIndices { get; set; } = []; + public int Q8StableNgl { get; set; } + public List ProbeSamples { get; set; } = new(); +} + +internal static class BenchmarkTopologyCacheCodec +{ + public static string Serialize( + int maxOffloadNgl, + ulong independentMaxModelSizeBytes, + BenchmarkTopologyProfile shared, + BenchmarkTopologyProfile independent) + { + var envelope = new BenchmarkTopologyCacheEnvelope + { + MaxOffloadNgl = maxOffloadNgl, + IndependentMaxModelSizeBytes = independentMaxModelSizeBytes, + Shared = ToCacheProfile(shared), + Independent = ToCacheProfile(independent) + }; + + return JsonSerializer.Serialize(envelope); + } + + public static bool TryDeserialize( + string json, + out int maxOffloadNgl, + out ulong independentMaxModelSizeBytes, + out BenchmarkTopologyProfile? shared, + out BenchmarkTopologyProfile? independent) + { + maxOffloadNgl = 0; + independentMaxModelSizeBytes = 0; + shared = null; + independent = null; + + try + { + var envelope = JsonSerializer.Deserialize(json); + if (envelope == null || + envelope.Version != 1 || + envelope.MaxOffloadNgl <= 0 || + envelope.Shared.Slots.Count == 0) + { + return false; + } + + maxOffloadNgl = envelope.MaxOffloadNgl; + independentMaxModelSizeBytes = envelope.IndependentMaxModelSizeBytes; + shared = FromCacheProfile(envelope.Shared); + independent = FromCacheProfile(envelope.Independent); + return true; + } + catch (JsonException) + { + return false; + } + } + + private static BenchmarkTopologyCacheProfile ToCacheProfile(BenchmarkTopologyProfile profile) + => new() + { + Name = profile.Name, + MeasuredJobsPerSecond = profile.MeasuredJobsPerSecond, + MeasuredSecondsPerPass = profile.MeasuredSecondsPerPass, + Slots = profile.Slots.Select(x => new BenchmarkTopologyCacheSlot + { + SlotId = x.SlotId, + DeviceIndices = x.DeviceIndices, + Q8StableNgl = x.Q8StableNgl, + ProbeSamples = x.ProbeSamples.ToList() + }).ToList() + }; + + private static BenchmarkTopologyProfile FromCacheProfile(BenchmarkTopologyCacheProfile profile) + => new( + string.IsNullOrWhiteSpace(profile.Name) ? "cached" : profile.Name, + profile.Slots.Select(x => new BenchmarkSlot( + x.SlotId, + string.IsNullOrWhiteSpace(profile.Name) ? "cached" : profile.Name, + x.DeviceIndices ?? [], + x.Q8StableNgl, + x.ProbeSamples ?? [])).ToList(), + profile.MeasuredJobsPerSecond, + profile.MeasuredSecondsPerPass); +} + +internal static class BenchmarkGpuPlanner +{ + internal const double DefaultIndependentSpeedupMargin = 1.10d; + internal const int NearFullOffloadToleranceLayers = 1; + + public static bool ShouldUseIndependentTopology( + ulong modelSizeBytes, + ulong independentMaxModelSizeBytes, + int independentSlotCount, + bool allowIndependentTopology) + => allowIndependentTopology && + independentMaxModelSizeBytes > 0 && + modelSizeBytes > 0 && + modelSizeBytes <= independentMaxModelSizeBytes && + independentSlotCount > 0; + + public static int ResolveNglForModel( + ulong q8ModelSizeBytes, + int q8StableNgl, + int maxOffloadNgl, + ulong modelSizeBytes) + { + if (q8StableNgl <= 0 || maxOffloadNgl <= 0) + return 0; + if (q8ModelSizeBytes == 0 || modelSizeBytes == 0) + return Math.Min(q8StableNgl, maxOffloadNgl); + + double scaled = Math.Floor(q8StableNgl * (q8ModelSizeBytes / (double)modelSizeBytes)); + return (int)Math.Clamp(scaled, 0d, maxOffloadNgl); + } + + public static IReadOnlyList RankIndependentSlotsForModel( + IReadOnlyList slots, + ulong q8ModelSizeBytes, + int maxOffloadNgl, + ulong modelSizeBytes, + int nearFullOffloadToleranceLayers = NearFullOffloadToleranceLayers) + { + ArgumentNullException.ThrowIfNull(slots); + if (slots.Count <= 1 || q8ModelSizeBytes == 0 || modelSizeBytes == 0 || maxOffloadNgl <= 0) + return slots; + + int nearFullThreshold = Math.Max(0, maxOffloadNgl - Math.Max(0, nearFullOffloadToleranceLayers)); + var ranked = slots + .Select(slot => new + { + Slot = slot, + Ngl = ResolveNglForModel( + q8ModelSizeBytes, + slot.Q8StableNgl, + maxOffloadNgl, + modelSizeBytes) + }) + .ToArray(); + + // A weaker device that is at most one layer shy of full offload is the best fit: + // using it preserves the stronger device for a larger concurrent model. When only + // one device is close to full offload, prefer that device. Otherwise use the device + // that can offload the most layers and accept the unavoidable partial-offload tail. + if (ranked.Any(x => x.Ngl >= nearFullThreshold)) + { + return ranked + .OrderByDescending(x => x.Ngl >= nearFullThreshold) + .ThenBy(x => x.Ngl >= nearFullThreshold ? x.Slot.Q8StableNgl : int.MaxValue) + .ThenByDescending(x => x.Ngl) + .ThenBy(x => x.Slot.SlotId) + .Select(x => x.Slot) + .ToArray(); + } + + return ranked + .OrderByDescending(x => x.Ngl) + .ThenByDescending(x => x.Slot.Q8StableNgl) + .ThenBy(x => x.Slot.SlotId) + .Select(x => x.Slot) + .ToArray(); + } + + public static ulong EstimateIndependentCrossoverBytes( + ulong q8ModelSizeBytes, + int maxOffloadNgl, + double sharedSecondsPerPass, + IReadOnlyList independentSlots, + double requiredSpeedup = DefaultIndependentSpeedupMargin) + { + if (q8ModelSizeBytes == 0 || + maxOffloadNgl <= 0 || + sharedSecondsPerPass <= 0 || + independentSlots.Count < 2 || + requiredSpeedup < 1d) + { + return 0; + } + + var models = independentSlots + .Select(BuildPassTimeModel) + .ToArray(); + + if (models.Any(x => x == null)) + return 0; + + double requiredJobsPerSecond = (1d / sharedSecondsPerPass) * requiredSpeedup; + const int scanSteps = 1000; + + // Search from largest to smallest so the returned boundary is the first model size + // where independent workers have a meaningful, not merely noise-level, advantage. + for (int step = 0; step <= scanSteps; step++) + { + double sizeRatio = 1d - (0.8d * step / scanSteps); + ulong modelSize = (ulong)Math.Max(1d, Math.Floor(q8ModelSizeBytes * sizeRatio)); + double aggregateJobsPerSecond = 0d; + + for (int i = 0; i < independentSlots.Count; i++) + { + var slot = independentSlots[i]; + int ngl = ResolveNglForModel( + q8ModelSizeBytes, + slot.Q8StableNgl, + maxOffloadNgl, + modelSize); + + double seconds = models[i]!.Value.EstimateSeconds(ngl); + if (seconds <= 0 || double.IsNaN(seconds) || double.IsInfinity(seconds)) + { + aggregateJobsPerSecond = 0d; + break; + } + + aggregateJobsPerSecond += 1d / seconds; + } + + if (aggregateJobsPerSecond >= requiredJobsPerSecond) + return modelSize; + } + + return 0; + } + + private static PassTimeModel? BuildPassTimeModel(BenchmarkSlot slot) + { + var successful = slot.ProbeSamples + .Where(x => x.Success && x.Ngl > 0 && x.SecondsPerPass > 0) + .GroupBy(x => x.Ngl) + .Select(x => new + { + Ngl = x.Key, + Seconds = x.Average(y => y.SecondsPerPass) + }) + .OrderBy(x => x.Ngl) + .ToArray(); + + if (successful.Length < 2) + return null; + + double meanX = successful.Average(x => (double)x.Ngl); + double meanY = successful.Average(x => x.Seconds); + double denominator = successful.Sum(x => Math.Pow(x.Ngl - meanX, 2)); + if (denominator <= double.Epsilon) + return null; + + double slope = successful.Sum(x => (x.Ngl - meanX) * (x.Seconds - meanY)) / denominator; + if (slope >= 0) + return null; + + double intercept = meanY - slope * meanX; + double minimumObserved = successful.Min(x => x.Seconds); + return new PassTimeModel(intercept, slope, Math.Max(0.05d, minimumObserved * 0.55d)); + } + + private readonly record struct PassTimeModel(double Intercept, double Slope, double MinimumSeconds) + { + public double EstimateSeconds(int ngl) => Math.Max(MinimumSeconds, Intercept + Slope * ngl); + } +} + +internal sealed class GpuResourceScheduler +{ + private readonly object _sync = new(); + private readonly HashSet _busyDevices = new(); + private readonly LinkedList _waiters = new(); + + public ValueTask AcquireAsync( + IReadOnlyList candidates, + CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(candidates); + if (candidates.Count == 0) + throw new ArgumentException("At least one benchmark slot candidate is required.", nameof(candidates)); + + lock (_sync) + { + if (_waiters.Count == 0 && TryReserveFirstAvailable(candidates, out var immediate)) + return ValueTask.FromResult(new GpuResourceLease(this, immediate!)); + + var waiter = new Waiter(candidates); + waiter.Node = _waiters.AddLast(waiter); + + if (ct.CanBeCanceled) + { + waiter.Cancellation = ct.Register( + static state => + { + var pair = ((GpuResourceScheduler Scheduler, Waiter Waiter))state!; + pair.Scheduler.Cancel(pair.Waiter); + }, + (this, waiter)); + } + + return new ValueTask(waiter.Completion.Task); + } + } + + private void Cancel(Waiter waiter) + { + lock (_sync) + { + if (waiter.Node?.List == null) + return; + + _waiters.Remove(waiter.Node); + waiter.Node = null; + waiter.Cancellation.Dispose(); + waiter.Completion.TrySetCanceled(); + } + } + + private bool TryReserveFirstAvailable( + IReadOnlyList candidates, + out BenchmarkSlot? selected) + { + selected = candidates.FirstOrDefault(slot => + ReservationKeys(slot).All(device => !_busyDevices.Contains(device))); + + if (selected == null) + return false; + + foreach (int device in ReservationKeys(selected)) + _busyDevices.Add(device); + + return true; + } + + private void Release(BenchmarkSlot slot) + { + List<(Waiter Waiter, BenchmarkSlot Slot)> ready = new(); + + lock (_sync) + { + foreach (int device in ReservationKeys(slot)) + _busyDevices.Remove(device); + + while (_waiters.First != null) + { + var waiter = _waiters.First.Value; + if (!TryReserveFirstAvailable(waiter.Candidates, out var selected)) + break; + + _waiters.RemoveFirst(); + waiter.Node = null; + waiter.Cancellation.Dispose(); + ready.Add((waiter, selected!)); + } + } + + foreach (var item in ready) + item.Waiter.Completion.TrySetResult(new GpuResourceLease(this, item.Slot)); + } + + private static IEnumerable ReservationKeys(BenchmarkSlot slot) => + slot.DeviceIndices.Length == 0 ? [-1] : slot.DeviceIndices; + + private sealed class Waiter + { + public IReadOnlyList Candidates { get; } + public TaskCompletionSource Completion { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + public LinkedListNode? Node { get; set; } + public CancellationTokenRegistration Cancellation { get; set; } + + public Waiter(IReadOnlyList candidates) + { + Candidates = candidates; + } + } + + internal sealed class GpuResourceLease : IAsyncDisposable + { + private GpuResourceScheduler? _owner; + public BenchmarkSlot Slot { get; } + + internal GpuResourceLease(GpuResourceScheduler owner, BenchmarkSlot slot) + { + _owner = owner; + Slot = slot; + } + + public ValueTask DisposeAsync() + { + Interlocked.Exchange(ref _owner, null)?.Release(Slot); + return ValueTask.CompletedTask; + } + } +} diff --git a/src/MagicQuant/Services/BenchmarkLogParser.cs b/src/MagicQuant/Services/BenchmarkLogParser.cs new file mode 100644 index 0000000..acd425a --- /dev/null +++ b/src/MagicQuant/Services/BenchmarkLogParser.cs @@ -0,0 +1,112 @@ +using System.Globalization; +using System.Text.RegularExpressions; +using MQ.DB.Models; + +namespace MagicQuant.Services; + +/// Parses llama.cpp logs without scheduling work or opening the benchmark database. +public static class BenchmarkLogParser +{ + public static LlamaBenchMetrics ParseLlamaBench(string logPath) + { + var metrics = new LlamaBenchMetrics { LogPath = Path.GetFileName(logPath) }; + if (!File.Exists(logPath)) + return metrics; + + var lines = File.ReadAllLines(logPath); + int headerIdx = -1; + for (int i = 0; i < lines.Length; i++) + { + if (lines[i].Contains("|") && lines[i].Contains("backend")) + { + headerIdx = i; + break; + } + } + + if (headerIdx == -1 || lines.Length <= headerIdx + 2) + return metrics; + + var headers = lines[headerIdx] + .Split('|', StringSplitOptions.RemoveEmptyEntries) + .Select(h => h.Trim()) + .ToList(); + + var dataRow = lines[headerIdx + 2] + .Split('|', StringSplitOptions.RemoveEmptyEntries) + .Select(d => d.Trim()) + .ToList(); + + if (headers.Count != dataRow.Count) + return metrics; + + var row = headers + .Zip(dataRow, (h, d) => new { Header = h, Data = d }) + .ToDictionary(x => x.Header, x => x.Data, StringComparer.OrdinalIgnoreCase); + + string tpsStr = row.ContainsKey("t/s") + ? row["t/s"] + : (row.ContainsKey("tps") ? row["tps"] : "0"); + + var match = Regex.Match(tpsStr, @"([0-9.]+)"); + if (match.Success && + double.TryParse(match.Groups[1].Value, NumberStyles.Any, CultureInfo.InvariantCulture, out double tps)) + { + metrics.Tps = tps; + metrics.Backend = row.ContainsKey("backend") ? row["backend"] : "unknown"; + metrics.Test = row.ContainsKey("test") ? row["test"] : "unknown"; + + if (row.ContainsKey("ngl") && + int.TryParse(row["ngl"], NumberStyles.Any, CultureInfo.InvariantCulture, out int ngl)) + { + metrics.Ngl = ngl; + } + } + + return metrics; + } + + public static PplMetrics ParsePerplexity(string logPath, bool allowMissingKld) + { + var metrics = new PplMetrics { LogPath = Path.GetFileName(logPath) }; + + if (!File.Exists(logPath)) + throw new FileNotFoundException($"Perplexity log file was not created: {logPath}"); + + string text = File.ReadAllText(logPath); + string cleanText = StripAnsi(text); + + var pplMatch = Regex.Match( + cleanText, + @"(?:Mean PPL\(Q\)|PPL)\s*[:=]\s*([-+]?\d*\.?\d+(?:[eE][-+]?\d+)?)\s*(?:±|\+/-)\s*([-+]?\d*\.?\d+(?:[eE][-+]?\d+)?)", + RegexOptions.IgnoreCase); + + if (!pplMatch.Success) + { + throw new InvalidOperationException( + $"Failed to parse PPL from log: {logPath}\n\nLast log content:\n{cleanText}"); + } + + metrics.Ppl = double.Parse(pplMatch.Groups[1].Value, CultureInfo.InvariantCulture); + metrics.PplError = double.Parse(pplMatch.Groups[2].Value, CultureInfo.InvariantCulture); + + var kldMatch = Regex.Match( + cleanText, + @"(?:Mean\s+KLD|Mean\s+KL|KL[-_\s]*divergence|KLD|kl[-_\s]*div)\s*[:=]\s*([-+]?\d*\.?\d+(?:[eE][-+]?\d+)?)", + RegexOptions.IgnoreCase); + + if (kldMatch.Success) + { + metrics.Kld = double.Parse(kldMatch.Groups[1].Value, CultureInfo.InvariantCulture); + } + else if (!allowMissingKld) + { + throw new InvalidOperationException( + $"KLD was expected but could not be parsed from log: {logPath}\n\nLast log content:\n{cleanText}"); + } + + return metrics; + } + + public static string StripAnsi(string text) => Regex.Replace(text, @"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", ""); +} diff --git a/src/MagicQuant/Services/BenchmarkService.cs b/src/MagicQuant/Services/BenchmarkService.cs new file mode 100644 index 0000000..29a1ad9 --- /dev/null +++ b/src/MagicQuant/Services/BenchmarkService.cs @@ -0,0 +1,2692 @@ +using MagicQuant.Runtime; +using System.Diagnostics; +using System.Globalization; +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.RegularExpressions; +using MagicQuant.Helpers; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; +using Microsoft.EntityFrameworkCore; +using Spectre.Console; + +namespace MagicQuant.Services; + +public class BenchmarkService +{ + private readonly LlamaBinaries _bins; + private readonly GgufMetadataReader _ggufMetadataReader; + public readonly PythonManager _pyManager; + + private static readonly string[] BaseDomains = { "general", "code", "math" }; + private static readonly string[] SampleDomains = { "general" }; + + private static readonly string[] OomMarkers = + { + "out of memory", + "cudamalloc failed", + "unable to allocate cuda", + "try reducing --n-gpu-layers", + "cannot fulfill margin", + "failed to fit params", + "cuda error" + }; + + private static readonly int[] LegacyNglFallbacks = { 35, 30, 24, 20, 16, 12, 8, 4 }; + + // Version 4 adds measured shared/independent GPU topology profiles, per-device Q8 + // anchors, exact model-layer ceilings, and the llama.cpp binary fingerprint. + private const int DynamicProbeSchemaVersion = 4; + private const int PplCharsPerTokenEstimate = 4; + internal const string GeneralPplDatasetId = "Salesforce/wikitext"; + internal const string MathPplDatasetId = "openai/gsm8k"; + + // ---------------------------------------------------------------- + // Static execution-plan state + // ---------------------------------------------------------------- + + private static readonly SemaphoreSlim PlanInitLock = new(1, 1); + private static readonly object SlotSync = new(); + + private static BenchmarkExecutionPlan? _currentPlan; + private static string _currentPlanQuantizationKey = "Q8_0"; + private static GpuResourceScheduler _resourceScheduler = new(); + + public int CurrentParallelSlotCount + { + get + { + lock (SlotSync) + { + if (_currentPlan == null) + return 1; + + return Math.Max( + 1, + Math.Max(_currentPlan.SharedProfile.Slots.Count, _currentPlan.IndependentProfile.Slots.Count)); + } + } + } + + // ---------------------------------------------------------------- + // Construction + // ---------------------------------------------------------------- + + public BenchmarkService(PythonManager pyManager) + { + _bins = new LlamaBinaries(Cache.LlamaRoot); + _bins.Validate(); + _pyManager = pyManager; + _ggufMetadataReader = new GgufMetadataReader(pyManager); + } + + // ---------------------------------------------------------------- + // Execution-plan discovery + // ---------------------------------------------------------------- + + public async Task EnsureExecutionPlanAsync( + string q8ModelPath, + int discoveryTokenTarget = 8192, + string quantizationKey = "Q8_0", + bool forceRediscovery = false, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + await EnsureDynamicExecutionPlanAsync( + q8ModelPath: q8ModelPath, + nativeModelPath: q8ModelPath, + q8QuantizationKey: quantizationKey, + nativeQuantizationKey: quantizationKey, + discoveryTokenTarget: discoveryTokenTarget, + forceRediscovery: forceRediscovery, + ct: ct); + } + + public async Task EnsureDynamicExecutionPlanAsync( + string q8ModelPath, + string nativeModelPath, + string q8QuantizationKey = "Q8_0", + string nativeQuantizationKey = "BF16", + int discoveryTokenTarget = 8192, + bool forceRediscovery = false, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (string.IsNullOrWhiteSpace(q8ModelPath)) + throw new ArgumentException("Q8 model path was null or empty.", nameof(q8ModelPath)); + if (string.IsNullOrWhiteSpace(nativeModelPath)) + throw new ArgumentException("Native model path was null or empty.", nameof(nativeModelPath)); + if (string.IsNullOrWhiteSpace(q8QuantizationKey)) + throw new ArgumentException("Quantization key was null or empty.", nameof(q8QuantizationKey)); + + string normalizedPath = Path.GetFullPath(q8ModelPath); + string normalizedNativePath = Path.GetFullPath(nativeModelPath); + string normalizedQuantizationKey = q8QuantizationKey.Trim().ToUpperInvariant(); + string normalizedNativeQuantizationKey = nativeQuantizationKey.Trim().ToUpperInvariant(); + + if (!forceRediscovery && + _currentPlan != null && + string.Equals(_currentPlanQuantizationKey, normalizedQuantizationKey, StringComparison.OrdinalIgnoreCase) && + string.Equals(_currentPlan.PlanModelPath, normalizedPath, StringComparison.OrdinalIgnoreCase)) + { + return; + } + + await PlanInitLock.WaitAsync(ct); + try + { + if (!forceRediscovery && + _currentPlan != null && + string.Equals(_currentPlanQuantizationKey, normalizedQuantizationKey, StringComparison.OrdinalIgnoreCase) && + string.Equals(_currentPlan.PlanModelPath, normalizedPath, StringComparison.OrdinalIgnoreCase)) + { + return; + } + + var cacheKey = BuildExecutionPlanCacheKey(normalizedPath, discoveryTokenTarget, normalizedQuantizationKey); + + BenchmarkExecutionPlan? plan = null; + if (!forceRediscovery) + { + plan = await TryLoadCachedExecutionPlanAsync( + key: cacheKey, + nativeModelPath: normalizedNativePath, + nativeQuantizationKey: normalizedNativeQuantizationKey, + ct: ct); + if (plan != null) + AnsiConsole.MarkupLine("[green]Loaded benchmark execution plan from SQLite cache.[/]"); + } + + if (plan == null) + { + plan = await BuildDynamicExecutionPlanAsync( + q8ModelPath: normalizedPath, + nativeModelPath: normalizedNativePath, + q8QuantizationKey: normalizedQuantizationKey, + nativeQuantizationKey: normalizedNativeQuantizationKey, + discoveryTokenTarget: discoveryTokenTarget, + ct: ct); + await UpsertCachedExecutionPlanAsync(cacheKey, plan, ct); + } + + lock (SlotSync) + { + _currentPlan = plan; + _currentPlanQuantizationKey = normalizedQuantizationKey; + _resourceScheduler = new GpuResourceScheduler(); + } + + AnsiConsole.Write(new Rule("[yellow]Benchmark Execution Plan[/]") { Justification = Justify.Left }); + AnsiConsole.MarkupLine($"[green]Static ngl:[/] [cyan]{plan.StaticNgl}[/]"); + AnsiConsole.MarkupLine($"[green]Q8 anchor:[/] [cyan]{(plan.Q8ModelSizeBytes / 1024d / 1024d / 1024d):F2} GB @ ngl={plan.Q8StableNgl}[/]"); + AnsiConsole.MarkupLine($"[green]Native anchor:[/] [cyan]{(plan.NativeModelSizeBytes / 1024d / 1024d / 1024d):F2} GB @ ngl={plan.NativeStableNgl}[/]"); + AnsiConsole.MarkupLine($"[green]Uses GPU:[/] [cyan]{plan.UsesGpu}[/]"); + AnsiConsole.MarkupLine($"[green]GPU group size:[/] [cyan]{plan.GroupSize}[/]"); + AnsiConsole.MarkupLine($"[green]Max parallel benchmark slots:[/] [cyan]{CurrentParallelSlotCount}[/]"); + if (plan.IndependentMaxModelSizeBytes > 0) + { + AnsiConsole.MarkupLine( + $"[green]Independent-worker crossover:[/] [cyan]{plan.IndependentMaxModelSizeBytes / 1024d / 1024d / 1024d:F2} GB[/]"); + } + AnsiConsole.MarkupLine($"[green]Quantization key:[/] [cyan]{Markup.Escape(normalizedQuantizationKey)}[/]"); + if (Cache.GpuMemoryLimitsGb.Count == 0) + { + AnsiConsole.MarkupLine("[green]GPU memory limits:[/] [grey]none[/]"); + } + else + { + string limits = string.Join(", ", Cache.GpuMemoryLimitsGb.OrderBy(x => x.Key).Select(x => $"GPU {x.Key}={x.Value:0.###} GB")); + AnsiConsole.MarkupLine($"[green]GPU memory limits:[/] [cyan]{Markup.Escape(limits)}[/]"); + } + + foreach (var slot in plan.SharedProfile.Slots.Concat(plan.IndependentProfile.Slots)) + { + AnsiConsole.MarkupLine( + $" [grey]Slot {slot.SlotId}:[/] {Markup.Escape(slot.DisplayName)} @ Q8 ngl={slot.Q8StableNgl}"); + string tensorSplit = BuildTensorSplitArgs(slot, LlamaGpuTool.CommonCli); + if (!string.IsNullOrWhiteSpace(tensorSplit)) + { + AnsiConsole.MarkupLine($" [grey]tensor split:[/] {Markup.Escape(tensorSplit.Trim())}"); + } + } + } + finally + { + PlanInitLock.Release(); + } + } + + public async Task TryInitializeExecutionPlanFromCacheAsync( + int discoveryTokenTarget = 8192, + string quantizationKey = "Q8_0", + string? nativeModelPath = null, + string nativeQuantizationKey = "BF16", + string? preferredPlanModelPath = null, + CancellationToken ct = default) + => await TryInitializeDynamicExecutionPlanFromCacheAsync( + discoveryTokenTarget: discoveryTokenTarget, + q8QuantizationKey: quantizationKey, + nativeModelPath: nativeModelPath, + nativeQuantizationKey: nativeQuantizationKey, + preferredPlanModelPath: preferredPlanModelPath, + ct: ct); + + public async Task TryInitializeDynamicExecutionPlanFromCacheAsync( + int discoveryTokenTarget = 8192, + string q8QuantizationKey = "Q8_0", + string? nativeModelPath = null, + string nativeQuantizationKey = "BF16", + string? preferredPlanModelPath = null, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (string.IsNullOrWhiteSpace(q8QuantizationKey)) + throw new ArgumentException("Quantization key was null or empty.", nameof(q8QuantizationKey)); + + string normalizedQuantizationKey = q8QuantizationKey.Trim().ToUpperInvariant(); + string planModelPath = string.IsNullOrWhiteSpace(preferredPlanModelPath) + ? $"cached://{normalizedQuantizationKey}" + : Path.GetFullPath(preferredPlanModelPath); + + var cacheKey = BuildExecutionPlanCacheKey(planModelPath, discoveryTokenTarget, normalizedQuantizationKey); + AnsiConsole.MarkupLine( + $"[grey]Checking execution-plan cache:[/] quant={Markup.Escape(normalizedQuantizationKey)}, tokens={discoveryTokenTarget}"); + + var plan = await TryLoadCachedExecutionPlanAsync( + key: cacheKey, + nativeModelPath: nativeModelPath, + nativeQuantizationKey: nativeQuantizationKey, + ct: ct); + if (plan == null) + { + AnsiConsole.MarkupLine("[yellow]Execution-plan cache miss:[/] full Q8 probe will run."); + return false; + } + + lock (SlotSync) + { + _currentPlan = plan; + _currentPlanQuantizationKey = normalizedQuantizationKey; + _resourceScheduler = new GpuResourceScheduler(); + } + + AnsiConsole.MarkupLine("[green]Loaded benchmark execution plan from SQLite cache (no Q8 rebuild needed).[/]"); + return true; + } + + public async Task ClampStaticNglWithBaseModelAsync( + string baseModelPath, + int discoveryTokenTarget = 8192, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (string.IsNullOrWhiteSpace(baseModelPath)) + throw new ArgumentException("Base model path was null or empty.", nameof(baseModelPath)); + + if (_currentPlan == null) + throw new InvalidOperationException( + "Benchmark execution plan has not been initialized. Call EnsureExecutionPlanAsync() first."); + + if (!_currentPlan.UsesGpu) + return; + + await PlanInitLock.WaitAsync(ct); + try + { + if (_currentPlan == null || !_currentPlan.UsesGpu) + return; + + var slot = _currentPlan.Slots[0]; + + string probeRoot = Path.Combine(Cache.ModelMagicQuantDirectory!, "_benchmark_plan_probe_base"); + Directory.CreateDirectory(probeRoot); + + string probeCorpusDir = Path.Combine(probeRoot, "_ppl_corpora"); + Directory.CreateDirectory(probeCorpusDir); + + string corpusPath = Path.Combine(probeCorpusDir, "ppl_corpus_general.txt"); + await PreparePplCorpusAsync("general", corpusPath, discoveryTokenTarget); + + int startingNgl = _currentPlan.StaticNgl; + int? chosen = null; + + AnsiConsole.Write(new Rule("[yellow]Clamping Static ngl With Base Model[/]") + { Justification = Justify.Left }); + AnsiConsole.MarkupLine($"[grey]Base model:[/] {Markup.Escape(baseModelPath)}"); + AnsiConsole.MarkupLine($"[grey]Starting from Q8-discovered ngl:[/] [cyan]{startingNgl}[/]"); + + foreach (int ngl in BuildNglFallbackList(startingNgl).Where(n => n > 0)) + { + ct.ThrowIfCancellationRequested(); + + AnsiConsole.MarkupLine($"[grey]Base clamp probe:[/] [cyan]ngl={ngl}[/]"); + + bool benchOk = await ProbeLlamaBenchAtFixedNglAsync(baseModelPath, slot, ngl, probeRoot); + if (!benchOk) + { + AnsiConsole.MarkupLine($"[grey] llama-bench failed at ngl={ngl}[/]"); + continue; + } + + bool pplOk = await ProbePerplexityAtFixedNglAsync(baseModelPath, slot, ngl, corpusPath, probeRoot); + if (!pplOk) + { + AnsiConsole.MarkupLine($"[grey] perplexity failed at ngl={ngl}[/]"); + continue; + } + + chosen = ngl; + break; + } + + if (!chosen.HasValue) + { + AnsiConsole.MarkupLine( + "[yellow]Base model could not sustain the discovered GPU ngl. Falling back to a CPU benchmark plan.[/]"); + + var cpuPlan = BenchmarkExecutionPlan.CreateCpuPlan(_currentPlan.PlanModelPath) with + { + ProbeSchemaVersion = _currentPlan.ProbeSchemaVersion, + Q8ModelSizeBytes = _currentPlan.Q8ModelSizeBytes, + Q8StableNgl = 0, + NativeModelSizeBytes = _currentPlan.NativeModelSizeBytes, + NativeStableNgl = 0, + NativeQuantizationKey = _currentPlan.NativeQuantizationKey, + MaxCandidateNgl = _currentPlan.MaxCandidateNgl, + GpuMemoryLimitsJson = _currentPlan.GpuMemoryLimitsJson, + TensorSplitJson = "{}" + }; + + lock (SlotSync) + { + _currentPlan = cpuPlan; + _resourceScheduler = new GpuResourceScheduler(); + } + + var cacheKeyCpu = BuildExecutionPlanCacheKey( + _currentPlan.PlanModelPath, + discoveryTokenTarget, + _currentPlanQuantizationKey); + await UpsertCachedExecutionPlanAsync(cacheKeyCpu, _currentPlan, ct); + + return; + } + + if (chosen.Value != _currentPlan.StaticNgl) + { + var updated = _currentPlan with { StaticNgl = chosen.Value }; + + lock (SlotSync) + { + _currentPlan = updated; + _resourceScheduler = new GpuResourceScheduler(); + } + } + + var cacheKey = BuildExecutionPlanCacheKey( + _currentPlan.PlanModelPath, + discoveryTokenTarget, + _currentPlanQuantizationKey); + await UpsertCachedExecutionPlanAsync(cacheKey, _currentPlan, ct); + + AnsiConsole.MarkupLine($"[green]Base-model clamped static ngl:[/] [cyan]{chosen.Value}[/]"); + } + finally + { + PlanInitLock.Release(); + } + } + + private async Task BuildDynamicExecutionPlanAsync( + string q8ModelPath, + string nativeModelPath, + string q8QuantizationKey, + string nativeQuantizationKey, + int discoveryTokenTarget, + CancellationToken ct) + { + var plan = await BuildExecutionPlanAsync(q8ModelPath, discoveryTokenTarget, ct); + if (!plan.UsesGpu) + { + return plan with + { + ProbeSchemaVersion = DynamicProbeSchemaVersion, + Q8ModelSizeBytes = TryGetModelSize(q8ModelPath), + Q8StableNgl = 0, + NativeModelSizeBytes = TryGetModelSize(nativeModelPath), + NativeStableNgl = 0, + NativeQuantizationKey = nativeQuantizationKey, + MaxCandidateNgl = plan.MaxCandidateNgl, + GpuMemoryLimitsJson = SerializeGpuMemoryLimits(), + TensorSplitJson = SerializeTensorSplitMap(plan.Slots) + }; + } + + string nativeProbeRoot = Path.Combine(Cache.ModelMagicQuantDirectory!, "_benchmark_plan_probe_native"); + Directory.CreateDirectory(nativeProbeRoot); + string nativeCorpusDir = Path.Combine(nativeProbeRoot, "_ppl_corpora"); + Directory.CreateDirectory(nativeCorpusDir); + string nativeCorpusPath = Path.Combine(nativeCorpusDir, "ppl_corpus_general.txt"); + await PreparePplCorpusAsync("general", nativeCorpusPath, discoveryTokenTarget); + + BenchmarkSlot? nativeProbeSlot = await ProbeSlotCapacityAsync( + nativeModelPath, + plan.Slots[0] with { ProbeSamples = [] }, + plan.MaxCandidateNgl, + nativeCorpusPath, + nativeProbeRoot, + ct); + int? nativeStableNgl = nativeProbeSlot?.Q8StableNgl; + + if (!nativeStableNgl.HasValue || nativeStableNgl.Value <= 0) + { + AnsiConsole.MarkupLine( + "[yellow]Native anchor unavailable; keeping Q8 GPU plan and using conservative Q8-only dynamic NGL fallback.[/]"); + return plan with + { + ProbeSchemaVersion = DynamicProbeSchemaVersion, + Q8ModelSizeBytes = TryGetModelSize(q8ModelPath), + Q8StableNgl = plan.StaticNgl, + NativeModelSizeBytes = TryGetModelSize(nativeModelPath), + NativeStableNgl = 0, + NativeQuantizationKey = nativeQuantizationKey, + MaxCandidateNgl = plan.MaxCandidateNgl, + GpuMemoryLimitsJson = SerializeGpuMemoryLimits(), + TensorSplitJson = SerializeTensorSplitMap(plan.Slots) + }; + } + + return plan with + { + ProbeSchemaVersion = DynamicProbeSchemaVersion, + Q8ModelSizeBytes = TryGetModelSize(q8ModelPath), + Q8StableNgl = plan.StaticNgl, + NativeModelSizeBytes = TryGetModelSize(nativeModelPath), + NativeStableNgl = nativeStableNgl.Value, + NativeQuantizationKey = nativeQuantizationKey, + MaxCandidateNgl = plan.MaxCandidateNgl, + GpuMemoryLimitsJson = SerializeGpuMemoryLimits(), + TensorSplitJson = SerializeTensorSplitMap(plan.Slots) + }; + } + + private async Task BuildExecutionPlanAsync( + string q8ModelPath, + int discoveryTokenTarget, + CancellationToken ct) + { + int gpuCount = Cache.SysInfo?.GpuInfo? + .Count(x => x.GpuVendor != GpuVendor.Cpu && x.GpuVendor != GpuVendor.Unknown) ?? 0; + + if (gpuCount <= 0) + { + return BenchmarkExecutionPlan.CreateCpuPlan(q8ModelPath); + } + + string probeRoot = Path.Combine(Cache.ModelMagicQuantDirectory!, "_benchmark_plan_probe"); + Directory.CreateDirectory(probeRoot); + + var metadata = await _ggufMetadataReader.ReadAsync(q8ModelPath, probeRoot, ct); + int maxOffloadNgl = metadata.BlockCount is > 0 + ? checked(metadata.BlockCount.Value + 1) + : throw new InvalidOperationException( + "Q8 GGUF metadata did not expose a positive architecture block_count; " + + "an exact full-offload ceiling cannot be planned safely."); + + string probeCorpusDir = Path.Combine(probeRoot, "_ppl_corpora"); + Directory.CreateDirectory(probeCorpusDir); + string corpusPath = Path.Combine(probeCorpusDir, "ppl_corpus_general.txt"); + await PreparePplCorpusAsync("general", corpusPath, discoveryTokenTarget); + + var allGpuIndices = Enumerable.Range(0, gpuCount).ToArray(); + var sharedSeed = new BenchmarkSlot(0, "shared", allGpuIndices, 0, []); + _ = BuildTensorSplitArgs(sharedSeed, LlamaGpuTool.CommonCli); + + BenchmarkSlot? sharedSlot = await ProbeSlotCapacityAsync( + q8ModelPath, + sharedSeed, + maxOffloadNgl, + corpusPath, + probeRoot, + ct); + + if (sharedSlot == null || sharedSlot.Q8StableNgl <= 0) + { + AnsiConsole.MarkupLine( + "[yellow]Q8 discovery could not establish a stable GPU ngl. Falling back to a single CPU slot.[/]"); + return BenchmarkExecutionPlan.CreateCpuPlan(q8ModelPath); + } + + var independentSlots = new List(); + if (gpuCount > 1) + { + for (int gpuIndex = 0; gpuIndex < gpuCount; gpuIndex++) + { + var seed = new BenchmarkSlot(gpuIndex, "independent", [gpuIndex], 0, []); + BenchmarkSlot? discovered = await ProbeSlotCapacityAsync( + q8ModelPath, + seed, + maxOffloadNgl, + corpusPath, + probeRoot, + ct); + + if (discovered == null || discovered.Q8StableNgl <= 0) + { + AnsiConsole.MarkupLine( + $"[yellow]Independent slot GPU[{gpuIndex}] was not stable; independent topology disabled.[/]"); + independentSlots.Clear(); + break; + } + + independentSlots.Add(discovered); + } + } + + BenchmarkTopologyProfile sharedProfile = await MeasureTopologyProfileAsync( + "shared", q8ModelPath, [sharedSlot], corpusPath, probeRoot, ct); + + if (sharedProfile.Slots.Count == 0) + { + AnsiConsole.MarkupLine( + "[yellow]Shared GPU topology failed its concurrent throughput validation. Falling back to CPU.[/]"); + return BenchmarkExecutionPlan.CreateCpuPlan(q8ModelPath); + } + + BenchmarkTopologyProfile independentProfile = independentSlots.Count > 1 + ? await MeasureTopologyProfileAsync( + "independent", q8ModelPath, independentSlots, corpusPath, probeRoot, ct) + : new BenchmarkTopologyProfile("independent", [], 0, 0); + + ulong q8Size = TryGetModelSize(q8ModelPath); + ulong independentMaxModelSizeBytes = independentProfile.Slots.Count > 1 + ? BenchmarkGpuPlanner.EstimateIndependentCrossoverBytes( + q8Size, + maxOffloadNgl, + sharedProfile.MeasuredSecondsPerPass, + independentProfile.Slots) + : 0; + + if (independentMaxModelSizeBytes > 0) + { + AnsiConsole.MarkupLine( + $"[green]Measured topology crossover:[/] models up to " + + $"[cyan]{independentMaxModelSizeBytes / 1024d / 1024d / 1024d:F2} GB[/] use independent GPU workers; larger models use shared GPUs."); + } + + return new BenchmarkExecutionPlan( + PlanModelPath: q8ModelPath, + StaticNgl: sharedSlot.Q8StableNgl, + UsesGpu: true, + GroupSize: gpuCount, + Slots: sharedProfile.Slots, + MaxCandidateNgl: maxOffloadNgl, + IndependentSlots: independentProfile.Slots, + IndependentMaxModelSizeBytes: independentMaxModelSizeBytes, + SharedMeasuredJobsPerSecond: sharedProfile.MeasuredJobsPerSecond, + SharedMeasuredSecondsPerPass: sharedProfile.MeasuredSecondsPerPass, + IndependentMeasuredJobsPerSecond: independentProfile.MeasuredJobsPerSecond, + IndependentMeasuredSecondsPerPass: independentProfile.MeasuredSecondsPerPass); + } + + private async Task ProbeSlotCapacityAsync( + string modelPath, + BenchmarkSlot seed, + int maxOffloadNgl, + string corpusPath, + string probeRoot, + CancellationToken ct) + { + var samples = new List(); + + async Task Probe(int ngl) + { + var sample = await ProbePerplexitySampleAsync( + modelPath, seed, ngl, corpusPath, probeRoot, "capacity", ct); + samples.Add(sample); + return sample; + } + + AnsiConsole.MarkupLine( + $"[grey]Capacity probe:[/] {Markup.Escape(seed.DisplayName)} full-offload ngl={maxOffloadNgl}"); + + var full = await Probe(maxOffloadNgl); + int stableNgl; + + if (full.Success) + { + stableNgl = maxOffloadNgl; + int lowerNgl = Math.Max(1, (int)Math.Floor(maxOffloadNgl * 0.72d)); + if (lowerNgl < stableNgl) + await Probe(lowerNgl); + } + else + { + int low = 0; + int high = maxOffloadNgl - 1; + + while (low < high) + { + ct.ThrowIfCancellationRequested(); + int candidate = low + ((high - low + 1) / 2); + var sample = await Probe(candidate); + if (sample.Success) + low = candidate; + else + high = candidate - 1; + } + + stableNgl = low; + } + + if (stableNgl <= 0) + return null; + + int successfulDistinct = samples.Where(x => x.Success).Select(x => x.Ngl).Distinct().Count(); + if (successfulDistinct < 2) + { + int lowerNgl = Math.Max(1, (int)Math.Floor(stableNgl * 0.72d)); + if (lowerNgl < stableNgl) + await Probe(lowerNgl); + } + + AnsiConsole.MarkupLine( + $"[green]Stable capacity:[/] {Markup.Escape(seed.DisplayName)} ngl={stableNgl}/{maxOffloadNgl}"); + + return seed with + { + Q8StableNgl = stableNgl, + ProbeSamples = samples + }; + } + + private async Task MeasureTopologyProfileAsync( + string profileName, + string modelPath, + IReadOnlyList slots, + string corpusPath, + string probeRoot, + CancellationToken ct) + { + var stopwatch = Stopwatch.StartNew(); + var tasks = slots.Select(slot => ProbePerplexitySampleAsync( + modelPath, + slot, + slot.Q8StableNgl, + corpusPath, + probeRoot, + $"throughput_{profileName}", + ct)); + + GpuProbeSample[] samples = await Task.WhenAll(tasks); + stopwatch.Stop(); + + if (samples.Any(x => !x.Success)) + { + AnsiConsole.MarkupLine( + $"[yellow]Topology throughput validation failed for {Markup.Escape(profileName)}.[/]"); + return new BenchmarkTopologyProfile(profileName, [], 0, 0); + } + + double jobsPerSecond = slots.Count / Math.Max(0.001d, stopwatch.Elapsed.TotalSeconds); + double secondsPerPass = samples.Average(x => x.SecondsPerPass); + AnsiConsole.MarkupLine( + $"[green]Topology throughput:[/] {Markup.Escape(profileName)} = " + + $"[cyan]{jobsPerSecond:F4} jobs/s[/], {secondsPerPass:F2} s/pass"); + + return new BenchmarkTopologyProfile( + profileName, + slots, + jobsPerSecond, + secondsPerPass); + } + + private async Task ProbePerplexitySampleAsync( + string modelPath, + BenchmarkSlot slot, + int fixedNgl, + string corpusPath, + string probeRoot, + string phase, + CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + + string devices = slot.DeviceIndices.Length == 0 + ? "cpu" + : string.Join("-", slot.DeviceIndices); + string logFile = Path.Combine( + probeRoot, + $"probe_ppl_{phase}_{slot.ProfileName}_gpu{devices}_ngl{fixedNgl}.log"); + + var cmd = BenchmarkCommands.Perplexity(_bins.Ppl, modelPath, corpusPath, slot.UsesGpu, + fixedNgl, BuildTensorSplitArgs(slot, LlamaGpuTool.CommonCli)); + + var stopwatch = Stopwatch.StartNew(); + var result = await RunNativeCommandAsync(cmd, logFile, slot.BuildProcessEnv()); + stopwatch.Stop(); + + if (!result.Success) + return new GpuProbeSample(fixedNgl, false, 0, stopwatch.Elapsed.TotalSeconds); + + try + { + var parsed = BenchmarkLogParser.ParsePerplexity(logFile, allowMissingKld: true); + string clean = BenchmarkLogParser.StripAnsi(result.LogOutput); + var passMatch = Regex.Match( + clean, + @"([0-9]+(?:\.[0-9]+)?)\s+seconds per pass", + RegexOptions.IgnoreCase); + double secondsPerPass = passMatch.Success + ? double.Parse(passMatch.Groups[1].Value, CultureInfo.InvariantCulture) + : stopwatch.Elapsed.TotalSeconds; + + return new GpuProbeSample( + fixedNgl, + parsed.Ppl > 0, + secondsPerPass, + stopwatch.Elapsed.TotalSeconds); + } + catch + { + return new GpuProbeSample(fixedNgl, false, 0, stopwatch.Elapsed.TotalSeconds); + } + } + + private async Task TryLoadCachedExecutionPlanAsync( + ExecutionPlanCacheKey key, + string? nativeModelPath, + string nativeQuantizationKey, + CancellationToken ct) + { + await using var db = new MagicQuantContext(); + var aiModelHashId = await GetOrCreateAiModelHashIdAsync(db, ct); + var imatrixDefinitionId = await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, aiModelHashId, createIfMissing: false, ct); + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + + // Hardware execution-plan probes are intentionally NOT invalidated by tensor grouping + // profile changes. Regex/profile edits change benchmark/learned-truth semantics, but the + // Q8/native hardware capability plan is still valid for the same architecture family, + // exact model hash, imatrix identity, quantized artifact fingerprint, hardware, and token + // target. Prefer a current-profile row when present, then fall back to the newest + // compatible row from any prior TensorGroupProfile. + var compatibleRows = await db.ExecutionPlanProbeCaches + .AsNoTracking() + .Where(x => + x.ArchitectureFamilyId == architectureFamilyId && + x.AiModelHashId == aiModelHashId && + x.ImatrixDefinitionId == imatrixDefinitionId && + x.HardwareFingerprint == key.HardwareFingerprint && + x.QuantizedModelFingerprint == key.QuantizedModelFingerprint && + x.QuantizationKey == key.QuantizationKey && + x.DiscoveryTokenTarget == key.DiscoveryTokenTarget) + .OrderByDescending(x => x.TensorGroupProfileId == tensorGroupProfileId) + .ThenByDescending(x => x.UpdatedUtc) + .ThenByDescending(x => x.CreatedUtc) + .ToListAsync(ct); + + var row = compatibleRows.FirstOrDefault(); + + if (row == null) + return null; + + if (row.TensorGroupProfileId != tensorGroupProfileId) + { + AnsiConsole.MarkupLine( + $"[grey]Execution-plan cache reused from prior tensor profile {row.TensorGroupProfileId}; hardware probe cache is profile-compatible.[/]"); + } + + if (row.ProbeSchemaVersion < DynamicProbeSchemaVersion) + { + AnsiConsole.MarkupLine("[yellow]Execution-plan cache row uses old probe schema; re-probing.[/]"); + return null; + } + + if (row.GpuMemoryLimitsJson != SerializeGpuMemoryLimits()) + { + AnsiConsole.MarkupLine("[yellow]Execution-plan cache row GPU memory limits differ from current config; re-probing.[/]"); + return null; + } + + string normalizedNativeQuantizationKey = (nativeQuantizationKey ?? string.Empty).Trim().ToUpperInvariant(); + if (!string.Equals((row.NativeQuantizationKey ?? string.Empty).Trim().ToUpperInvariant(), normalizedNativeQuantizationKey, StringComparison.Ordinal)) + { + AnsiConsole.MarkupLine("[yellow]Execution-plan cache native quantization key changed; re-probing.[/]"); + return null; + } + + if (!string.IsNullOrWhiteSpace(nativeModelPath)) + { + ulong nativeSize = TryGetModelSize(Path.GetFullPath(nativeModelPath)); + if (nativeSize > 0 && row.NativeModelSizeBytes != nativeSize) + { + AnsiConsole.MarkupLine("[yellow]Execution-plan cache native model size changed; re-probing.[/]"); + return null; + } + } + + if (!row.UsesGpu) + { + return BenchmarkExecutionPlan.CreateCpuPlan(key.PlanModelPath) with + { + ProbeSchemaVersion = row.ProbeSchemaVersion, + Q8ModelSizeBytes = row.Q8ModelSizeBytes, + NativeModelSizeBytes = row.NativeModelSizeBytes, + NativeQuantizationKey = row.NativeQuantizationKey ?? string.Empty, + GpuMemoryLimitsJson = row.GpuMemoryLimitsJson ?? "{}" + }; + } + + if (!BenchmarkTopologyCacheCodec.TryDeserialize( + row.SlotsJson, + out int maxOffloadNgl, + out ulong independentMaxModelSizeBytes, + out var sharedProfile, + out var independentProfile) || + sharedProfile == null || + independentProfile == null) + { + AnsiConsole.MarkupLine("[yellow]Execution-plan cache topology JSON was unreadable. Re-probing.[/]"); + return null; + } + + foreach (var slot in sharedProfile.Slots.Concat(independentProfile.Slots)) + { + _ = BuildTensorSplitArgs(slot, LlamaGpuTool.CommonCli); + } + + // NativeStableNgl == 0 is valid and represents "native anchor unavailable" + // while still running a Q8-based GPU plan. + if (row.UsesGpu && + (row.Q8ModelSizeBytes == 0 || row.NativeModelSizeBytes == 0 || row.Q8StableNgl <= 0)) + { + AnsiConsole.MarkupLine("[yellow]Execution-plan cache row is missing dynamic anchor metadata; re-probing.[/]"); + return null; + } + + return new BenchmarkExecutionPlan( + PlanModelPath: key.PlanModelPath, + StaticNgl: row.StaticNgl, + UsesGpu: row.UsesGpu, + GroupSize: row.GroupSize, + Slots: sharedProfile.Slots, + ProbeSchemaVersion: row.ProbeSchemaVersion, + Q8ModelSizeBytes: row.Q8ModelSizeBytes, + Q8StableNgl: row.Q8StableNgl, + NativeModelSizeBytes: row.NativeModelSizeBytes, + NativeStableNgl: row.NativeStableNgl, + NativeQuantizationKey: row.NativeQuantizationKey ?? string.Empty, + MaxCandidateNgl: maxOffloadNgl, + GpuMemoryLimitsJson: row.GpuMemoryLimitsJson ?? "{}", + TensorSplitJson: row.TensorSplitJson ?? "{}", + IndependentSlots: independentProfile.Slots, + IndependentMaxModelSizeBytes: independentMaxModelSizeBytes, + SharedMeasuredJobsPerSecond: sharedProfile.MeasuredJobsPerSecond, + SharedMeasuredSecondsPerPass: sharedProfile.MeasuredSecondsPerPass, + IndependentMeasuredJobsPerSecond: independentProfile.MeasuredJobsPerSecond, + IndependentMeasuredSecondsPerPass: independentProfile.MeasuredSecondsPerPass); + } + + private async Task UpsertCachedExecutionPlanAsync( + ExecutionPlanCacheKey key, + BenchmarkExecutionPlan plan, + CancellationToken ct) + { + await using var db = new MagicQuantContext(); + var aiModelHashId = await GetOrCreateAiModelHashIdAsync(db, ct); + var imatrixDefinitionId = await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, aiModelHashId, createIfMissing: true, ct); + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + + var existing = await db.ExecutionPlanProbeCaches + .FirstOrDefaultAsync(x => + x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.AiModelHashId == aiModelHashId && + x.ImatrixDefinitionId == imatrixDefinitionId && + x.HardwareFingerprint == key.HardwareFingerprint && + x.QuantizedModelFingerprint == key.QuantizedModelFingerprint && + x.QuantizationKey == key.QuantizationKey && + x.DiscoveryTokenTarget == key.DiscoveryTokenTarget, ct); + + string slotsJson = plan.UsesGpu + ? BenchmarkTopologyCacheCodec.Serialize( + plan.MaxCandidateNgl, + plan.IndependentMaxModelSizeBytes, + plan.SharedProfile, + plan.IndependentProfile) + : "[]"; + var now = DateTime.UtcNow; + + if (existing == null) + { + existing = new ExecutionPlanProbeCache + { + ArchitectureFamilyId = architectureFamilyId, + TensorGroupProfileId = tensorGroupProfileId, + AiModelHashId = aiModelHashId, + ImatrixDefinitionId = imatrixDefinitionId, + HardwareFingerprint = key.HardwareFingerprint, + QuantizedModelFingerprint = key.QuantizedModelFingerprint, + QuantizationKey = key.QuantizationKey, + DiscoveryTokenTarget = key.DiscoveryTokenTarget, + CreatedUtc = now + }; + + db.ExecutionPlanProbeCaches.Add(existing); + } + + existing.StaticNgl = plan.StaticNgl; + existing.UsesGpu = plan.UsesGpu; + existing.GroupSize = plan.GroupSize; + existing.SlotsJson = slotsJson; + existing.ProbeSchemaVersion = plan.ProbeSchemaVersion; + existing.Q8ModelSizeBytes = plan.Q8ModelSizeBytes; + existing.Q8StableNgl = plan.Q8StableNgl; + existing.NativeModelSizeBytes = plan.NativeModelSizeBytes; + existing.NativeStableNgl = plan.NativeStableNgl; + existing.NativeQuantizationKey = plan.NativeQuantizationKey; + existing.MaxCandidateNgl = plan.MaxCandidateNgl; + existing.GpuMemoryLimitsJson = plan.GpuMemoryLimitsJson; + existing.TensorSplitJson = plan.TensorSplitJson; + existing.UpdatedUtc = now; + + await db.SaveChangesAsync(ct); + } + + private static ExecutionPlanCacheKey BuildExecutionPlanCacheKey( + string planModelPath, + int discoveryTokenTarget, + string quantizationKey) + { + string quantizedModelFingerprint = BuildQuantizedModelFingerprint(quantizationKey); + + var sys = Cache.SysInfo; + string hardwareFingerprint = sys == null + ? $"unknown-hardware|llama:{BuildLlamaRuntimeFingerprint()}" + : string.Join("|", new[] + { + $"threads:{sys.ThreadCount}", + $"ram:{sys.RamGb:F2}", + $"gpu:{string.Join(";", sys.GpuInfo.Select(g => $"{g.GpuVendor}:{g.GpuName}:{g.VramGb:F2}:{g.UniqueId ?? "none"}"))}", + $"llama:{BuildLlamaRuntimeFingerprint()}" + }); + + return new ExecutionPlanCacheKey( + hardwareFingerprint, + quantizedModelFingerprint, + quantizationKey, + discoveryTokenTarget, + planModelPath); + } + + private static string BuildLlamaRuntimeFingerprint() + { + try + { + var bins = new LlamaBinaries(Cache.LlamaRoot); + var ppl = new FileInfo(bins.Ppl); + if (!ppl.Exists) + return "missing"; + + return $"ppl:{ppl.Length}:{ppl.LastWriteTimeUtc.Ticks}"; + } + catch + { + return "unknown"; + } + } + + private static string BuildQuantizedModelFingerprint(string quantizationKey) + { + if (string.IsNullOrWhiteSpace(Cache.CurrentModelId)) + throw new InvalidOperationException("Cache.CurrentModelId is not set."); + + string imatrix = Cache.IsImatrixAvailable ? (Cache.ActiveImatrixIdentityHash ?? "imatrix-unknown") : "no-imatrix"; + string family = string.IsNullOrWhiteSpace(Cache.CurrentArchitectureFamilyName) ? Cache.CurrentModelId : Cache.CurrentArchitectureFamilyNormalizedName; + return $"family:{family}|model:{Cache.CurrentModelId}|imatrix:{imatrix}|quant:{quantizationKey}"; + } + + private static string SerializeGpuMemoryLimits() + { + var ordered = Cache.GpuMemoryLimitsGb + .OrderBy(x => x.Key) + .ToDictionary(x => x.Key, x => x.Value); + return JsonSerializer.Serialize(ordered); + } + + private static string SerializeTensorSplitMap(IReadOnlyList slots) + { + var map = slots + .Where(s => s.UsesGpu && s.DeviceIndices.Length > 1) + .ToDictionary( + s => s.DisplayName, + s => BuildTensorSplitArgs(s, LlamaGpuTool.CommonCli).Trim(), + StringComparer.Ordinal); + return JsonSerializer.Serialize(map); + } + + private static string BuildTensorSplitArgs(BenchmarkSlot slot, LlamaGpuTool tool) + { + try + { + return LlamaGpuArgumentBuilder.BuildTensorSplitArgs( + slot.DeviceIndices, + Cache.GpuMemoryLimitsGb, + tool); + } + catch (InvalidOperationException ex) + { + throw new InvalidOperationException( + $"Invalid tensor split for slot {slot.DisplayName}: {ex.Message}", ex); + } + } + + private static async Task GetOrCreateAiModelHashIdAsync(MagicQuantContext db, CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(Cache.CurrentModelId)) + throw new InvalidOperationException("Cache.CurrentModelId is not set."); + + var model = await db.AiModelHashes.FirstOrDefaultAsync(x => x.UniqueHash == Cache.CurrentModelId, ct); + if (model == null) + { + model = new AiModelHash { UniqueHash = Cache.CurrentModelId }; + db.AiModelHashes.Add(model); + await db.SaveChangesAsync(ct); + } + + return model.Id; + } + + private async Task ProbeLlamaBenchAtFixedNglAsync( + string modelPath, + BenchmarkSlot slot, + int fixedNgl, + string probeRoot) + { + string logFile = Path.Combine( + probeRoot, + $"probe_llamabench_slot{slot.SlotId}_g{slot.DeviceCount}_ngl{fixedNgl}.md"); + + var cmd = BenchmarkCommands.Bench(_bins.Bench, modelPath, slot.UsesGpu, + fixedNgl, BuildTensorSplitArgs(slot, LlamaGpuTool.LlamaBench)); + + var result = await RunNativeCommandAsync(cmd, logFile, slot.BuildProcessEnv()); + + if (!result.Success) + return false; + + try + { + var parsed = BenchmarkLogParser.ParseLlamaBench(logFile); + return parsed.Tps.HasValue && parsed.Tps.Value > 0; + } + catch + { + return false; + } + } + + private async Task ProbePerplexityAtFixedNglAsync( + string modelPath, + BenchmarkSlot slot, + int fixedNgl, + string corpusPath, + string probeRoot) + { + string logFile = Path.Combine( + probeRoot, + $"probe_ppl_general_slot{slot.SlotId}_g{slot.DeviceCount}_ngl{fixedNgl}.log"); + + var cmd = BenchmarkCommands.Perplexity(_bins.Ppl, modelPath, corpusPath, slot.UsesGpu, + fixedNgl, BuildTensorSplitArgs(slot, LlamaGpuTool.CommonCli)); + + var result = await RunNativeCommandAsync(cmd, logFile, slot.BuildProcessEnv()); + + if (!result.Success) + return false; + + try + { + var parsed = BenchmarkLogParser.ParsePerplexity(logFile, allowMissingKld: true); + return parsed.Ppl > 0; + } + catch + { + return false; + } + } + + private static async ValueTask AcquireBenchmarkSlotAsync( + ulong modelSizeBytes, + bool allowIndependentTopology, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (_currentPlan == null) + throw new InvalidOperationException( + "Benchmark execution plan has not been initialized. Call EnsureExecutionPlanAsync() first."); + + bool useIndependentTopology = BenchmarkGpuPlanner.ShouldUseIndependentTopology( + modelSizeBytes, + _currentPlan.IndependentMaxModelSizeBytes, + _currentPlan.IndependentProfile.Slots.Count, + allowIndependentTopology); + + BenchmarkTopologyProfile profile = useIndependentTopology + ? _currentPlan.IndependentProfile + : _currentPlan.SharedProfile; + + IReadOnlyList candidates = useIndependentTopology + ? BenchmarkGpuPlanner.RankIndependentSlotsForModel( + profile.Slots, + _currentPlan.Q8ModelSizeBytes, + _currentPlan.MaxCandidateNgl, + modelSizeBytes) + : profile.Slots; + + return await _resourceScheduler.AcquireAsync(candidates, ct); + } + + // ---------------------------------------------------------------- + // Public entry points + // ---------------------------------------------------------------- + + private static bool IsNativeBaseModel(HybridQuant quantConfig) + { + return quantConfig.BaseQuant.UniqueId == BaselineQuants.NativeSourceUniqueId; + } + + private static IReadOnlyCollection ResolveRequestedDomains( + HybridQuant quantConfig, + IReadOnlyCollection? domainsOverride) + { + if (domainsOverride != null && domainsOverride.Count > 0) + { + return domainsOverride + .Select(x => x.Trim().ToLowerInvariant()) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToArray(); + } + + return IsNativeBaseModel(quantConfig) ? BaseDomains : SampleDomains; + } + + private static bool RequiresKld(HybridQuant quantConfig) + { + return !IsNativeBaseModel(quantConfig); + } + + private static ulong TryGetModelSize(string modelPath) + { + return File.Exists(modelPath) ? (ulong)new FileInfo(modelPath).Length : 0UL; + } + + private int ResolveDynamicNglForModel(ulong modelSizeBytes, BenchmarkSlot slot) + { + if (_currentPlan == null || !_currentPlan.UsesGpu || !slot.UsesGpu) + return 0; + + if (modelSizeBytes == 0) + return Math.Max(0, _currentPlan.StaticNgl); + + if (_currentPlan.Q8ModelSizeBytes == 0 || _currentPlan.Q8StableNgl <= 0) + return Math.Max(0, _currentPlan.StaticNgl); + + int maxNgl = _currentPlan.MaxCandidateNgl > 0 ? _currentPlan.MaxCandidateNgl : _currentPlan.StaticNgl; + int slotQ8StableNgl = slot.Q8StableNgl > 0 + ? slot.Q8StableNgl + : _currentPlan.Q8StableNgl; + + if (modelSizeBytes <= _currentPlan.Q8ModelSizeBytes) + { + return BenchmarkGpuPlanner.ResolveNglForModel( + _currentPlan.Q8ModelSizeBytes, + slotQ8StableNgl, + maxNgl, + modelSizeBytes); + } + + double estimateRaw; + if (_currentPlan.NativeModelSizeBytes > _currentPlan.Q8ModelSizeBytes && _currentPlan.NativeStableNgl > 0 && modelSizeBytes < _currentPlan.NativeModelSizeBytes) + { + double t = (modelSizeBytes - _currentPlan.Q8ModelSizeBytes) / (double)(_currentPlan.NativeModelSizeBytes - _currentPlan.Q8ModelSizeBytes); + estimateRaw = Math.Floor(slotQ8StableNgl + ((_currentPlan.NativeStableNgl - slotQ8StableNgl) * t)); + } + else if (_currentPlan.NativeModelSizeBytes > 0 && _currentPlan.NativeStableNgl > 0) + { + estimateRaw = Math.Floor(_currentPlan.NativeStableNgl * (_currentPlan.NativeModelSizeBytes / (double)modelSizeBytes)); + estimateRaw = Math.Min(estimateRaw, _currentPlan.NativeStableNgl); + } + else + { + estimateRaw = Math.Floor(slotQ8StableNgl * (_currentPlan.Q8ModelSizeBytes / (double)modelSizeBytes)); + } + + int estimate = (int)Math.Clamp(estimateRaw, 0, maxNgl); + return Math.Max(0, Math.Min(estimate, maxNgl)); + } + + private static List BuildNglFallbackList(int startNgl) + { + var result = new List { Math.Max(0, startNgl) }; + result.AddRange(Enumerable.Range(Math.Max(1, startNgl - 4), Math.Min(4, Math.Max(0, startNgl - 1))) + .Reverse()); + result.AddRange(LegacyNglFallbacks.Where(x => x < startNgl).OrderByDescending(x => x)); + if (!result.Contains(0)) + result.Add(0); + + return result.Distinct().ToList(); + } + + private const double KldEpsilon = 1e-8; + + private static bool HasMeaningfulKld(double? kld) + { + return kld.HasValue && + !double.IsNaN(kld.Value) && + !double.IsInfinity(kld.Value) && + Math.Abs(kld.Value) > KldEpsilon; + } + + public async Task TryReuseExistingBenchmarksAsync( + HybridQuant quantConfig, + string modelPath, + string benchDir, + string? klLogitsDir, + IReadOnlyCollection? domainsOverride = null) + { + var requestedDomains = ResolveRequestedDomains(quantConfig, domainsOverride); + bool requireKld = RequiresKld(quantConfig); + + if (!TryReadExistingBenchmarkArtifacts( + benchDir: benchDir, + requestedDomains: requestedDomains, + requireKld: requireKld, + result: out var reused)) + { + return false; + } + + if (!reused.ModelSizeBytes.HasValue || reused.ModelSizeBytes.Value == 0) + { + var actualSize = TryGetModelSize(modelPath); + if (actualSize > 0) + reused.ModelSizeBytes = actualSize; + } + + if (!reused.ModelSizeBytes.HasValue || reused.ModelSizeBytes.Value == 0) + { + // Reuse cannot safely persist DB truth with unknown size. + // This is expected for transient scratch samples where modelPath may be intentionally empty. + return false; + } + + using var db = new MagicQuantContext(); + + var identity = await GetOrCreateBenchmarkIdentityAsync(db, quantConfig); + await SaveBenchmarkToDbAsync( + db: db, + model: identity.AiModelHash, + combo: identity.TensorCombo, + imatrixDefinitionId: identity.ImatrixDefinitionId, + res: reused, + modelPath: modelPath, + executedRunTimings: new List()); + + await WriteMetricsJsonAsync(benchDir, reused); + + return true; + } + + public async Task RunAllBenchmarksAsync( + HybridQuant quantConfig, + string modelPath, + string benchDir, + int tokenTarget = 32768, + int? startNgl = null, + string? klLogitsDir = null, + bool saveLogits = false, + IReadOnlyCollection? domainsOverride = null, + bool allowIndependentGpuTopology = true) + { + Directory.CreateDirectory(benchDir); + + var requestedDomains = ResolveRequestedDomains(quantConfig, domainsOverride); + bool requireKld = RequiresKld(quantConfig); + + if (Cache.SuppressBenchmarkPersistence) + { + return await RunAllBenchmarksTransientAsync( + quantConfig: quantConfig, + modelPath: modelPath, + benchDir: benchDir, + tokenTarget: tokenTarget, + klLogitsDir: klLogitsDir, + saveLogits: saveLogits, + requestedDomains: requestedDomains, + requireKld: requireKld, + allowIndependentGpuTopology: allowIndependentGpuTopology); + } + + using var db = new MagicQuantContext(); + + var identity = await GetOrCreateBenchmarkIdentityAsync(db, quantConfig); + var aiModelHash = identity.AiModelHash; + var tensorCombo = identity.TensorCombo; + + var existingBench = await db.AiBenchmarks + .Include(x => x.CategorBenchmarks) + .AsNoTracking() + .FirstOrDefaultAsync(b => b.ArchitectureFamilyId == TensorGroupProfileService.RequireCurrentArchitectureFamilyId() && b.TensorGroupProfileId == TensorGroupProfileService.RequireCurrentProfileId() && b.AiModelHashId == aiModelHash.Id && b.ImatrixDefinitionId == identity.ImatrixDefinitionId && b.TensorComboId == tensorCombo.Id); + + // 1. DB truth first + if (existingBench != null && HasRequiredCategories(existingBench, requestedDomains, requireKld)) + { + if (existingBench.SizeBytes == 0) + { + var repairedSize = TryGetModelSize(modelPath); + if (repairedSize > 0) + { + var trackedRepair = await db.AiBenchmarks + .FirstOrDefaultAsync(x => x.Id == existingBench.Id); + + if (trackedRepair != null) + { + trackedRepair.SizeBytes = repairedSize; + await db.SaveChangesAsync(); + existingBench.SizeBytes = repairedSize; + } + } + } + + if (TryReadExistingBenchmarkArtifacts(benchDir, requestedDomains, requireKld, out var diskResult)) + { + if (!diskResult.ModelSizeBytes.HasValue || diskResult.ModelSizeBytes.Value == 0) + diskResult.ModelSizeBytes = existingBench.SizeBytes > 0 + ? existingBench.SizeBytes + : TryGetModelSize(modelPath); + + return diskResult; + } + + return BuildResultFromDb(existingBench, requestedDomains); + } + + // 2. Disk truth second + if (TryReadExistingBenchmarkArtifacts(benchDir, requestedDomains, requireKld, out var reused)) + { + reused.ModelSizeBytes ??= TryGetModelSize(modelPath); + await SaveBenchmarkToDbAsync( + db: db, + model: aiModelHash, + combo: tensorCombo, + imatrixDefinitionId: identity.ImatrixDefinitionId, + res: reused, + modelPath: modelPath, + executedRunTimings: new List()); + + await WriteMetricsJsonAsync(benchDir, reused); + return reused; + } + + // 3. Real execution: fixed slot + fixed ngl + if (_currentPlan == null) + { + throw new InvalidOperationException( + "No benchmark execution plan has been discovered yet. " + + "You must call EnsureExecutionPlanAsync() with the pure Q8 model first."); + } + + var trackedBench = await db.AiBenchmarks + .Include(x => x.CategorBenchmarks) + .FirstOrDefaultAsync(x => x.ArchitectureFamilyId == TensorGroupProfileService.RequireCurrentArchitectureFamilyId() && x.TensorGroupProfileId == TensorGroupProfileService.RequireCurrentProfileId() && x.AiModelHashId == aiModelHash.Id && x.ImatrixDefinitionId == identity.ImatrixDefinitionId && x.TensorComboId == tensorCombo.Id); + + if (trackedBench == null) + { + trackedBench = new AiBenchmark + { + ArchitectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(), + TensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(), + AiModelHashId = aiModelHash.Id, + ImatrixDefinitionId = identity.ImatrixDefinitionId, + TensorComboId = tensorCombo.Id, + Ngl = 0, + SizeBytes = 0, + TokensPerSecond = 0 + }; + + db.AiBenchmarks.Add(trackedBench); + await db.SaveChangesAsync(); + } + + ulong modelSizeBytes = TryGetModelSize(modelPath); + await using var slotLease = await AcquireBenchmarkSlotAsync( + modelSizeBytes, + allowIndependentGpuTopology); + var slot = slotLease.Slot; + + int initialNgl = ResolveDynamicNglForModel(modelSizeBytes, slot); + var runtimeNgl = new RuntimeNglState + { + CurrentNgl = initialNgl, + LastSuccessfulNgl = initialNgl + }; + AnsiConsole.MarkupLine( + $"[grey]Dynamic NGL:[/] model={Markup.Escape(Path.GetFileName(modelPath))}, size={(modelSizeBytes / 1024d / 1024d / 1024d):F2} GB, q8={(_currentPlan.Q8ModelSizeBytes / 1024d / 1024d / 1024d):F2} GB/{_currentPlan.Q8StableNgl}, native={(_currentPlan.NativeModelSizeBytes / 1024d / 1024d / 1024d):F2} GB/{_currentPlan.NativeStableNgl}, slot={Markup.Escape(slot.DisplayName)}, chosen={initialNgl}"); + + var result = new BenchmarkResult + { + ModelSizeBytes = modelSizeBytes + }; + + var executedRunTimings = new List(); + + // Disabled for now. Too many variables that're annoying to track + result.LlamaBench = new LlamaBenchMetrics + { + LogPath = null, + Backend = slot.UsesGpu ? "disabled" : "cpu-disabled", + Ngl = runtimeNgl.CurrentNgl, + Test = "disabled", + Tps = 0 + }; + + var corporaRoot = Path.Combine(Path.GetDirectoryName(benchDir)!, "_ppl_corpora"); + Directory.CreateDirectory(corporaRoot); + + if (saveLogits && !string.IsNullOrEmpty(klLogitsDir)) + Directory.CreateDirectory(klLogitsDir); + + foreach (var domain in requestedDomains) + { + if (TryReadExistingPplLog( + benchDir: benchDir, + domain: domain, + allowMissingKld: !requireKld, + requirePositiveKld: requireKld, + metrics: out var existingPpl)) + { + result.Perplexity[domain] = existingPpl; + continue; + } + + string corpusPath = Path.Combine(corporaRoot, $"ppl_corpus_{domain}.txt"); + await PreparePplCorpusAsync(domain, corpusPath, tokenTarget); + + DateTime startedUtc = DateTime.UtcNow; + var sw = Stopwatch.StartNew(); + + try + { + AnsiConsole.MarkupLine( + $"[yellow]Running Perplexity ({Markup.Escape(domain)})[/] [grey]({Markup.Escape(slot.DisplayName)}, ngl={runtimeNgl.CurrentNgl})[/]"); + + var metrics = await RunPplBenchmarkWithNglFallbackAsync( + modelPath: modelPath, + benchDir: benchDir, + domain: domain, + corpusPath: corpusPath, + slot: slot, + runtimeNgl: runtimeNgl, + klLogitsDir: klLogitsDir, + saveLogits: saveLogits); + + if (requireKld && !HasMeaningfulKld(metrics.Kld)) + { + throw new InvalidOperationException( + $"Non-base benchmark produced invalid KLD for domain '{domain}'. " + + $"KLD must exist and be > 0. Parsed value: {(metrics.Kld.HasValue ? metrics.Kld.Value.ToString(CultureInfo.InvariantCulture) : "null")}"); + } + + sw.Stop(); + + result.Perplexity[domain] = metrics; + result.LlamaBench.Ngl = runtimeNgl.LastSuccessfulNgl; + + executedRunTimings.Add(new PendingBenchmarkRunTiming + { + Domain = domain, + Category = DomainToCategory(domain), + StartedUtc = startedUtc, + CompletedUtc = DateTime.UtcNow, + Succeeded = true, + Error = null + }); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + sw.Stop(); + + if (!Cache.SuppressBenchmarkPersistence) + { + await PersistFailedBenchmarkRunAsync( + db: db, + aiModelHashId: aiModelHash.Id, + tensorComboId: tensorCombo.Id, + aiBenchmarkId: trackedBench.Id, + imatrixDefinitionId: identity.ImatrixDefinitionId, + category: DomainToCategory(domain), + startedUtc: startedUtc, + completedUtc: DateTime.UtcNow, + error: ex.ToString()); + } + + throw; + } + } + + await WriteMetricsJsonAsync(benchDir, result); + + await SaveBenchmarkToDbAsync( + db: db, + model: aiModelHash, + combo: tensorCombo, + imatrixDefinitionId: identity.ImatrixDefinitionId, + res: result, + modelPath: modelPath, + executedRunTimings: executedRunTimings); + + return result; + } + + + private async Task RunAllBenchmarksTransientAsync( + HybridQuant quantConfig, + string modelPath, + string benchDir, + int tokenTarget, + string? klLogitsDir, + bool saveLogits, + IReadOnlyCollection requestedDomains, + bool requireKld, + bool allowIndependentGpuTopology) + { + if (TryReadExistingBenchmarkArtifacts(benchDir, requestedDomains, requireKld, out var reused)) + { + reused.ModelSizeBytes ??= TryGetModelSize(modelPath); + await WriteMetricsJsonAsync(benchDir, reused); + return reused; + } + + if (_currentPlan == null) + { + throw new InvalidOperationException( + "No benchmark execution plan has been discovered yet. " + + "You must call EnsureExecutionPlanAsync() with the pure Q8 model first."); + } + + ulong modelSizeBytes = TryGetModelSize(modelPath); + await using var slotLease = await AcquireBenchmarkSlotAsync( + modelSizeBytes, + allowIndependentGpuTopology); + var slot = slotLease.Slot; + + int initialNgl = ResolveDynamicNglForModel(modelSizeBytes, slot); + var runtimeNgl = new RuntimeNglState + { + CurrentNgl = initialNgl, + LastSuccessfulNgl = initialNgl + }; + AnsiConsole.MarkupLine( + $"[grey]Dynamic NGL:[/] model={Markup.Escape(Path.GetFileName(modelPath))}, size={(modelSizeBytes / 1024d / 1024d / 1024d):F2} GB, q8={(_currentPlan.Q8ModelSizeBytes / 1024d / 1024d / 1024d):F2} GB/{_currentPlan.Q8StableNgl}, native={(_currentPlan.NativeModelSizeBytes / 1024d / 1024d / 1024d):F2} GB/{_currentPlan.NativeStableNgl}, slot={Markup.Escape(slot.DisplayName)}, chosen={initialNgl}"); + + var result = new BenchmarkResult + { + ModelSizeBytes = modelSizeBytes, + LlamaBench = new LlamaBenchMetrics + { + LogPath = null, + Backend = slot.UsesGpu ? "disabled" : "cpu-disabled", + Ngl = runtimeNgl.CurrentNgl, + Test = "disabled", + Tps = 0 + } + }; + + var corporaRoot = Path.Combine(Path.GetDirectoryName(benchDir)!, "_ppl_corpora"); + Directory.CreateDirectory(corporaRoot); + + if (saveLogits && !string.IsNullOrEmpty(klLogitsDir)) + Directory.CreateDirectory(klLogitsDir); + + foreach (var domain in requestedDomains) + { + if (TryReadExistingPplLog( + benchDir: benchDir, + domain: domain, + allowMissingKld: !requireKld, + requirePositiveKld: requireKld, + metrics: out var existingPpl)) + { + result.Perplexity[domain] = existingPpl; + continue; + } + + string corpusPath = Path.Combine(corporaRoot, $"ppl_corpus_{domain}.txt"); + await PreparePplCorpusAsync(domain, corpusPath, tokenTarget); + + AnsiConsole.MarkupLine( + $"[yellow]Running transient Perplexity ({Markup.Escape(domain)})[/] [grey]({Markup.Escape(slot.DisplayName)}, ngl={runtimeNgl.CurrentNgl})[/]"); + + var metrics = await RunPplBenchmarkWithNglFallbackAsync( + modelPath: modelPath, + benchDir: benchDir, + domain: domain, + corpusPath: corpusPath, + slot: slot, + runtimeNgl: runtimeNgl, + klLogitsDir: klLogitsDir, + saveLogits: saveLogits); + + if (requireKld && !HasMeaningfulKld(metrics.Kld)) + { + throw new InvalidOperationException( + $"Non-base transient benchmark produced invalid KLD for domain '{domain}'. " + + $"KLD must exist and be > 0. Parsed value: {(metrics.Kld.HasValue ? metrics.Kld.Value.ToString(CultureInfo.InvariantCulture) : "null")}"); + } + + result.Perplexity[domain] = metrics; + result.LlamaBench.Ngl = runtimeNgl.LastSuccessfulNgl; + } + + await WriteMetricsJsonAsync(benchDir, result); + return result; + } + + // ---------------------------------------------------------------- + // Database helpers + // ---------------------------------------------------------------- + + private async Task<(AiModelHash AiModelHash, TensorCombo TensorCombo, int? ImatrixDefinitionId)> GetOrCreateBenchmarkIdentityAsync( + MagicQuantContext db, + HybridQuant quantConfig, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + var currentHashStr = Cache.CurrentModelId; + if (string.IsNullOrWhiteSpace(currentHashStr)) + throw new InvalidOperationException("Cache.CurrentModelId is not set."); + + var aiModelHash = await db.AiModelHashes + .FirstOrDefaultAsync(x => x.UniqueHash == currentHashStr, ct); + + if (aiModelHash == null) + { + aiModelHash = new AiModelHash { UniqueHash = currentHashStr }; + db.AiModelHashes.Add(aiModelHash); + await db.SaveChangesAsync(ct); + } + + if (Cache.CurrentArchitectureFamilyId != null) + { + uint scopedId = await ArchitectureFamilyService.ResolveScopedAiModelHashIdAsync(db, ct); + aiModelHash = await db.AiModelHashes.FirstAsync(x => x.Id == scopedId, ct); + } + + var tensorCombo = await GetOrCreateTensorComboAsync(db, quantConfig, ct); + + var imatrixDefinitionId = await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, aiModelHash.Id, createIfMissing: true, ct); + return (aiModelHash, tensorCombo, imatrixDefinitionId); + } + + private async Task GetOrCreateTensorComboAsync( + MagicQuantContext db, + HybridQuant quant, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + var c = (TensorConfig)quant; + + var existing = await db.TensorCombos.FirstOrDefaultAsync(x => + x.BaseQuant == c.BaseQuant && + x.Embeddings == c.Embeddings && + x.LmHead == c.LmHead && + x.AttnQ == c.AttnQ && + x.AttnKV == c.AttnKV && + x.AttnOutput == c.AttnOutput && + x.FfnUpGate == c.FfnUpGate && + x.FfnDown == c.FfnDown && + x.MoeExperts == c.MoeExperts && + x.MoeRouter == c.MoeRouter, ct); + + if (existing != null) + return existing; + + var newCombo = new TensorCombo(c); + db.TensorCombos.Add(newCombo); + await db.SaveChangesAsync(ct); + return newCombo; + } + + private async Task SaveBenchmarkToDbAsync( + MagicQuantContext db, + AiModelHash model, + TensorCombo combo, + int? imatrixDefinitionId, + BenchmarkResult res, + string modelPath, + IReadOnlyCollection executedRunTimings) + { + using var transaction = await db.Database.BeginTransactionAsync(); + + try + { + bool isBaseModel = + combo.BaseQuant == BaselineQuants.NativeSourceUniqueId && + combo.Embeddings == 0 && + combo.LmHead == 0 && + combo.AttnQ == 0 && + combo.AttnKV == 0 && + combo.AttnOutput == 0 && + combo.FfnUpGate == 0 && + combo.FfnDown == 0 && + combo.MoeExperts == 0 && + combo.MoeRouter == 0; + + ulong sizeBytes = + res.ModelSizeBytes.GetValueOrDefault() > 0 + ? res.ModelSizeBytes!.Value + : (File.Exists(modelPath) ? (ulong)new FileInfo(modelPath).Length : 0UL); + + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + + var bench = await db.AiBenchmarks + .Include(x => x.CategorBenchmarks) + .FirstOrDefaultAsync(x => + x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.AiModelHashId == model.Id && + x.ImatrixDefinitionId == imatrixDefinitionId && + x.TensorComboId == combo.Id); + + if (bench == null) + { + bench = new AiBenchmark + { + ArchitectureFamilyId = architectureFamilyId, + TensorGroupProfileId = tensorGroupProfileId, + AiModelHashId = model.Id, + ImatrixDefinitionId = imatrixDefinitionId, + TensorComboId = combo.Id + }; + + db.AiBenchmarks.Add(bench); + await db.SaveChangesAsync(); + } + + bench.TokensPerSecond = res.LlamaBench?.Tps ?? 0; + bench.Ngl = (byte)(res.LlamaBench?.Ngl ?? 0); + + if (sizeBytes > 0) + { + bench.SizeBytes = sizeBytes; + } + else if (bench.SizeBytes == 0) + { + bench.SizeBytes = 0; + } + + await db.SaveChangesAsync(); + + if (bench.CategorBenchmarks != null && bench.CategorBenchmarks.Count > 0) + { + db.Set().RemoveRange(bench.CategorBenchmarks); + await db.SaveChangesAsync(); + } + + var categories = new List(); + + foreach (var kvp in res.Perplexity) + { + string domain = kvp.Key.ToLowerInvariant(); + var m = kvp.Value; + + byte category = DomainToCategory(domain); + + double kld; + if (isBaseModel) + { + kld = 0d; + } + else + { + if (!HasMeaningfulKld(m.Kld)) + { + throw new InvalidOperationException( + $"Refusing to save non-base benchmark with invalid KLD. Domain='{domain}', KLD='{m.Kld?.ToString(CultureInfo.InvariantCulture) ?? "null"}'"); + } + + kld = m.Kld!.Value; + } + + categories.Add(new CategoryBenchmark + { + AiBenchmarkId = bench.Id, + Category = category, + Ppl = m.Ppl, + PplError = m.PplError, + Kld = kld + }); + } + + if (categories.Count > 0) + { + db.Set().AddRange(categories); + await db.SaveChangesAsync(); + } + + if (executedRunTimings.Count > 0) + { + var categoryIdLookup = await db.Set() + .Where(x => x.AiBenchmarkId == bench.Id) + .ToDictionaryAsync(x => x.Category, x => x.Id); + + foreach (var timing in executedRunTimings) + { + Guid? categoryBenchmarkId = null; + if (categoryIdLookup.TryGetValue(timing.Category, out var foundCategoryId)) + categoryBenchmarkId = foundCategoryId; + + db.BenchmarkRuns.Add(new BenchmarkRun + { + Id = Guid.NewGuid(), + ArchitectureFamilyId = architectureFamilyId, + TensorGroupProfileId = tensorGroupProfileId, + AiModelHashId = model.Id, + ImatrixDefinitionId = imatrixDefinitionId, + TensorComboId = combo.Id, + AiBenchmarkId = bench.Id, + CategoryBenchmarkId = categoryBenchmarkId, + Category = timing.Category, + StartedUtc = timing.StartedUtc, + CompletedUtc = timing.CompletedUtc, + DurationMs = Math.Max(0L, (long)(timing.CompletedUtc - timing.StartedUtc).TotalMilliseconds), + Succeeded = timing.Succeeded, + Error = timing.Error + }); + } + + await db.SaveChangesAsync(); + } + + await ReplaceBenchmarkLearnedSourcesAsync(db, bench, combo, architectureFamilyId, tensorGroupProfileId); + + await transaction.CommitAsync(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + await transaction.RollbackAsync(); + + var inner = ex.InnerException?.Message; + if (!string.IsNullOrWhiteSpace(inner)) + { + AnsiConsole.MarkupLine($"[red]Failed to save benchmarks to DB:[/] {Markup.Escape(ex.Message)}"); + AnsiConsole.MarkupLine($"[red]Inner Exception:[/] {Markup.Escape(inner)}"); + } + else + { + AnsiConsole.MarkupLine($"[red]Failed to save benchmarks to DB:[/] {Markup.Escape(ex.Message)}"); + } + + throw; + } + } + + + private static async Task ReplaceBenchmarkLearnedSourcesAsync( + MagicQuantContext db, + AiBenchmark bench, + TensorCombo combo, + int architectureFamilyId, + int tensorGroupProfileId) + { + await db.AiBenchmarkLearnedSources + .Where(x => x.AiBenchmarkId == bench.Id) + .ExecuteDeleteAsync(); + + var groupSlots = new (byte GroupId, byte StoredValue)[] + { + (TReg.Embeddings.UniqueId, combo.Embeddings), + (TReg.LmHead.UniqueId, combo.LmHead), + (TReg.AttnQ.UniqueId, combo.AttnQ), + (TReg.AttnKV.UniqueId, combo.AttnKV), + (TReg.AttnOutput.UniqueId, combo.AttnOutput), + (TReg.FfnUpGate.UniqueId, combo.FfnUpGate), + (TReg.FfnDown.UniqueId, combo.FfnDown), + (TReg.MoeExperts.UniqueId, combo.MoeExperts), + (TReg.MoeRouter.UniqueId, combo.MoeRouter) + }; + + foreach (var (groupId, storedValue) in groupSlots) + { + if (storedValue == 0) + continue; + + var runtimeBaselineId = BaselineQuants.DecodeTensorConfigGroupSlotToBaselineId(storedValue); + if (runtimeBaselineId == BaselineQuants.BF16_Hybrid.UniqueId || + runtimeBaselineId == BaselineQuants.F16_Hybrid.UniqueId || + runtimeBaselineId == BaselineQuants.NativeSourceUniqueId) + continue; + + var definition = await db.BaselineQuantDefinitions + .AsNoTracking() + .Where(x => (x.ArchitectureFamilyId == architectureFamilyId || x.ArchitectureFamilyId == null) && + x.RuntimeBaselineId == runtimeBaselineId) + .OrderByDescending(x => x.ArchitectureFamilyId.HasValue) + .FirstOrDefaultAsync(); + + if (definition == null) + continue; + + db.AiBenchmarkLearnedSources.Add(new AiBenchmarkLearnedSource + { + Id = Guid.NewGuid(), + AiBenchmarkId = bench.Id, + ArchitectureFamilyId = architectureFamilyId, + TensorGroupProfileId = tensorGroupProfileId, + TensorComboId = combo.Id, + TensorGroupId = groupId, + BaselineQuantDefinitionId = definition.Id, + SourceLearningBenchmarkId = null, + BaselineCanonicalKey = definition.CanonicalKey, + CreatedUtc = DateTime.UtcNow + }); + } + + await db.SaveChangesAsync(); + } + + private static byte DomainToCategory(string domain) + { + return domain.Trim().ToLowerInvariant() switch + { + "general" => (byte)BenchmarkCategory.General, + "math" => (byte)BenchmarkCategory.Math, + "code" => (byte)BenchmarkCategory.Code, + _ => throw new InvalidOperationException($"Unknown benchmark domain '{domain}'.") + }; + } + + private async Task PersistFailedBenchmarkRunAsync( + MagicQuantContext db, + uint aiModelHashId, + Guid tensorComboId, + Guid aiBenchmarkId, + int? imatrixDefinitionId, + byte category, + DateTime startedUtc, + DateTime completedUtc, + string error) + { + db.BenchmarkRuns.Add(new BenchmarkRun + { + Id = Guid.NewGuid(), + ArchitectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(), + TensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(), + AiModelHashId = aiModelHashId, + ImatrixDefinitionId = imatrixDefinitionId, + TensorComboId = tensorComboId, + AiBenchmarkId = aiBenchmarkId, + CategoryBenchmarkId = null, + Category = category, + StartedUtc = startedUtc, + CompletedUtc = completedUtc, + DurationMs = Math.Max(0L, (long)(completedUtc - startedUtc).TotalMilliseconds), + Succeeded = false, + Error = error + }); + + await db.SaveChangesAsync(); + } + + private sealed class PendingBenchmarkRunTiming + { + public string Domain { get; set; } = string.Empty; + public byte Category { get; set; } + public DateTime StartedUtc { get; set; } + public DateTime CompletedUtc { get; set; } + public bool Succeeded { get; set; } + public string? Error { get; set; } + } + + // ---------------------------------------------------------------- + // Artifact reuse helpers + // ---------------------------------------------------------------- + + private async Task WriteMetricsJsonAsync(string benchDir, BenchmarkResult result) + { + Directory.CreateDirectory(benchDir); + + string jsonPath = Path.Combine(benchDir, "bench_metrics.json"); + await File.WriteAllTextAsync( + jsonPath, + JsonSerializer.Serialize(result, new JsonSerializerOptions { WriteIndented = true })); + } + + private bool TryReadExistingBenchmarkArtifacts( + string benchDir, + IReadOnlyCollection requestedDomains, + bool requireKld, + out BenchmarkResult result) + { + result = new BenchmarkResult(); + + string jsonPath = Path.Combine(benchDir, "bench_metrics.json"); + if (File.Exists(jsonPath)) + { + try + { + var parsed = JsonSerializer.Deserialize(File.ReadAllText(jsonPath)); + if (parsed != null && IsReusableBenchmarkResult(parsed, requestedDomains, requireKld)) + { + result = parsed; + return true; + } + } + catch + { + // fall through + } + } + + var rebuilt = new BenchmarkResult + { + LlamaBench = new LlamaBenchMetrics + { + LogPath = null, + Backend = "disabled", + Ngl = 0, + Test = "disabled", + Tps = 0 + } + }; + + foreach (var domain in requestedDomains) + { + if (!TryReadExistingPplLog( + benchDir: benchDir, + domain: domain, + allowMissingKld: !requireKld, + requirePositiveKld: requireKld, + metrics: out var ppl)) + { + return false; + } + + rebuilt.Perplexity[domain] = ppl; + } + + result = rebuilt; + return true; + } + + private bool IsReusableBenchmarkResult( + BenchmarkResult result, + IReadOnlyCollection requestedDomains, + bool requireKld) + { + foreach (var domain in requestedDomains) + { + if (!result.Perplexity.TryGetValue(domain, out var ppl)) + return false; + + if (ppl.Ppl <= 0 || ppl.PplError < 0) + return false; + + if (requireKld && !HasMeaningfulKld(ppl.Kld)) + return false; + } + + return true; + } + + private bool TryReadExistingLlamaBenchLog(string logPath, out LlamaBenchMetrics metrics) + { + metrics = null!; + + if (!File.Exists(logPath) || new FileInfo(logPath).Length == 0) + return false; + + try + { + var parsed = BenchmarkLogParser.ParseLlamaBench(logPath); + if (parsed.Tps.HasValue && parsed.Tps.Value > 0) + { + metrics = parsed; + return true; + } + } + catch + { + // ignore + } + + return false; + } + + private bool TryReadExistingPplLog( + string benchDir, + string domain, + bool allowMissingKld, + bool requirePositiveKld, + out PplMetrics metrics) + { + metrics = null!; + + string logPath = Path.Combine(benchDir, $"perplexity_{domain}.log"); + if (!File.Exists(logPath) || new FileInfo(logPath).Length == 0) + return false; + + try + { + var parsed = BenchmarkLogParser.ParsePerplexity(logPath, allowMissingKld); + + if (parsed.Ppl <= 0) + return false; + + if (requirePositiveKld && !HasMeaningfulKld(parsed.Kld)) + return false; + + metrics = parsed; + return true; + } + catch + { + return false; + } + } + + private static bool HasRequiredCategories( + AiBenchmark bench, + IReadOnlyCollection requestedDomains, + bool requireKld) + { + if (bench.CategorBenchmarks == null || bench.CategorBenchmarks.Count == 0) + return false; + + foreach (var domain in requestedDomains) + { + byte category = domain switch + { + "general" => (byte)BenchmarkCategory.General, + "math" => (byte)BenchmarkCategory.Math, + "code" => (byte)BenchmarkCategory.Code, + _ => throw new InvalidOperationException($"Unknown benchmark domain '{domain}'.") + }; + + var existing = bench.CategorBenchmarks.FirstOrDefault(x => x.Category == category); + if (existing == null) + return false; + + if (existing.Ppl <= 0) + return false; + + if (requireKld && existing.Kld <= 0) + return false; + } + + return bench.SizeBytes > 0; + } + + private BenchmarkResult BuildResultFromDb( + AiBenchmark bench, + IReadOnlyCollection requestedDomains) + { + var result = new BenchmarkResult + { + ModelSizeBytes = bench.SizeBytes, + LlamaBench = new LlamaBenchMetrics + { + Ngl = bench.Ngl, + Tps = bench.TokensPerSecond + } + }; + + foreach (var domain in requestedDomains) + { + byte category = domain switch + { + "general" => (byte)BenchmarkCategory.General, + "math" => (byte)BenchmarkCategory.Math, + "code" => (byte)BenchmarkCategory.Code, + _ => throw new InvalidOperationException($"Unknown benchmark domain '{domain}'.") + }; + + var existing = bench.CategorBenchmarks.First(x => x.Category == category); + + result.Perplexity[domain] = new PplMetrics + { + Ppl = existing.Ppl, + PplError = existing.PplError, + Kld = existing.Kld + }; + } + + return result; + } + + // ---------------------------------------------------------------- + // Real benchmark execution (fixed slot + fixed ngl) + // ---------------------------------------------------------------- + + private async Task RunLlamaBenchAsync( + string modelPath, + string benchDir, + int fixedNgl, + BenchmarkSlot slot) + { + string logFile = Path.Combine(benchDir, "llamabench.md"); + + var cmd = BenchmarkCommands.Bench(_bins.Bench, modelPath, slot.UsesGpu, + fixedNgl, BuildTensorSplitArgs(slot, LlamaGpuTool.LlamaBench)); + + await RunFixedCommandWithRetryAsync( + label: "llama-bench", + cmd: cmd, + logFile: logFile, + slot: slot, + attempts: 2, + requirePplMarker: false); + + var parsed = BenchmarkLogParser.ParseLlamaBench(logFile); + if (!parsed.Tps.HasValue || parsed.Tps.Value <= 0) + { + throw new InvalidOperationException( + $"llama-bench completed but no valid TPS could be parsed from {logFile}"); + } + + return parsed; + } + + private async Task RunPplBenchmarkAsync( + string modelPath, + string benchDir, + string domain, + string corpusPath, + int fixedNgl, + BenchmarkSlot slot, + string? klLogitsDir, + bool saveLogits) + { + string logFile = Path.Combine(benchDir, $"perplexity_{domain}.log"); + + string? logitsPath = null; + bool expectKld = false; + + if (!string.IsNullOrEmpty(klLogitsDir)) + { + string logitsFile = Path.Combine(klLogitsDir, $"kld_logits_{domain}.bin"); + + if (saveLogits) + { + logitsPath = logitsFile; + expectKld = false; + } + else if (File.Exists(logitsFile)) + { + logitsPath = logitsFile; + expectKld = true; + } + } + + var cmd = BenchmarkCommands.Perplexity(_bins.Ppl, modelPath, corpusPath, slot.UsesGpu, + fixedNgl, BuildTensorSplitArgs(slot, LlamaGpuTool.CommonCli), logitsPath, expectKld); + + await RunFixedCommandWithRetryAsync( + label: $"perplexity-{domain}", + cmd: cmd, + logFile: logFile, + slot: slot, + attempts: 2, + requirePplMarker: true); + + bool allowMissingKld = !expectKld; + var parsed = BenchmarkLogParser.ParsePerplexity(logFile, allowMissingKld); + + if (expectKld && !HasMeaningfulKld(parsed.Kld)) + { + throw new InvalidOperationException( + $"Expected a real KLD for domain '{domain}', but parsed '{parsed.Kld?.ToString(CultureInfo.InvariantCulture) ?? "null"}' from {logFile}"); + } + + return parsed; + } + + private async Task RunPplBenchmarkWithNglFallbackAsync( + string modelPath, + string benchDir, + string domain, + string corpusPath, + BenchmarkSlot slot, + RuntimeNglState runtimeNgl, + string? klLogitsDir, + bool saveLogits) + { + var fallbackNgls = BuildNglFallbackList(runtimeNgl.CurrentNgl); + Exception? lastException = null; + + for (int i = 0; i < fallbackNgls.Count; i++) + { + int ngl = fallbackNgls[i]; + try + { + var metrics = await RunPplBenchmarkAsync( + modelPath: modelPath, + benchDir: benchDir, + domain: domain, + corpusPath: corpusPath, + fixedNgl: ngl, + slot: slot, + klLogitsDir: klLogitsDir, + saveLogits: saveLogits); + + runtimeNgl.CurrentNgl = ngl; + runtimeNgl.LastSuccessfulNgl = ngl; + return metrics; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + lastException = ex; + string content = ex.ToString(); + bool retryable = LooksLikeRetryableGpuFailure(content); + if (!retryable || i == fallbackNgls.Count - 1) + throw; + + int next = fallbackNgls[i + 1]; + AnsiConsole.MarkupLine( + $"[yellow]GPU failure at ngl={ngl} for model {Markup.Escape(Path.GetFileName(modelPath))}; retrying at ngl={next}.[/]"); + } + } + + throw lastException ?? new InvalidOperationException("Perplexity benchmark failed after NGL fallbacks."); + } + + private async Task RunFixedCommandWithRetryAsync( + string label, + NativeCommand cmd, + string logFile, + BenchmarkSlot slot, + int attempts, + bool requirePplMarker) + { + CommandRunResult? last = null; + + for (int attempt = 1; attempt <= attempts; attempt++) + { + last = await RunNativeCommandAsync(cmd, logFile, slot.BuildProcessEnv()); + + string logContent = !string.IsNullOrWhiteSpace(last.LogOutput) + ? last.LogOutput + : (File.Exists(logFile) ? File.ReadAllText(logFile) : string.Empty); + + bool success = last.Success && logContent.Length >= 50; + + if (success && requirePplMarker) + { + success = LooksLikeSuccessfulPerplexityRun(logFile, logContent); + } + + if (success) + return; + + bool retryable = LooksLikeRetryableGpuFailure(logContent); + + if (attempt < attempts && retryable) + { + int retryNgl = ExtractNglFromCommand(cmd); + AnsiConsole.MarkupLine( + $"[yellow]Transient benchmark failure detected at ngl={retryNgl} on slot {slot.SlotId} ({Markup.Escape(slot.DisplayName)}); retrying same NGL once before fallback.[/]"); + await Task.Delay(1500); + continue; + } + + throw new InvalidOperationException( + $"{label} failed on fixed benchmark slot {slot.SlotId} ({Markup.Escape(slot.DisplayName)}).\n" + + $"Command: {cmd}\n\nLog Output:\n{logContent}"); + } + + throw new InvalidOperationException( + $"{label} failed after {attempts} attempts on slot {slot.SlotId} ({Markup.Escape(slot.DisplayName)}).\n" + + $"{last?.LogOutput}"); + } + + private bool LooksLikeSuccessfulPerplexityRun(string logFile, string logContent) + { + if (string.IsNullOrWhiteSpace(logContent) || logContent.Length < 50) + return false; + + try + { + var parsed = BenchmarkLogParser.ParsePerplexity(logFile, allowMissingKld: true); + return parsed.Ppl > 0; + } + catch + { + return false; + } + } + + private static bool LooksLikeRetryableGpuFailure(string logContent) + { + if (string.IsNullOrWhiteSpace(logContent)) + return false; + + if (OomMarkers.Any(m => logContent.Contains(m, StringComparison.OrdinalIgnoreCase))) + return true; + + if (logContent.Contains("failed to load model", StringComparison.OrdinalIgnoreCase)) + return true; + + if (logContent.Contains("error:", StringComparison.OrdinalIgnoreCase)) + return true; + + return false; + } + + private static int ExtractNglFromCommand(NativeCommand cmd) + { + for (int i = 0; i < cmd.Arguments.Count - 1; i++) + if (cmd.Arguments[i] == "-ngl" && int.TryParse(cmd.Arguments[i + 1], out int ngl)) return ngl; + return 0; + } + + // ---------------------------------------------------------------- + // Parsers + // ---------------------------------------------------------------- + + // ---------------------------------------------------------------- + // Corpus preparation + // ---------------------------------------------------------------- + + private async Task PreparePplCorpusAsync(string domain, string outPath, int tokenTarget) + { + if (tokenTarget <= 0) + throw new ArgumentOutOfRangeException(nameof(tokenTarget), "Token target must be greater than zero."); + + if (IsPplCorpusUsable(outPath, tokenTarget)) + return; + + AnsiConsole.MarkupLine($"[grey]Generating corpus for domain: {domain}[/]"); + + string pyScript = $@" +import sys +from datasets import load_dataset + +domain = '{domain}' +out_path = r'{outPath}' +max_chars = {tokenTarget} * {PplCharsPerTokenEstimate} + +def get_sources(d): + if d == 'general': return [('{GeneralPplDatasetId}', 'wikitext-103-raw-v1', 'test', 'text'), ('{GeneralPplDatasetId}', 'wikitext-2-raw-v1', 'test', 'text')] + if d == 'code': return [('codeparrot/codeparrot-clean', None, 'train', 'content')] + if d == 'math': return [('{MathPplDatasetId}', 'main', 'test', 'question')] + return [] + +parts = [] +total = 0 +source_errors = [] +for ds, conf, split, field in get_sources(domain): + try: + load_args = {{'split': split, 'streaming': True}} + d = load_dataset(ds, conf, **load_args) if conf else load_dataset(ds, **load_args) + for row in d: + text = row.get(field) + if not text or not isinstance(text, str): + continue + chunk = text.strip() + '\n' + parts.append(chunk) + total += len(chunk) + if total >= max_chars: + break + except Exception as e: + message = f'Error loading {{ds}}: {{e}}' + source_errors.append(message) + print(message, file=sys.stderr) + if total >= max_chars: + break + +if total < max_chars: + details = '; '.join(source_errors) or 'dataset sources returned insufficient text' + raise RuntimeError( + f'Failed to build corpus for domain {{domain!r}}: collected ' + f'{{total}} of {{max_chars}} required characters. {{details}}' + ) + +with open(out_path, 'w', encoding='utf-8') as f: + f.write(''.join(parts)) +"; + + string scriptPath = Path.Combine(Path.GetDirectoryName(outPath)!, $"gen_{domain}.py"); + await File.WriteAllTextAsync(scriptPath, pyScript); + + await _pyManager.RunPipInstallAsync("datasets"); + var generationResult = await RunNativeCommandAsync( + new NativeCommand(_pyManager.GetPythonExecutable(), [scriptPath]), null); + + if (!generationResult.Success) + { + throw new InvalidOperationException( + $"Failed to generate perplexity corpus for domain '{domain}'.\n\n{generationResult.LogOutput}"); + } + + if (!IsPplCorpusUsable(outPath, tokenTarget)) + { + throw new InvalidOperationException( + $"Generated perplexity corpus for domain '{domain}' did not contain the required " + + $"{(long)tokenTarget * PplCharsPerTokenEstimate:N0} characters: {outPath}"); + } + } + + internal static bool IsPplCorpusUsable(string path, int tokenTarget) + { + if (tokenTarget <= 0 || !File.Exists(path)) + return false; + + try + { + long minimumCharacters = (long)tokenTarget * PplCharsPerTokenEstimate; + using var reader = new StreamReader(path); + var buffer = new char[4096]; + long totalCharacters = 0; + + while (totalCharacters < minimumCharacters) + { + int read = reader.Read(buffer, 0, buffer.Length); + if (read == 0) + break; + + totalCharacters += read; + } + + return totalCharacters >= minimumCharacters; + } + catch (IOException) + { + return false; + } + catch (UnauthorizedAccessException) + { + return false; + } + } + + // ---------------------------------------------------------------- + // Process / shell utilities + // ---------------------------------------------------------------- + + private sealed class CommandRunResult + { + public bool Success { get; init; } + public int ExitCode { get; init; } + public string LogOutput { get; init; } = string.Empty; + } + + private async Task RunNativeCommandAsync( + NativeCommand cmd, + string? logPath, + IReadOnlyDictionary? extraEnv = null) + { + var startInfo = cmd.CreateStartInfo(extraEnv); + var result = await new MagicQuant.Runtime.ProcessRunner().RunAsync(startInfo, logPath); + return new CommandRunResult + { + Success = result.Success, + ExitCode = result.ExitCode, + LogOutput = result.CombinedOutput + }; + } + + private string GetRelativePath(string fullPath) + { + return Path.GetFileName(fullPath); + } + + // ---------------------------------------------------------------- + // Internal plan / slot types + // ---------------------------------------------------------------- + + private sealed record BenchmarkExecutionPlan( + string PlanModelPath, + int StaticNgl, + bool UsesGpu, + int GroupSize, + IReadOnlyList Slots, + int ProbeSchemaVersion = DynamicProbeSchemaVersion, + ulong Q8ModelSizeBytes = 0, + int Q8StableNgl = 0, + ulong NativeModelSizeBytes = 0, + int NativeStableNgl = 0, + string NativeQuantizationKey = "", + int MaxCandidateNgl = 0, + string GpuMemoryLimitsJson = "{}", + string TensorSplitJson = "{}", + IReadOnlyList? IndependentSlots = null, + ulong IndependentMaxModelSizeBytes = 0, + double SharedMeasuredJobsPerSecond = 0, + double SharedMeasuredSecondsPerPass = 0, + double IndependentMeasuredJobsPerSecond = 0, + double IndependentMeasuredSecondsPerPass = 0) + { + public BenchmarkTopologyProfile SharedProfile => new( + "shared", + Slots, + SharedMeasuredJobsPerSecond, + SharedMeasuredSecondsPerPass); + + public BenchmarkTopologyProfile IndependentProfile => new( + "independent", + IndependentSlots ?? [], + IndependentMeasuredJobsPerSecond, + IndependentMeasuredSecondsPerPass); + + public static BenchmarkExecutionPlan CreateCpuPlan(string q8ModelPath) + => new( + PlanModelPath: q8ModelPath, + StaticNgl: 0, + UsesGpu: false, + GroupSize: 0, + Slots: new List { new(0, Array.Empty()) }, + ProbeSchemaVersion: DynamicProbeSchemaVersion, + Q8ModelSizeBytes: 0, + Q8StableNgl: 0, + NativeModelSizeBytes: 0, + NativeStableNgl: 0, + NativeQuantizationKey: string.Empty, + MaxCandidateNgl: 0, + GpuMemoryLimitsJson: SerializeGpuMemoryLimits(), + TensorSplitJson: "{}"); + } + + public sealed class RuntimeNglState + { + public int CurrentNgl { get; set; } + public int LastSuccessfulNgl { get; set; } + } + + private sealed record ExecutionPlanCacheKey( + string HardwareFingerprint, + string QuantizedModelFingerprint, + string QuantizationKey, + int DiscoveryTokenTarget, + string PlanModelPath); +} diff --git a/src/MagicQuant/Services/CloneConfigManifestGenerationService.cs b/src/MagicQuant/Services/CloneConfigManifestGenerationService.cs new file mode 100644 index 0000000..e8331d5 --- /dev/null +++ b/src/MagicQuant/Services/CloneConfigManifestGenerationService.cs @@ -0,0 +1,303 @@ +using System.Diagnostics; +using System.Text.Json; +using MagicQuant.Models; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class CloneConfigManifestGenerationService +{ + public const string FileName = MagicQuantManifestPathService.CloneConfigsFileName; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + WriteIndented = true + }; + + private readonly QuantizationService _quantizationService; + private readonly HybridBenchmarkRepository _benchmarkRepository = new(); + private readonly FinalArtifactNamingService _namingService = new(); + + public CloneConfigManifestGenerationService(QuantizationService quantizationService) + { + _quantizationService = quantizationService; + } + + public async Task GenerateAsync( + string outputDirectory, + IReadOnlyCollection exportedArtifacts, + BenchmarkSnapshotRecord? pplReference = null, + string? sourceRepository = null, + string? sourceJson = null, + CancellationToken ct = default) + { + string manifestDirectory = MagicQuantManifestPathService.EnsureManifestDirectory(outputDirectory); + + var manifest = new MagicQuantCloneManifest + { + SchemaVersion = 1, + GeneratedUtc = DateTime.UtcNow, + Generator = "MagicQuant", + SourceRepository = sourceRepository, + SourceJson = sourceJson, + SourceModelId = Cache.CurrentModelId, + SourceArchitectureFamily = Cache.CurrentArchitectureFamilyName, + Notes = "Exact GGUF tensor quantization map for repository clone/reproducibility mode. This file is not a proof that another cloned model went through the full MagicQuant discovery pipeline. External reference finalists use persisted SQLite learned tensor truth when no local final GGUF was exported." + }; + + double? referencePpl = ResolveReferencePpl(pplReference, exportedArtifacts.Select(x => x.Snapshot)); + + var orderedArtifacts = exportedArtifacts + .OrderBy(x => x.Snapshot.Kld) + .ThenBy(x => x.Snapshot.SizeBytes) + .ThenBy(x => x.DisplayName, StringComparer.Ordinal) + .ToList(); + + WriteCloneLog($"Starting clone manifest generation for {orderedArtifacts.Count:N0} finalist artifacts."); + + int index = 0; + foreach (var artifact in orderedArtifacts) + { + ct.ThrowIfCancellationRequested(); + index++; + + var artifactSw = Stopwatch.StartNew(); + WriteCloneLog( + $"[{index:N0}/{orderedArtifacts.Count:N0}] Resolving tensor map for '{artifact.DisplayName}' " + + $"provider='{artifact.ProviderName}' family='{artifact.BaselineFamily}' externalReference={artifact.IsExternalReference} externalPureBaseline={artifact.Snapshot.IsExternalPureBaseline} hybrid={artifact.Snapshot.IsHybrid}."); + + CloneTensorMapResolution resolution; + try + { + resolution = await ResolveTensorTypesForCloneAsync(artifact, ct); + } + catch (Exception ex) + { + artifactSw.Stop(); + WriteCloneLog( + $"[{index:N0}/{orderedArtifacts.Count:N0}] FAILED resolving tensor map for '{artifact.DisplayName}' after {FormatDuration(artifactSw.Elapsed)}: {ex.GetType().Name}: {ex.Message}", + isError: true); + throw; + } + + artifactSw.Stop(); + WriteCloneLog( + $"[{index:N0}/{orderedArtifacts.Count:N0}] Resolved '{artifact.DisplayName}' via {resolution.SourceDescription} in {FormatDuration(artifactSw.Elapsed)}; tensors={resolution.TensorTypes.Count:N0}."); + + manifest.Artifacts.Add(new MagicQuantCloneArtifact + { + FileName = ResolveManifestFileName(artifact), + DisplayName = artifact.DisplayName, + ShortName = _namingService.ToShortDisplayName(artifact.DisplayName), + Provider = artifact.ProviderName, + QuantFamily = artifact.BaselineFamily, + BaseQuant = ResolveBaseQuantName(artifact), + IsHybrid = artifact.Snapshot.IsHybrid, + UsedImatrix = Cache.UseImatrix && Cache.IsImatrixAvailable, + SourceKld = artifact.Snapshot.Kld, + SourcePpl = artifact.Snapshot.Ppl, + SourcePplDeltaPercent = FinalReleaseMetadataService.CalculatePplDeltaPercent(artifact.Snapshot.Ppl, referencePpl), + SourceSizeBytes = artifact.ActualSizeBytes ?? artifact.ExpectedSizeBytes, + SourceSizeGB = ToGBNumber(artifact.ActualSizeBytes ?? artifact.ExpectedSizeBytes), + SourceSizeGiB = ToGiBNumber(artifact.ActualSizeBytes ?? artifact.ExpectedSizeBytes), + TensorTypes = resolution.TensorTypes.ToDictionary(x => x.Key, x => x.Value, StringComparer.Ordinal) + }); + } + + string path = Path.Combine(manifestDirectory, FileName); + await File.WriteAllTextAsync(path, JsonSerializer.Serialize(manifest, JsonOptions), ct); + WriteCloneLog($"Clone configuration JSON generated: {path} | artifacts={manifest.Artifacts.Count:N0}"); + return path; + } + + private async Task ResolveTensorTypesForCloneAsync( + ExportedArtifactRecord artifact, + CancellationToken ct) + { + // Critical: an external reference means the final output directory intentionally does not contain + // a local GGUF for this finalist. Do not re-download the upstream GGUF here. The exact learned + // tensor truth was already captured in SQLite during learning/benchmarking, and that is the correct + // source for clone reproducibility metadata. + if (artifact.IsExternalReference) + { + var learned = await LoadExternalPureBaselineTensorTruthAsync(artifact, ct); + return new CloneTensorMapResolution(learned, "SQLite learned tensor truth for external reference"); + } + + if (!string.IsNullOrWhiteSpace(artifact.FullPath) && File.Exists(artifact.FullPath)) + { + var tensorTypes = await _quantizationService.ReadExactTensorTypesAsync(artifact.FullPath, ct); + return new CloneTensorMapResolution( + tensorTypes.ToDictionary(x => x.Key, x => x.Value, StringComparer.Ordinal), + $"local final GGUF '{artifact.FullPath}'"); + } + + if (!string.IsNullOrWhiteSpace(artifact.Snapshot.OutputModelPath) && File.Exists(artifact.Snapshot.OutputModelPath)) + { + var tensorTypes = await _quantizationService.ReadExactTensorTypesAsync(artifact.Snapshot.OutputModelPath, ct); + return new CloneTensorMapResolution( + tensorTypes.ToDictionary(x => x.Key, x => x.Value, StringComparer.Ordinal), + $"existing benchmark GGUF '{artifact.Snapshot.OutputModelPath}'"); + } + + if (artifact.Snapshot.IsExternalPureBaseline && !artifact.Snapshot.IsHybrid) + { + var learned = await LoadExternalPureBaselineTensorTruthAsync(artifact, ct); + return new CloneTensorMapResolution(learned, "SQLite learned tensor truth fallback for external pure baseline"); + } + + throw new InvalidOperationException( + $"Cannot generate clone tensor map for '{artifact.DisplayName}'. No local final GGUF exists at '{artifact.FullPath ?? ""}', " + + $"no existing benchmark GGUF exists at '{artifact.Snapshot.OutputModelPath ?? ""}', and the artifact is not an external pure baseline with persisted learned tensor truth."); + } + + private async Task> LoadExternalPureBaselineTensorTruthAsync( + ExportedArtifactRecord artifact, + CancellationToken ct) + { + var baseline = artifact.Snapshot.Quant.BaseQuant; + + if (!baseline.IsExternalRepositoryBaseline && !artifact.Snapshot.IsExternalPureBaseline) + { + throw new InvalidOperationException( + $"Artifact '{artifact.DisplayName}' was marked as an external reference, but its base quant '{baseline.Names[0]}' is not an external repository baseline and the snapshot is not marked as an external pure baseline."); + } + + if (string.IsNullOrWhiteSpace(baseline.CanonicalKey)) + { + throw new InvalidOperationException( + $"External baseline artifact '{artifact.DisplayName}' has no canonical baseline key, so SQLite learned tensor truth cannot be loaded."); + } + + WriteCloneLog( + $"Loading SQLite learned tensor truth for external baseline '{baseline.Names[0]}' canonicalKey='{baseline.CanonicalKey}' preferredScheme='{baseline.DefaultTensorScheme?.Names[0] ?? ""}'."); + + var strict = await _benchmarkRepository.LoadLearnedTensorMappingsAsync( + canonicalBaselineKey: baseline.CanonicalKey, + groupId: null, + preferredSourceScheme: baseline.DefaultTensorScheme, + allowDominantFallback: false, + ct: ct); + + if (strict.Count > 0) + return strict; + + WriteCloneLog( + $"Strict SQLite learned tensor truth lookup returned 0 rows for '{baseline.Names[0]}'. Trying dominant-scheme fallback for legacy/mixed rows.", + isWarning: true); + + var fallback = await _benchmarkRepository.LoadLearnedTensorMappingsAsync( + canonicalBaselineKey: baseline.CanonicalKey, + groupId: null, + preferredSourceScheme: baseline.DefaultTensorScheme, + allowDominantFallback: true, + ct: ct); + + if (fallback.Count > 0) + return fallback; + + throw new InvalidOperationException( + $"No SQLite learned tensor truth exists for external baseline '{baseline.Names[0]}' canonicalKey='{baseline.CanonicalKey}'. " + + "Final clone manifest generation will not re-download external GGUFs. Re-run the external baseline learning/benchmark stage for this model/context so the tensor truth is present in SQLite."); + } + + private static double ToGBNumber(ulong bytes) => bytes / 1000d / 1000d / 1000d; + private static double ToGiBNumber(ulong bytes) => bytes / 1024d / 1024d / 1024d; + + private static string ResolveBaseQuantName(ExportedArtifactRecord artifact) + { + var quant = artifact.Snapshot.Quant; + if (!string.IsNullOrWhiteSpace(quant.BaseQuant.QuantizeBaseArgumentName)) + return quant.BaseQuant.QuantizeBaseArgumentName; + + if (!quant.BaseQuant.Names.IsDefaultOrEmpty) + return quant.BaseQuant.Names[0]; + + return "Q8_0"; + } + + private static string ResolveManifestFileName(ExportedArtifactRecord artifact) + { + if (!string.IsNullOrWhiteSpace(artifact.FileName)) + return artifact.FileName; + + if (!string.IsNullOrWhiteSpace(artifact.FullPath)) + return Path.GetFileName(artifact.FullPath); + + if (!string.IsNullOrWhiteSpace(artifact.Snapshot.OutputModelPath)) + return Path.GetFileName(artifact.Snapshot.OutputModelPath); + + string? targetName = TryGetFileNameFromDownloadTarget(artifact.DownloadTarget); + if (!string.IsNullOrWhiteSpace(targetName)) + return targetName; + + return ToSafeGgufFileName(artifact.DisplayName); + } + + private static string? TryGetFileNameFromDownloadTarget(string? downloadTarget) + { + if (string.IsNullOrWhiteSpace(downloadTarget)) + return null; + + string value = downloadTarget.Trim(); + int queryIndex = value.IndexOf('?', StringComparison.Ordinal); + if (queryIndex >= 0) + value = value[..queryIndex]; + + value = value.TrimEnd('/'); + string fileName = Path.GetFileName(value.Replace('\\', '/')); + return string.IsNullOrWhiteSpace(fileName) ? null : fileName; + } + + private static string ToSafeGgufFileName(string value) + { + string safe = new string((value ?? string.Empty) + .Select(ch => char.IsLetterOrDigit(ch) || ch is '.' or '_' or '-' ? ch : '_') + .ToArray()) + .Trim('_', '.', '-'); + + if (string.IsNullOrWhiteSpace(safe)) + safe = "external-baseline"; + + return safe.EndsWith(".gguf", StringComparison.OrdinalIgnoreCase) + ? safe + : $"{safe}.gguf"; + } + + private static double? ResolveReferencePpl( + BenchmarkSnapshotRecord? pplReference, + IEnumerable snapshots) + { + if (pplReference is { Ppl: > 0d }) + return pplReference.Ppl; + + return snapshots + .Where(x => x.Ppl > 0d) + .OrderBy(x => x.Kld) + .FirstOrDefault() + ?.Ppl; + } + + private static string FormatDuration(TimeSpan value) => value.ToString(@"hh\:mm\:ss"); + + private static void WriteCloneLog(string message, bool isWarning = false, bool isError = false) + { + string color = isError ? "red" : isWarning ? "yellow" : "grey"; + string line = $"[{DateTime.Now:HH:mm:ss}] Clone manifest: {message}"; + AnsiConsole.MarkupLine($"[{color}]{Markup.Escape(line)}[/]"); + } + + private sealed class CloneTensorMapResolution + { + public CloneTensorMapResolution(Dictionary tensorTypes, string sourceDescription) + { + TensorTypes = tensorTypes; + SourceDescription = sourceDescription; + } + + public Dictionary TensorTypes { get; } + public string SourceDescription { get; } + } +} diff --git a/src/MagicQuant/Services/CloneManifestTensorMapBuildService.cs b/src/MagicQuant/Services/CloneManifestTensorMapBuildService.cs new file mode 100644 index 0000000..223f581 --- /dev/null +++ b/src/MagicQuant/Services/CloneManifestTensorMapBuildService.cs @@ -0,0 +1,529 @@ +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using MagicQuant.Helpers; +using MagicQuant.Models; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed record CloneManifestTensorMapBuildResult( + string OutputPath, + bool UsedManifestSubset, + string EffectiveBaseQuantName, + IReadOnlyList MissingInManifest, + IReadOnlyDictionary ResolvedTensorTypes); + +/// +/// Clone-mode exact tensor-map builder. +/// +/// Normal exact-map builds remain strict. Clone mode can optionally allow a source model +/// to contain extra tensors that are absent from an older manifest. In that case, only the +/// manifest tensors receive explicit --tensor-type overrides; the extra source tensors are +/// intentionally left to llama.cpp's normal base-quant behavior unless a clone-specific +/// missing-manifest base-quant override is provided. +/// +/// After a clone artifact is produced, the service re-reads the output GGUF and returns the +/// actual emitted tensor qtypes. That lets the generated clone manifest become the next run's +/// exact recipe without inventing missing tensor overrides before llama.cpp has decided how +/// to store norms and other special tensors. +/// +public sealed class CloneManifestTensorMapBuildService +{ + public const string AllowMissingManifestTensorsFlag = "allow-missing-manifest-tensors"; + public const string AllowMissingManifestTensorsCliSwitch = "--" + AllowMissingManifestTensorsFlag; + public const string MissingManifestBaseQuantFlag = "missing-manifest-base-quant"; + public const string MissingManifestBaseQuantCliSwitch = "--" + MissingManifestBaseQuantFlag; + + private static readonly JsonSerializerOptions ManifestJsonOptions = new() + { + WriteIndented = true, + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true + }; + + private readonly QuantizationService _quantizationService; + private readonly ImatrixService _imatrixService; + + public CloneManifestTensorMapBuildService( + QuantizationService quantizationService, + ImatrixService imatrixService) + { + _quantizationService = quantizationService ?? throw new ArgumentNullException(nameof(quantizationService)); + _imatrixService = imatrixService ?? throw new ArgumentNullException(nameof(imatrixService)); + } + + public async Task BuildAsync( + IReadOnlyDictionary tensorTypes, + string outputPath, + string baseQuantName, + bool allowMissingManifestTensors, + string? missingManifestBaseQuantName = null, + bool forceRebuild = false, + CancellationToken ct = default) + { + if (tensorTypes == null || tensorTypes.Count == 0) + throw new ArgumentException("A clone tensor map must contain at least one tensor entry.", nameof(tensorTypes)); + + if (string.IsNullOrWhiteSpace(outputPath)) + throw new InvalidOperationException("Export output path is required."); + + bool hasMissingManifestBaseQuantOverride = !string.IsNullOrWhiteSpace(missingManifestBaseQuantName); + bool allowManifestSubset = allowMissingManifestTensors || hasMissingManifestBaseQuantOverride; + + string nativeBasePath = await _quantizationService.EnsureBaseModelFileAsync(); + var sourceTensorTypes = await _quantizationService.ReadExactTensorTypesAsync(nativeBasePath, ct); + var sourceTensorNames = sourceTensorTypes.Keys.ToList(); + + var missingInManifest = sourceTensorNames + .Except(tensorTypes.Keys, StringComparer.Ordinal) + .OrderBy(x => x, StringComparer.Ordinal) + .ToList(); + + var unexpectedInManifest = tensorTypes.Keys + .Except(sourceTensorNames, StringComparer.Ordinal) + .OrderBy(x => x, StringComparer.Ordinal) + .ToList(); + + bool exactMatch = missingInManifest.Count == 0 && + unexpectedInManifest.Count == 0 && + sourceTensorNames.Count == tensorTypes.Count; + + if (exactMatch) + { + var exactOutputPath = await _quantizationService.BuildExportArtifactFromExactTensorMapAsync( + tensorTypes: tensorTypes, + outputPath: outputPath, + baseQuantName: baseQuantName, + forceRebuild: forceRebuild, + ct: ct); + + var resolvedTensorTypes = await CaptureResolvedTensorTypesAsync(exactOutputPath, tensorTypes, ct); + await PersistResolvedTensorTypesToOutputManifestAsync(exactOutputPath, resolvedTensorTypes, ct); + + return new CloneManifestTensorMapBuildResult( + OutputPath: exactOutputPath, + UsedManifestSubset: false, + EffectiveBaseQuantName: baseQuantName, + MissingInManifest: Array.Empty(), + ResolvedTensorTypes: resolvedTensorTypes); + } + + bool sourceModelIsManifestSuperset = missingInManifest.Count > 0 && unexpectedInManifest.Count == 0; + if (!allowManifestSubset || !sourceModelIsManifestSuperset) + { + throw new InvalidOperationException(BuildManifestMismatchError( + missingInManifest, + unexpectedInManifest, + modelTensorCount: sourceTensorNames.Count, + manifestTensorCount: tensorTypes.Count, + includeSubsetHint: sourceModelIsManifestSuperset)); + } + + return await BuildSubsetOverrideCloneAsync( + inputFile: nativeBasePath, + outputFile: outputPath, + tensorTypes: tensorTypes, + baseQuantName: baseQuantName, + missingManifestBaseQuantName: missingManifestBaseQuantName, + missingInManifest: missingInManifest, + forceRebuild: forceRebuild, + ct: ct); + } + + private async Task BuildSubsetOverrideCloneAsync( + string inputFile, + string outputFile, + IReadOnlyDictionary tensorTypes, + string baseQuantName, + string? missingManifestBaseQuantName, + IReadOnlyList missingInManifest, + bool forceRebuild, + CancellationToken ct) + { + var baseQuant = ResolveCloneBaseQuantOrThrow(baseQuantName, missingManifestBaseQuantName); + bool hasMissingManifestBaseQuantOverride = !string.IsNullOrWhiteSpace(missingManifestBaseQuantName); + + Directory.CreateDirectory(Path.GetDirectoryName(outputFile)!); + + if (!forceRebuild && File.Exists(outputFile) && new FileInfo(outputFile).Length > 0) + { + var reusedResolvedTensorTypes = await CaptureResolvedTensorTypesAsync(outputFile, tensorTypes, ct); + await PersistResolvedTensorTypesToOutputManifestAsync(outputFile, reusedResolvedTensorTypes, ct); + + return new CloneManifestTensorMapBuildResult( + OutputPath: outputFile, + UsedManifestSubset: true, + EffectiveBaseQuantName: baseQuant.Names[0], + MissingInManifest: missingInManifest.ToArray(), + ResolvedTensorTypes: reusedResolvedTensorTypes); + } + + if (forceRebuild) + { + await HardDeleteHelper.DeleteFileIfExistsAsync(outputFile); + await HardDeleteHelper.DeleteFileIfExistsAsync(outputFile + ".success.json"); + } + + AnsiConsole.MarkupLine( + $"[yellow]Clone manifest subset allowed:[/] [cyan]{missingInManifest.Count:N0}[/] source tensor(s) are absent from the manifest and will receive no explicit --tensor-type override."); + + AnsiConsole.MarkupLine(hasMissingManifestBaseQuantOverride + ? $"[yellow]Missing-manifest base quant override:[/] [cyan]{Markup.Escape(baseQuant.Names[0])}[/] will be used for source tensors absent from the manifest." + : $"[grey]Missing-manifest tensors will use artifact base quant:[/] {Markup.Escape(baseQuant.Names[0])}"); + + foreach (var tensorName in missingInManifest.Take(15)) + AnsiConsole.MarkupLine($"[grey] basequant fallback tensor:[/] {Markup.Escape(tensorName)}"); + + if (missingInManifest.Count > 15) + AnsiConsole.MarkupLine($"[grey] ...and {missingInManifest.Count - 15:N0} more tensor(s).[/]"); + + var args = new List(capacity: tensorTypes.Count * 2 + 8); + foreach (var kv in tensorTypes.OrderBy(x => x.Key, StringComparer.Ordinal)) + { + args.Add("--tensor-type"); + args.Add($"{kv.Key}={NormalizeCloneQuantName(kv.Value)}"); + } + + if (_imatrixService.ShouldUseImatrixForQuant(HybridQuant.CreatePureBaseline(baseQuant))) + { + string imatrixPath = _imatrixService.GetCanonicalImatrixPath(); + if (!File.Exists(imatrixPath)) + throw new InvalidOperationException($"Imatrix was marked active but canonical artifact is missing: {imatrixPath}"); + + args.Add("--imatrix"); + args.Add(imatrixPath); + } + + args.Add(inputFile); + args.Add(outputFile); + args.Add(baseQuant.QuantizeBaseArgumentName); + args.Add(ResolveCloneQuantizeThreadCount().ToString()); + + string bin = Path.Combine( + Cache.LlamaBin!, + RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "llama-quantize.exe" : "llama-quantize"); + + string quantizeLogPath = outputFile + ".quantize.log"; + Directory.CreateDirectory(Path.GetDirectoryName(quantizeLogPath)!); + + var psi = new ProcessStartInfo + { + FileName = bin + }; + + foreach (var arg in args) + psi.ArgumentList.Add(arg); + + AnsiConsole.MarkupLine( + $"[cyan]Quantizing clone artifact from manifest subset:[/] {Markup.Escape(Path.GetFileName(outputFile))} [grey](base quant: {Markup.Escape(baseQuant.Names[0])}; log: {Markup.Escape(quantizeLogPath)})[/]"); + + var result = await RunLoggedProcessAsync(psi, quantizeLogPath, ct); + if (result.ExitCode != 0) + { + await HardDeleteHelper.DeleteFileIfExistsAsync(outputFile); + throw new InvalidOperationException( + $"Clone quantization failed for '{outputFile}'. ExitCode={result.ExitCode}. See '{quantizeLogPath}'."); + } + + if (!File.Exists(outputFile) || new FileInfo(outputFile).Length == 0) + { + await HardDeleteHelper.DeleteFileIfExistsAsync(outputFile); + throw new InvalidOperationException( + $"Clone quantization exited successfully but produced no valid GGUF output: {outputFile}"); + } + + await File.WriteAllTextAsync(outputFile + ".success.json", "{\"status\":\"success\"}", ct); + AnsiConsole.MarkupLine($"[green]Clone quantized model ready:[/] {Markup.Escape(outputFile)}"); + + var resolvedTensorTypes = await CaptureResolvedTensorTypesAsync(outputFile, tensorTypes, ct); + await PersistResolvedTensorTypesToOutputManifestAsync(outputFile, resolvedTensorTypes, ct); + + return new CloneManifestTensorMapBuildResult( + OutputPath: outputFile, + UsedManifestSubset: true, + EffectiveBaseQuantName: baseQuant.Names[0], + MissingInManifest: missingInManifest.ToArray(), + ResolvedTensorTypes: resolvedTensorTypes); + } + + private async Task> CaptureResolvedTensorTypesAsync( + string outputFile, + IReadOnlyDictionary fallbackTensorTypes, + CancellationToken ct) + { + try + { + var resolved = await _quantizationService.ReadExactTensorTypesAsync(outputFile, ct); + if (resolved.Count > 0) + { + AnsiConsole.MarkupLine($"[green]Captured resolved clone tensor map from GGUF:[/] {resolved.Count:N0} tensor(s)"); + return resolved + .OrderBy(x => x.Key, StringComparer.Ordinal) + .ToDictionary(x => x.Key, x => x.Value, StringComparer.Ordinal); + } + } + catch (Exception ex) + { + AnsiConsole.MarkupLine($"[yellow]Could not capture resolved tensor map from clone GGUF; preserving manifest tensor map:[/] {Markup.Escape(ex.Message)}"); + } + + return fallbackTensorTypes + .OrderBy(x => x.Key, StringComparer.Ordinal) + .ToDictionary(x => x.Key, x => x.Value, StringComparer.Ordinal); + } + + private static async Task PersistResolvedTensorTypesToOutputManifestAsync( + string outputFile, + IReadOnlyDictionary resolvedTensorTypes, + CancellationToken ct) + { + if (resolvedTensorTypes.Count == 0) + return; + + string? outputDirectory = Path.GetDirectoryName(outputFile); + if (string.IsNullOrWhiteSpace(outputDirectory)) + return; + + string manifestPath = MagicQuantManifestPathService.GetManifestFilePath(outputDirectory, MagicQuantManifestPathService.CloneConfigsFileName); + if (!File.Exists(manifestPath)) + return; + + try + { + var root = JsonNode.Parse(await File.ReadAllTextAsync(manifestPath, ct)) as JsonObject; + var artifacts = TryGetProperty(root, "artifacts") as JsonArray; + if (root == null || artifacts == null) + return; + + string fileName = Path.GetFileName(outputFile); + JsonObject? matchingArtifact = null; + foreach (var node in artifacts.OfType()) + { + string? artifactFileName = TryGetString(node, "fileName"); + if (string.Equals(artifactFileName, fileName, StringComparison.OrdinalIgnoreCase)) + { + matchingArtifact = node; + break; + } + } + + if (matchingArtifact == null) + return; + + var tensorTypesNode = new JsonObject(); + foreach (var kv in resolvedTensorTypes.OrderBy(x => x.Key, StringComparer.Ordinal)) + tensorTypesNode[kv.Key] = kv.Value; + + matchingArtifact["tensorTypes"] = tensorTypesNode; + matchingArtifact["resolvedTensorTypeCount"] = resolvedTensorTypes.Count; + matchingArtifact["resolvedTensorTypesGeneratedAtUtc"] = DateTimeOffset.UtcNow.ToString("O"); + + await File.WriteAllTextAsync(manifestPath, root.ToJsonString(ManifestJsonOptions), ct); + AnsiConsole.MarkupLine( + $"[green]Resolved tensorTypes persisted to clone manifest:[/] {Markup.Escape(fileName)} [grey]({resolvedTensorTypes.Count:N0} tensor(s))[/]"); + } + catch (Exception ex) + { + AnsiConsole.MarkupLine( + $"[yellow]Could not persist resolved tensorTypes to clone manifest; build output is still valid:[/] {Markup.Escape(ex.Message)}"); + } + } + + private static JsonNode? TryGetProperty(JsonObject? obj, string name) + { + if (obj == null) + return null; + + foreach (var kv in obj) + { + if (string.Equals(kv.Key, name, StringComparison.OrdinalIgnoreCase)) + return kv.Value; + } + + return null; + } + + private static string? TryGetString(JsonObject obj, string name) + => TryGetProperty(obj, name)?.GetValue(); + + private static BaselineQuants ResolveCloneBaseQuantOrThrow(string baseQuantName, string? missingManifestBaseQuantName) + { + bool hasOverride = !string.IsNullOrWhiteSpace(missingManifestBaseQuantName); + string quantName = hasOverride ? missingManifestBaseQuantName!.Trim() : baseQuantName; + + var resolved = BaselineQuants.ResolveBuiltInStandardBaseline(quantName); + if (resolved != null) + return resolved; + + if (hasOverride) + { + throw new InvalidOperationException( + $"Unknown {MissingManifestBaseQuantCliSwitch} value '{missingManifestBaseQuantName}'. " + + "Use a built-in llama.cpp base quant name such as Q8_0, Q6_K, Q5_K_M, or Q4_K_M."); + } + + return BaselineQuants.Q8_0; + } + + private static string BuildManifestMismatchError( + IReadOnlyList missingInManifest, + IReadOnlyList unexpectedInManifest, + int modelTensorCount, + int manifestTensorCount, + bool includeSubsetHint) + { + var builder = new StringBuilder(); + builder.Append("Clone tensor manifest does not exactly match this model architecture. "); + builder.Append($"MissingInManifest=[{string.Join(", ", missingInManifest.Take(20))}] "); + builder.Append($"UnexpectedInManifest=[{string.Join(", ", unexpectedInManifest.Take(20))}] "); + builder.Append($"ModelTensorCount={modelTensorCount} ManifestTensorCount={manifestTensorCount}."); + + if (includeSubsetHint) + { + builder.AppendLine(); + builder.Append("This looks like a clone manifest subset: every manifest tensor exists in the current model, "); + builder.Append("but the current model has extra tensors. To let those extra tensors fall through to the artifact base quant, rerun clone mode with "); + builder.Append(AllowMissingManifestTensorsCliSwitch); + builder.Append(". To force those extra tensors to a specific base quant, use "); + builder.Append(MissingManifestBaseQuantCliSwitch); + builder.Append(" Q8_0."); + } + + return builder.ToString(); + } + + private static string NormalizeCloneQuantName(string value) + { + if (string.IsNullOrWhiteSpace(value)) + return "UNKNOWN"; + + string token = value.Trim().Replace("-", "_").Replace(" ", string.Empty).ToUpperInvariant(); + + foreach (var scheme in TensorWeightScheme.All) + { + if (scheme.Names.IsDefaultOrEmpty) + continue; + + if (scheme.Names.Any(name => string.Equals( + name.Trim().Replace("-", "_").Replace(" ", string.Empty).ToUpperInvariant(), + token, + StringComparison.Ordinal))) + { + return scheme.Names[0]; + } + } + + return token; + } + + private static int ResolveCloneQuantizeThreadCount() + { + int threadCount = Cache.SysInfo?.ThreadCount ?? Environment.ProcessorCount; + int reservedThreads = threadCount switch + { + >= 16 => 2, + >= 8 => 2, + >= 4 => 1, + _ => 0 + }; + + return Math.Max(1, threadCount - reservedThreads); + } + + private static async Task RunLoggedProcessAsync( + ProcessStartInfo psi, + string logPath, + CancellationToken ct) + { + psi.RedirectStandardOutput = true; + psi.RedirectStandardError = true; + psi.UseShellExecute = false; + psi.CreateNoWindow = true; + + using var process = new Process + { + StartInfo = psi, + EnableRaisingEvents = true + }; + + var stdoutBuilder = new StringBuilder(); + var stderrBuilder = new StringBuilder(); + object sync = new(); + + var stdoutClosed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var stderrClosed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + await using var logStream = new FileStream(logPath, FileMode.Create, FileAccess.Write, FileShare.Read); + await using var logWriter = new StreamWriter(logStream) { AutoFlush = true }; + + void HandleLine(string? line, bool isError) + { + if (line == null) + { + if (isError) + stderrClosed.TrySetResult(true); + else + stdoutClosed.TrySetResult(true); + + return; + } + + lock (sync) + { + if (isError) + stderrBuilder.AppendLine(line); + else + stdoutBuilder.AppendLine(line); + + logWriter.WriteLine(line); + } + + if (Cache.VerboseProcessOutput) + AnsiConsole.WriteLine(line); + } + + process.OutputDataReceived += (_, e) => HandleLine(e.Data, isError: false); + process.ErrorDataReceived += (_, e) => HandleLine(e.Data, isError: true); + + if (!process.Start()) + throw new InvalidOperationException($"Failed to start process: {psi.FileName}"); + + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + using var ctr = ct.Register(() => + { + try + { + if (!process.HasExited) + process.Kill(entireProcessTree: true); + } + catch + { + } + }); + + await process.WaitForExitAsync(ct); + await Task.WhenAll(stdoutClosed.Task, stderrClosed.Task); + + return new LoggedProcessResult + { + ExitCode = process.ExitCode, + StdOut = stdoutBuilder.ToString(), + StdErr = stderrBuilder.ToString() + }; + } + + private sealed class LoggedProcessResult + { + public int ExitCode { get; init; } + public string StdOut { get; init; } = string.Empty; + public string StdErr { get; init; } = string.Empty; + } +} diff --git a/src/MagicQuant/Services/CloneReadmeGenerationService.cs b/src/MagicQuant/Services/CloneReadmeGenerationService.cs new file mode 100644 index 0000000..bc7e1c2 --- /dev/null +++ b/src/MagicQuant/Services/CloneReadmeGenerationService.cs @@ -0,0 +1,31 @@ +using MagicQuant.Models; + +namespace MagicQuant.Services; + +/// +/// Compatibility wrapper kept so existing call sites can move over gradually. +/// The actual README body/table/frontmatter logic is centralized in ReadmeGenerationService. +/// +public sealed class CloneReadmeGenerationService +{ + private readonly ReadmeGenerationService _readmeGenerationService = new(); + + public Task GenerateAsync( + string outputDirectory, + string modelName, + string sourceDescription, + bool sourceWasHuggingFaceRepo, + IReadOnlyCollection records, + IReadOnlyCollection? archivedManifestFileNames = null, + CancellationToken ct = default) + { + return _readmeGenerationService.GenerateCloneAsync( + outputDirectory, + modelName, + sourceDescription, + sourceWasHuggingFaceRepo, + records, + archivedManifestFileNames, + ct); + } +} diff --git a/src/MagicQuant/Services/CombinationDatabasePathService.cs b/src/MagicQuant/Services/CombinationDatabasePathService.cs new file mode 100644 index 0000000..f1c5f47 --- /dev/null +++ b/src/MagicQuant/Services/CombinationDatabasePathService.cs @@ -0,0 +1,37 @@ +using MagicQuant.Helpers; +using MQ.DB; + +namespace MagicQuant.Services; + +/// +/// Shared path contract for the DuckDB writer and prediction reader. Changing this +/// filename opens a different candidate database; preserve it across refactors. +/// SQLite remains the authority for measured truth, while DuckDB is derived state. +/// +public static class CombinationDatabasePathService +{ + private const string DbFileNamePrefix = "MagicQuant_Combinations"; + + public static string GetPath() => Path.Combine(GetDirectory(), GetFileName()); + + public static string GetDirectory() + { + if (!string.IsNullOrWhiteSpace(Cache.ModelMagicQuantDirectory)) + return Cache.ModelMagicQuantDirectory!; + + if (!string.IsNullOrWhiteSpace(Cache.MagicQuantDirectory)) + return Cache.MagicQuantDirectory!; + + throw new InvalidOperationException( + "Neither Cache.ModelMagicQuantDirectory nor Cache.MagicQuantDirectory is set."); + } + + public static string GetFileName() + { + string model = string.IsNullOrWhiteSpace(Cache.CurrentModelId) ? "unknown-model" : Cache.CurrentModelId; + string imatrix = Cache.IsImatrixAvailable ? (Cache.ActiveImatrixIdentityHash ?? "imatrix-unknown") : "no-imatrix"; + string hp = RuntimeSearchSpace.AllowHighPrecisionHybrids ? "hp-on" : "hp-off"; + return $"{DbFileNamePrefix}_{model}_{imatrix}_{hp}.duckdb"; + } + +} diff --git a/src/MagicQuant/Services/CombinationDuckDbSchema.cs b/src/MagicQuant/Services/CombinationDuckDbSchema.cs new file mode 100644 index 0000000..b99b2af --- /dev/null +++ b/src/MagicQuant/Services/CombinationDuckDbSchema.cs @@ -0,0 +1,101 @@ +using System.Linq; + +namespace MagicQuant.Services; + +internal static class CombinationDuckDbSchema +{ + public const string TableName = "tensor_configs"; + public const string SlotColumnList = "BaseQuant, Embeddings, LmHead, AttnQ, AttnKV, AttnOutput, FfnUpGate, FfnDown, MoeExperts, MoeRouter"; + public const string PredictionColumnList = "PredictedKld, PredictedSizeBytes, PredictionConfidence, PredictionRank, BaseRankSafeKld, AnomalyAdjustmentKld, FinalPredictedKld, IsProtectedAnchor, IsVirtualPredictionAnchor, AnchorBaselineRuntimeId, AnchorBaselineCanonicalKey, AnchorDisplayName"; + public const string ActiveCandidatePredicateSql = "COALESCE(IsProtectedAnchor, FALSE) = FALSE"; + public const string VirtualPredictionAnchorPredicateSql = "COALESCE(IsVirtualPredictionAnchor, FALSE) = TRUE"; + public const string EffectivePredictedKldSql = "COALESCE(FinalPredictedKld, PredictedKld)"; + public const string HybridPredicateSql = "(Embeddings <> 0 OR LmHead <> 0 OR AttnQ <> 0 OR AttnKV <> 0 OR AttnOutput <> 0 OR FfnUpGate <> 0 OR FfnDown <> 0 OR MoeExperts <> 0 OR MoeRouter <> 0)"; + + public static readonly string[] SlotColumns = + [ + "BaseQuant", + "Embeddings", + "LmHead", + "AttnQ", + "AttnKV", + "AttnOutput", + "FfnUpGate", + "FfnDown", + "MoeExperts", + "MoeRouter" + ]; + + public static readonly string[] ExpectedColumnTypes = + [ + "utinyint","utinyint","utinyint","utinyint","utinyint","utinyint","utinyint","utinyint","utinyint","utinyint", + "double","ubigint","double","ubigint", + "double","double","double","boolean", + "boolean","utinyint","varchar","varchar" + ]; + + public static string CreateTableSql => $@" +DROP TABLE IF EXISTS {TableName}; +CREATE TABLE {TableName} ( + BaseQuant UTINYINT, + Embeddings UTINYINT, + LmHead UTINYINT, + AttnQ UTINYINT, + AttnKV UTINYINT, + AttnOutput UTINYINT, + FfnUpGate UTINYINT, + FfnDown UTINYINT, + MoeExperts UTINYINT, + MoeRouter UTINYINT, + + -- Transient DuckDB-only prediction metadata. + -- SQLite remains the real benchmark truth source. + PredictedKld DOUBLE, + PredictedSizeBytes UBIGINT, + PredictionConfidence DOUBLE, + PredictionRank UBIGINT, + + -- Normal PAVA output before scoped anomaly exceptions. + BaseRankSafeKld DOUBLE, + + -- Scoped post-PAVA anomaly/rule adjustment. This is prediction-space only. + AnomalyAdjustmentKld DOUBLE DEFAULT 0.0, + FinalPredictedKld DOUBLE, + + -- Protected/reference anchors may be stored for lookup/logging, but must + -- never become active search candidates. Normal generator rows default false. + IsProtectedAnchor BOOLEAN DEFAULT FALSE, + + -- Virtual prediction anchors are not real benchmark rows. They are ordinary + -- tensor-config rows shaped like uniform learned-baseline blankets so the + -- existing rank-safe prediction materializer can score them in the same + -- imaginary space as normal candidates. + IsVirtualPredictionAnchor BOOLEAN DEFAULT FALSE, + AnchorBaselineRuntimeId UTINYINT, + AnchorBaselineCanonicalKey VARCHAR, + AnchorDisplayName VARCHAR +);"; + + public static string BuildSlotEqualityPredicate(string leftAlias, string rightAlias) + { + return string.Join(" AND ", SlotColumns.Select(c => $"{leftAlias}.{c} = {rightAlias}.{c}")); + } + + public static string QualifySlotColumnList(string alias) + { + return string.Join(", ", SlotColumns.Select(c => $"{alias}.{c}")); + } + + public static string QualifyHybridPredicate(string alias) + { + return HybridPredicateSql.Replace("Embeddings", $"{alias}.Embeddings") + .Replace("LmHead", $"{alias}.LmHead") + .Replace("AttnQ", $"{alias}.AttnQ") + .Replace("AttnKV", $"{alias}.AttnKV") + .Replace("AttnOutput", $"{alias}.AttnOutput") + .Replace("FfnUpGate", $"{alias}.FfnUpGate") + .Replace("FfnDown", $"{alias}.FfnDown") + .Replace("MoeExperts", $"{alias}.MoeExperts") + .Replace("MoeRouter", $"{alias}.MoeRouter"); + } +} diff --git a/src/MagicQuant/Services/CombinationSurvivalPipelineService.cs b/src/MagicQuant/Services/CombinationSurvivalPipelineService.cs new file mode 100644 index 0000000..8f3aece --- /dev/null +++ b/src/MagicQuant/Services/CombinationSurvivalPipelineService.cs @@ -0,0 +1,260 @@ +using System.Diagnostics; +using MagicQuant.Helpers; +using MagicQuant.Models; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class CombinationSurvivalPipelineService +{ + private readonly QuantizationService _quantizationService; + private readonly RemainingCombinationStore _combinationStore; + private readonly HybridBenchmarkRepository _benchmarkRepository; + private readonly EffectiveCandidateStateResolverService _effectiveResolver; + private readonly RankSafeKldPredictionService _predictionService; + private readonly DuckDbPredictionMaterializationService _materializationService; + private readonly FinalRealBenchmarkEliminationService _finalEliminator; + private readonly PredictionGuidedHybridSelectionService _selectionEngine; + private readonly FinalSurvivorSelectionCliService _selectionCli; + private readonly HybridArtifactExportService _exportService; + private readonly ReadmeGenerationService _readmeService; + private readonly HybridMapGenerationService _hybridMapService; + private readonly SelectionDiagnosticsLogService _diagnosticsLogService; + private readonly FinalReleaseMetadataService _releaseMetadataService; + private readonly CloneConfigManifestGenerationService _cloneConfigManifestService; + private readonly FinalArtifactNamingService _namingService; + private readonly IsolationDiagnosticsManifestService _isolationDiagnosticsManifestService; + private readonly AnomalyWorkflowService _anomalyWorkflowService; + + public CombinationSurvivalPipelineService(QuantizationService quantizationService) + { + _quantizationService = quantizationService; + _combinationStore = new RemainingCombinationStore(); + _benchmarkRepository = new HybridBenchmarkRepository(); + _effectiveResolver = new EffectiveCandidateStateResolverService(_benchmarkRepository); + _predictionService = new RankSafeKldPredictionService(_benchmarkRepository, _effectiveResolver); + _finalEliminator = new FinalRealBenchmarkEliminationService(); + _materializationService = new DuckDbPredictionMaterializationService(_combinationStore, _predictionService); + _selectionEngine = new PredictionGuidedHybridSelectionService(_quantizationService, _benchmarkRepository, _finalEliminator, _combinationStore); + _selectionCli = new FinalSurvivorSelectionCliService(); + var pyManager = new PythonManager(Cache.MagicQuantDirectory!); + var sidecarService = new ModelSidecarArtifactService(pyManager); + _exportService = new HybridArtifactExportService(_quantizationService, _effectiveResolver, sidecarService); + _readmeService = new ReadmeGenerationService(); + _hybridMapService = new HybridMapGenerationService(); + _diagnosticsLogService = new SelectionDiagnosticsLogService(); + _releaseMetadataService = new FinalReleaseMetadataService(); + _cloneConfigManifestService = new CloneConfigManifestGenerationService(_quantizationService); + _namingService = new FinalArtifactNamingService(); + _isolationDiagnosticsManifestService = new IsolationDiagnosticsManifestService(); + _anomalyWorkflowService = new AnomalyWorkflowService(_combinationStore, _benchmarkRepository, _quantizationService); + } + + public async Task RunAsync( + RequiredSampleGenerationResult? isolationSamplePlan = null, + IsolationOptimizationResult? isolationOptimizationResult = null, + CancellationToken ct = default) + { + var report = new SurvivalStageReport + { + StartingCount = await _combinationStore.CountAsync(ct) + }; + + AnsiConsole.Write(new Rule("[yellow]Rank-Safe Prediction / Hybrid Selection Pipeline[/]") { Justification = Justify.Left }); + AnsiConsole.MarkupLine($"[green]Remaining DuckDB combinations available to score:[/] [cyan]{report.StartingCount:N0}[/]"); + AnsiConsole.MarkupLine("[grey]Old MDA bucket survival is disabled. DuckDB now defines the allowed search space; rank-safe isolation prediction selects what deserves real benchmarking.[/]"); + AnsiConsole.MarkupLine("[grey]DuckDB prediction materialization is enabled; final selection will query pre-ranked candidates instead of loading the full search space into memory.[/]"); + var pureBaselines = await _benchmarkRepository.LoadPureBaselineSnapshotsAsync(ct); + + if (pureBaselines.Count == 0) + throw new InvalidOperationException("No pure baseline benchmark snapshots were available. Run the baseline/isolation phases before final hybrid selection."); + + AnsiConsole.MarkupLine($"[green]Pure baseline snapshots loaded:[/] [cyan]{pureBaselines.Count:N0}[/]"); + + var materialization = await _materializationService.MaterializeAsync(ct); + AnsiConsole.MarkupLine($"[green]DuckDB predicted rows:[/] [cyan]{materialization.PredictedRows:N0}[/] / [cyan]{materialization.TotalRows:N0}[/] (ranked: {materialization.RankedRows:N0})"); + + var anomalyResult = await _anomalyWorkflowService.RunAsync(pureBaselines, ct); + if (anomalyResult.AdjustmentSummary.MatchedRowCount > 0) + { + AnsiConsole.MarkupLine($"[green]Anomaly-adjusted prediction rows:[/] [cyan]{anomalyResult.AdjustmentSummary.MatchedRowCount:N0}[/] matched by [cyan]{anomalyResult.AdjustmentSummary.AppliedRuleCount:N0}[/] scoped rules. Final selector will use adjusted prediction ranks."); + } + + var selection = await _selectionEngine.RunAsync( + pureBaselines, + ct); + + report.EndingCount = selection.Survivors.Count; + report.AddRemoval("prediction-guided-non-selected", Math.Max(0L, report.StartingCount - report.EndingCount)); + + AnsiConsole.MarkupLine($"[green]Final candidate/anchor survivors before manual enablement:[/] [cyan]{selection.Survivors.Count:N0}[/]"); + AnsiConsole.MarkupLine($"[yellow]Recorded baseline/anchor eliminations:[/] [cyan]{selection.Eliminations.Count:N0}[/]"); + AnsiConsole.MarkupLine($"[yellow]Prediction validation misses:[/] [cyan]{selection.ValidationFailures.Count:N0}[/]"); + RenderEliminationSummary(selection.Eliminations, pureBaselines); + + var nativeReference = await _benchmarkRepository.LoadBenchmarkSnapshotAsync( + (TensorConfig)HybridQuant.CreatePureBaseline(BaselineQuants.GetNativeQuant()), + ct); + + var selectedRows = _selectionCli.Prompt(selection.Survivors, pureBaselines, nativeReference); + + var exportedArtifacts = await _exportService.ExportAsync(selectedRows, pureBaselines, ct); + + string modelName = string.IsNullOrWhiteSpace(Cache.ModelDirectory) + ? "model" + : new DirectoryInfo(Cache.ModelDirectory!).Name; + + var benchmarkOverview = selection.Survivors + .Concat(pureBaselines) + .Concat(selection.ValidationFailures.Select(x => x.Snapshot).OfType()) + .DistinctBy(x => TensorConfigIdentity.ToKey(x.Config)) + .OrderBy(x => x.Kld) + .ThenBy(x => x.SizeBytes) + .ToList(); + + await RunFinalOutputStageAsync( + "selection diagnostics log", + () => _diagnosticsLogService.WriteAsync(benchmarkOverview, selection.ValidationFailures, ct)); + + await RunFinalOutputStageAsync( + "hybrid map JSON", + () => _hybridMapService.GenerateAsync(Cache.OutputDirectory!, exportedArtifacts, ct)); + + await RunFinalOutputStageAsync( + "final survivor / replacement metadata JSON", + () => _releaseMetadataService.GenerateAsync( + Cache.OutputDirectory!, + exportedArtifacts, + selection.Eliminations, + pureBaselines, + nativeReference, + ct)); + + await RunFinalOutputStageAsync( + "clone configuration manifest JSON", + () => _cloneConfigManifestService.GenerateAsync( + Cache.OutputDirectory!, + exportedArtifacts, + nativeReference, + ct: ct)); + + if (isolationSamplePlan != null) + { + await RunFinalOutputStageAsync( + "isolation sample manifest JSON", + () => _isolationDiagnosticsManifestService.GenerateIsolationSamplesAsync( + Cache.OutputDirectory!, + isolationSamplePlan, + ct)); + } + + if (isolationOptimizationResult != null) + { + await RunFinalOutputStageAsync( + "bad trade manifest JSON", + () => _isolationDiagnosticsManifestService.GenerateBadTradesAsync( + Cache.OutputDirectory!, + isolationOptimizationResult, + ct)); + } + + await RunFinalOutputStageAsync( + "README", + () => _readmeService.GenerateAsync( + Cache.OutputDirectory!, + modelName, + exportedArtifacts, + pureBaselines, + selection.Eliminations, + nativeReference, + ct)); + + return new CombinationSurvivalExecutionResult + { + BenchmarkedSnapshots = benchmarkOverview, + BrutalSurvivors = selection.Survivors, + SelectedRows = selectedRows, + ExportedArtifacts = exportedArtifacts, + BucketDiagnostics = Array.Empty(), + SurvivalReport = report, + Eliminations = selection.Eliminations, + ValidationFailures = selection.ValidationFailures + }; + } + + + private static async Task RunFinalOutputStageAsync(string stageName, Func action) + { + var sw = Stopwatch.StartNew(); + WriteFinalOutputLog($"START {stageName}"); + + try + { + await action(); + sw.Stop(); + WriteFinalOutputLog($"DONE {stageName} in {FormatDuration(sw.Elapsed)}"); + } + catch (Exception ex) + { + sw.Stop(); + WriteFinalOutputLog($"FAILED {stageName} after {FormatDuration(sw.Elapsed)}: {ex.GetType().Name}: {ex.Message}", isError: true); + throw; + } + } + + private static string FormatDuration(TimeSpan value) => value.ToString(@"hh\:mm\:ss"); + + private static void WriteFinalOutputLog(string message, bool isError = false) + { + string color = isError ? "red" : "grey"; + string line = $"[{DateTime.Now:HH:mm:ss}] Final output: {message}"; + AnsiConsole.MarkupLine($"[{color}]{Markup.Escape(line)}[/]"); + } + + private void RenderEliminationSummary( + IReadOnlyCollection eliminations, + IReadOnlyCollection pureBaselineSnapshots) + { + if (eliminations.Count == 0) + return; + + var namingContext = _namingService.CreateContext(pureBaselineSnapshots); + + AnsiConsole.Write(new Rule("[yellow]Baseline / Anchor Eliminations[/]") { Justification = Justify.Left }); + + var table = new Table().Border(TableBorder.Rounded); + table.AddColumn("Removed"); + table.AddColumn("Winner"); + table.AddColumn("KLD Δ"); + table.AddColumn("Size Δ (GB)"); + table.AddColumn("Code"); + + foreach (var row in eliminations + .DistinctBy(x => $"{TensorConfigIdentity.ToKey(x.Eliminated.Config)}::{TensorConfigIdentity.ToKey(x.Eliminator.Config)}::{x.Reason}") + .OrderBy(x => x.Eliminated.Kld) + .ThenBy(x => x.Eliminated.SizeBytes) + .Take(25)) + { + double kldDelta = row.Eliminated.Kld - row.Eliminator.Kld; + double sizeDeltaGb = (row.Eliminated.SizeBytes - (double)row.Eliminator.SizeBytes) / 1000d / 1000d / 1000d; + string removed = _namingService.ToShortDisplayName(_namingService.BuildDisplayLabel(row.Eliminated, namingContext)); + string winner = _namingService.ToShortDisplayName(_namingService.BuildDisplayLabel(row.Eliminator, namingContext)); + string code = FinalArtifactNamingService.ReasonCode(row.Reason); + + table.AddRow( + Markup.Escape(removed), + Markup.Escape(winner), + kldDelta.ToString("0.000000"), + sizeDeltaGb.ToString("0.00"), + Markup.Escape(code)); + } + + AnsiConsole.Write(table); + + if (eliminations.Count > 25) + AnsiConsole.MarkupLine($"[grey]Showing first 25 of {eliminations.Count:N0} elimination records. Full details are in magicquant-manifest/magicquant.replacements.json.[/]"); + } + +} diff --git a/src/MagicQuant/Services/DuckDbPredictionMaterializationService.cs b/src/MagicQuant/Services/DuckDbPredictionMaterializationService.cs new file mode 100644 index 0000000..29fd5b1 --- /dev/null +++ b/src/MagicQuant/Services/DuckDbPredictionMaterializationService.cs @@ -0,0 +1,979 @@ +using System.Globalization; +using System.Numerics; +using DuckDB.NET.Data; +using MagicQuant.Helpers; +using MagicQuant.Models; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Services; + +/// +/// Materializes transient prediction/ranking metadata into DuckDB. +/// SQLite remains the long-term benchmark truth source; these columns only order +/// candidates before real quantization/benchmark validation. +/// +public sealed class DuckDbPredictionMaterializationService +{ + private readonly RemainingCombinationStore _store; + private readonly RankSafeKldPredictionService _predictionService; + + private static readonly GroupSlot[] GroupSlots = + [ + new(TReg.Embeddings, "Embeddings"), + new(TReg.LmHead, "LmHead"), + new(TReg.AttnQ, "AttnQ"), + new(TReg.AttnKV, "AttnKV"), + new(TReg.AttnOutput, "AttnOutput"), + new(TReg.FfnUpGate, "FfnUpGate"), + new(TReg.FfnDown, "FfnDown"), + new(TReg.MoeExperts, "MoeExperts"), + new(TReg.MoeRouter, "MoeRouter") + ]; + + public DuckDbPredictionMaterializationService( + RemainingCombinationStore store, + RankSafeKldPredictionService predictionService) + { + _store = store; + _predictionService = predictionService; + } + + public async Task MaterializeAsync(CancellationToken ct = default) + { + var model = await _predictionService.BuildModelAsync(ct); + + using var c = new DuckDBConnection($"Data Source={_store.GetDatabaseFilePath()}"); + await c.OpenAsync(ct); + await ConfigureSessionAsync(c, ct); + + PrintModelCoverageDiagnostics(model); + + await ExecuteAsync(c, $@" +UPDATE {CombinationDuckDbSchema.TableName} +SET PredictedKld = NULL, + PredictedSizeBytes = NULL, + PredictionConfidence = NULL, + PredictionRank = NULL, + BaseRankSafeKld = NULL, + AnomalyAdjustmentKld = 0.0, + FinalPredictedKld = NULL;", ct); + + await BuildLookupTablesAsync(c, model, ct); + await PrintLookupDiagnosticsAsync(c, model, ct); + await BuildPredictionWorkTablesAsync(c, model, ct); + await PrintPredictionWorkDiagnosticsAsync(c, ct); + await BuildPavaBlocksAsync(c, ct); + await PersistProjectedPredictionsAsync(c, model, ct); + + var status = await _store.GetPredictionStatusAsync(ct); + await PrintFinalMaterializationDiagnosticsAsync(c, status, ct); + + foreach (var note in model.Notes) + AnsiConsole.MarkupLine($"[grey]Prediction materialization note:[/] {Markup.Escape(note)}"); + + if (status.TotalRows > 0 && status.PredictedRows == 0) + { + throw new InvalidOperationException( + "Prediction materialization produced zero predicted rows. This is not a valid no-hybrid result. " + + "The diagnostics above should identify whether DuckDB BaseQuant IDs, base-only anchors, " + + "or group isolation/profile-scoped truth rows are missing."); + } + + return status; + } + + private static async Task BuildLookupTablesAsync( + DuckDBConnection c, + RankSafeKldPredictionService.RankSafePredictionModel model, + CancellationToken ct) + { + await ExecuteAsync(c, @" +DROP TABLE IF EXISTS temp_effective_group_prediction; +DROP TABLE IF EXISTS temp_base_predicted_size; +DROP TABLE IF EXISTS temp_group_size_delta; + +CREATE TEMP TABLE temp_effective_group_prediction ( + BaseQuant UTINYINT, + GroupName VARCHAR, + GroupId UTINYINT, + StoredSlot UTINYINT, + EffectiveBaselineId UTINYINT, + ResolvedBaselineId UTINYINT, + KldContribution DOUBLE, + PplContribution DOUBLE, + BitRange DOUBLE, + IsZeroDamage BOOLEAN, + IsKldPredictable BOOLEAN, + IsolationSource VARCHAR, + IsSurrogateFallback BOOLEAN +); + +CREATE TEMP TABLE temp_base_predicted_size ( + BaseQuant UTINYINT, + BaseSizeBytes UBIGINT, + IsSizePredictable BOOLEAN +); + +CREATE TEMP TABLE temp_group_size_delta ( + BaseQuant UTINYINT, + GroupName VARCHAR, + GroupId UTINYINT, + StoredSlot UTINYINT, + DeltaBytes BIGINT, + IsSizePredictable BOOLEAN +);", ct); + + var duckDbBaseQuantIds = await LoadDistinctBaseQuantIdsAsync(c, ct); + var runtimeBaseQuantIds = RuntimeSearchSpace.GetActiveCombinationBaselines() + .Select(x => x.UniqueId) + .OrderBy(x => x) + .ToList(); + + if (!duckDbBaseQuantIds.SequenceEqual(runtimeBaseQuantIds)) + { + AnsiConsole.MarkupLine( + $"[yellow]Prediction carrier mismatch:[/] DuckDB BaseQuant IDs=[cyan]{Markup.Escape(FormatBaselineIds(duckDbBaseQuantIds))}[/], " + + $"Runtime active IDs=[cyan]{Markup.Escape(FormatBaselineIds(runtimeBaseQuantIds))}[/]. " + + "Using DuckDB BaseQuant IDs as the scoring source of truth."); + } + + var activeBaselines = duckDbBaseQuantIds + .Select(BaselineQuants.FromId) + .OrderBy(x => x.UniqueId) + .ToList(); + + var activeGroups = model.ActiveGroups + .Select(g => GroupSlots.First(x => x.Group.UniqueId == g.UniqueId)) + .OrderBy(x => x.Group.UniqueId) + .ToList(); + + var warnedMissingQ8IsolationGroups = new HashSet(); + + using var tx = c.BeginTransaction(); + + foreach (var baseline in activeBaselines) + { + bool hasBaseSize = RankSafeKldPredictionService.TryResolveBaseOnlySnapshotForPrediction( + baseline.UniqueId, + model, + notes: null, + out var baseOnly); + + await ExecuteAsync(c, + $"INSERT INTO temp_base_predicted_size VALUES ({baseline.UniqueId}, {SqlULong(hasBaseSize ? baseOnly.SizeBytes : 0UL)}, {SqlBool(hasBaseSize)});", + ct); + + foreach (var slot in activeGroups) + { + var storedSlotsForGroup = await LoadDistinctStoredSlotsAsync(c, baseline.UniqueId, slot.ColumnName, ct); + foreach (byte storedSlot in storedSlotsForGroup) + { + var effectiveBaselineId = GetEffectiveBaselineId(baseline.UniqueId, storedSlot); + bool zeroDamage = IsZeroDamageAlias(effectiveBaselineId); + + byte resolvedBaselineId = effectiveBaselineId; + string isolationSource = zeroDamage ? "zero" : "missing"; + bool isSurrogateFallback = false; + BenchmarkSnapshotRecord? resolvedIsolation = null; + double kldContribution = 0d; + double pplContribution = 0d; + bool kldPredictable = true; + + if (!zeroDamage) + { + if (RankSafeKldPredictionService.TryResolveIsolationBaselineForPrediction( + slot.Group, + effectiveBaselineId, + model, + notes: null, + out var resolved)) + { + resolvedBaselineId = resolved.BaselineId; + resolvedIsolation = resolved.Snapshot; + isSurrogateFallback = resolved.IsSurrogate; + isolationSource = resolved.IsSurrogate + ? $"surrogate:{FormatBaselineId(resolved.FallbackBaselineId ?? resolved.BaselineId)}" + : "exact"; + kldContribution = Math.Max(0d, resolved.Snapshot.Kld); + pplContribution = resolved.Snapshot.Ppl; + } + else + { + kldPredictable = false; + if (effectiveBaselineId == BaselineQuants.Q8_0.UniqueId && warnedMissingQ8IsolationGroups.Add(slot.Group.UniqueId)) + { + AnsiConsole.MarkupLine($"[yellow]Missing KLD isolation snapshot for group '{Markup.Escape(slot.Group.Name)}' and baseline Q8_0 while building DuckDB prediction lookup. Q8_0 is quantized damage, not native truth; matching rows will stay unpredicted instead of receiving zero KLD.[/]"); + } + } + } + + double bitRange = RankSafeKldPredictionService.GetStressBitRangeForPrediction( + slot.Group, + resolvedBaselineId, + model); + + await ExecuteAsync(c, $@" +INSERT INTO temp_effective_group_prediction VALUES ( + {baseline.UniqueId}, + '{slot.ColumnName}', + {slot.Group.UniqueId}, + {storedSlot}, + {effectiveBaselineId}, + {resolvedBaselineId}, + {SqlDouble(kldContribution)}, + {SqlDouble(pplContribution)}, + {SqlDouble(bitRange)}, + {SqlBool(zeroDamage)}, + {SqlBool(kldPredictable)}, + {SqlString(isolationSource)}, + {SqlBool(isSurrogateFallback)} +);", ct); + + long deltaBytes = 0L; + bool sizePredictable = true; + + // This mirrors RankSafeKldPredictionService.PredictSize: + // base-only anchor starts with native-exact groups, then every active + // effective group contributes its measured exact isolation size delta. + // External/custom surrogate fallbacks are intentionally disabled by + // TryResolveIsolationBaselineForPrediction and will throw before this point. + if (!BaselineQuants.IsNativeExactAlias(effectiveBaselineId)) + { + if (resolvedIsolation != null) + { + deltaBytes = (long)resolvedIsolation.SizeBytes - (long)model.Q8BaseOnly.SizeBytes; + } + else + { + sizePredictable = false; + } + } + + await ExecuteAsync(c, $@" +INSERT INTO temp_group_size_delta VALUES ( + {baseline.UniqueId}, + '{slot.ColumnName}', + {slot.Group.UniqueId}, + {storedSlot}, + {deltaBytes.ToString(CultureInfo.InvariantCulture)}, + {SqlBool(sizePredictable)} +);", ct); + } + } + } + + tx.Commit(); + } + + private static async Task BuildPredictionWorkTablesAsync( + DuckDBConnection c, + RankSafeKldPredictionService.RankSafePredictionModel model, + CancellationToken ct) + { + var active = model.ActiveGroups + .Select(g => GroupSlots.First(x => x.Group.UniqueId == g.UniqueId)) + .OrderBy(x => x.Group.UniqueId) + .ToList(); + + string JoinEffective(GroupSlot slot, string alias) => + $"LEFT JOIN temp_effective_group_prediction {alias} ON {alias}.BaseQuant = t.BaseQuant AND {alias}.GroupName = '{slot.ColumnName}' AND {alias}.StoredSlot = t.{slot.ColumnName}"; + + string JoinDelta(GroupSlot slot, string alias) => + $"LEFT JOIN temp_group_size_delta {alias} ON {alias}.BaseQuant = t.BaseQuant AND {alias}.GroupName = '{slot.ColumnName}' AND {alias}.StoredSlot = t.{slot.ColumnName}"; + + string kldSum = active.Count == 0 + ? "0.0" + : string.Join(" + ", active.Select((_, i) => $"COALESCE(e{i}.KldContribution, 0.0)")); + + string sizeSum = active.Count == 0 + ? "0" + : string.Join(" + ", active.Select((_, i) => $"COALESCE(d{i}.DeltaBytes, 0)")); + + string kldPredictable = active.Count == 0 + ? "TRUE" + : string.Join(" AND ", active.Select((_, i) => $"COALESCE(e{i}.IsKldPredictable, FALSE)")); + + string sizePredictable = active.Count == 0 + ? "b.IsSizePredictable" + : "b.IsSizePredictable AND " + string.Join(" AND ", active.Select((_, i) => $"COALESCE(d{i}.IsSizePredictable, FALSE)")); + + string joins = string.Join(Environment.NewLine, active.Select((slot, i) => JoinEffective(slot, $"e{i}"))) + + Environment.NewLine + + string.Join(Environment.NewLine, active.Select((slot, i) => JoinDelta(slot, $"d{i}"))); + + await ExecuteAsync(c, $@" +DROP TABLE IF EXISTS temp_prediction_work; +CREATE TEMP TABLE temp_prediction_work AS +SELECT + CAST(ROW_NUMBER() OVER () AS UBIGINT) AS PredictionWorkId, + t.{CombinationDuckDbSchema.SlotColumnList.Replace(", ", ", t.")}, + ({kldSum})::DOUBLE AS AdditiveKld, + GREATEST(CAST(b.BaseSizeBytes AS BIGINT) + {sizeSum}, 0)::UBIGINT AS PredictedSizeBytesRaw, + ({kldPredictable})::BOOLEAN AS IsKldPredictable, + ({sizePredictable})::BOOLEAN AS IsSizePredictable +FROM {CombinationDuckDbSchema.TableName} t +JOIN temp_base_predicted_size b ON b.BaseQuant = t.BaseQuant +{joins};", ct); + + string contribUnions = string.Join(Environment.NewLine + "UNION ALL" + Environment.NewLine, + active.Select((slot, i) => $@" +SELECT + w.PredictionWorkId, + CAST({i} AS UTINYINT) AS GroupOrder, + e.KldContribution, + e.BitRange, + GREATEST(0.0, {SqlDouble(model.Fit.BitStressThreshold)} - e.BitRange) AS Stress +FROM temp_prediction_work w +JOIN temp_effective_group_prediction e + ON e.BaseQuant = w.BaseQuant + AND e.GroupName = '{slot.ColumnName}' + AND e.StoredSlot = w.{slot.ColumnName} +WHERE w.IsKldPredictable")); + + await ExecuteAsync(c, $@" +DROP TABLE IF EXISTS temp_work_group_contrib; +CREATE TEMP TABLE temp_work_group_contrib AS +{contribUnions};", ct); + + await ExecuteAsync(c, @" +DROP TABLE IF EXISTS temp_work_cross_term; +CREATE TEMP TABLE temp_work_cross_term AS +SELECT + a.PredictionWorkId, + SUM(a.KldContribution * b.KldContribution * a.Stress * b.Stress) AS CrossTerm +FROM temp_work_group_contrib a +JOIN temp_work_group_contrib b + ON a.PredictionWorkId = b.PredictionWorkId + AND a.GroupOrder < b.GroupOrder +GROUP BY a.PredictionWorkId;", ct); + + await ExecuteAsync(c, $@" +DROP TABLE IF EXISTS temp_projection; +CREATE TEMP TABLE temp_projection AS +SELECT + w.PredictionWorkId, + w.AdditiveKld, + GREATEST(0.0, ({SqlDouble(model.Fit.Alpha)} * w.AdditiveKld) + ({SqlDouble(model.Fit.Beta)} * COALESCE(x.CrossTerm, 0.0))) AS InteractionKld, + w.PredictedSizeBytesRaw AS PredictedSizeBytes, + COALESCE(x.CrossTerm, 0.0) AS CrossTerm +FROM temp_prediction_work w +LEFT JOIN temp_work_cross_term x ON x.PredictionWorkId = w.PredictionWorkId +WHERE w.IsKldPredictable + AND w.IsSizePredictable + AND w.PredictedSizeBytesRaw > 0;", ct); + + // PAVA is still the same rank-safe projection. It is just applied once + // over the DuckDB-ordered work table instead of over a giant C# object list. + await ExecuteAsync(c, @" +DROP TABLE IF EXISTS temp_prediction_order; +CREATE TEMP TABLE temp_prediction_order AS +SELECT + CAST(ROW_NUMBER() OVER ( + ORDER BY AdditiveKld ASC, + InteractionKld ASC, + PredictedSizeBytes ASC + ) AS UBIGINT) AS PredictionOrdinal, + PredictionWorkId, + AdditiveKld, + InteractionKld, + PredictedSizeBytes +FROM temp_projection;", ct); + } + + private static async Task BuildPavaBlocksAsync(DuckDBConnection c, CancellationToken ct) + { + await ExecuteAsync(c, @" +DROP TABLE IF EXISTS temp_pava_blocks; +CREATE TEMP TABLE temp_pava_blocks ( + StartOrdinal UBIGINT, + EndOrdinal UBIGINT, + ProjectedKld DOUBLE, + BlockCount UBIGINT, + MeanAdjustment DOUBLE +);", ct); + + var blocks = new List(); + + using (var cmd = c.CreateCommand()) + { + cmd.CommandText = @" +SELECT PredictionOrdinal, InteractionKld +FROM temp_prediction_order +ORDER BY PredictionOrdinal ASC;"; + + using var r = await cmd.ExecuteReaderAsync(ct); + while (await r.ReadAsync(ct)) + { + ct.ThrowIfCancellationRequested(); + + ulong ordinal = ToUInt64(r.GetValue(0)); + double value = Math.Max(0d, ToDouble(r.GetValue(1))); + + blocks.Add(new PavaBlock + { + StartOrdinal = ordinal, + EndOrdinal = ordinal, + Sum = value, + Count = 1 + }); + + while (blocks.Count >= 2 && blocks[^2].Mean > blocks[^1].Mean) + { + var right = blocks[^1]; + var left = blocks[^2]; + + left.EndOrdinal = right.EndOrdinal; + left.Sum += right.Sum; + left.Count += right.Count; + + blocks[^2] = left; + blocks.RemoveAt(blocks.Count - 1); + } + } + } + + if (blocks.Count == 0) + return; + + using var tx = c.BeginTransaction(); + using var insert = c.CreateCommand(); + insert.CommandText = "INSERT INTO temp_pava_blocks VALUES (?, ?, ?, ?, ?);"; + + foreach (var block in blocks) + { + insert.Parameters.Clear(); + double projected = Math.Max(0d, block.Mean); + insert.Parameters.Add(new DuckDBParameter { Value = block.StartOrdinal }); + insert.Parameters.Add(new DuckDBParameter { Value = block.EndOrdinal }); + insert.Parameters.Add(new DuckDBParameter { Value = projected }); + insert.Parameters.Add(new DuckDBParameter { Value = block.Count }); + insert.Parameters.Add(new DuckDBParameter { Value = Math.Abs(projected - block.Mean) }); + await insert.ExecuteNonQueryAsync(ct); + } + + tx.Commit(); + } + + private static async Task PersistProjectedPredictionsAsync( + DuckDBConnection c, + RankSafeKldPredictionService.RankSafePredictionModel model, + CancellationToken ct) + { + double baseConfidence = ComputeBaseConfidence(model.Fit); + + await ExecuteAsync(c, $@" +DROP TABLE IF EXISTS temp_projected_prediction; +CREATE TEMP TABLE temp_projected_prediction AS +SELECT + o.PredictionWorkId, + b.ProjectedKld AS PredictedKld, + o.InteractionKld, + o.PredictedSizeBytes, + b.BlockCount, + ABS(b.ProjectedKld - o.InteractionKld) / GREATEST(b.ProjectedKld, 1e-9) AS AdjustmentRatio, + GREATEST(0.25, 1.0 / SQRT(GREATEST(CAST(b.BlockCount AS DOUBLE), 1.0))) AS PlateauPenalty +FROM temp_prediction_order o +JOIN temp_pava_blocks b + ON o.PredictionOrdinal BETWEEN b.StartOrdinal AND b.EndOrdinal;", ct); + + await ExecuteAsync(c, $@" +DROP TABLE IF EXISTS temp_ranked_prediction; +CREATE TEMP TABLE temp_ranked_prediction AS +SELECT + w.{CombinationDuckDbSchema.SlotColumnList.Replace(", ", ", w.")}, + p.PredictedKld, + p.PredictedSizeBytes, + LEAST(1.0, GREATEST(0.0, + CASE WHEN NOT {CombinationDuckDbSchema.QualifyHybridPredicate("w")} + THEN 1.0 + ELSE {SqlDouble(baseConfidence)} * (1.0 / (1.0 + p.AdjustmentRatio)) * p.PlateauPenalty + END + )) AS PredictionConfidence +FROM temp_prediction_work w +JOIN temp_projected_prediction p ON p.PredictionWorkId = w.PredictionWorkId;", ct); + + await ExecuteAsync(c, $@" +DROP TABLE IF EXISTS temp_ranked_prediction_with_rank; +CREATE TEMP TABLE temp_ranked_prediction_with_rank AS +SELECT + *, + CAST(ROW_NUMBER() OVER ( + ORDER BY PredictedKld ASC, + PredictedSizeBytes ASC, + PredictionConfidence DESC, + BaseQuant ASC, + Embeddings ASC, + LmHead ASC, + AttnQ ASC, + AttnKV ASC, + AttnOutput ASC, + FfnUpGate ASC, + FfnDown ASC, + MoeExperts ASC, + MoeRouter ASC + ) AS UBIGINT) AS PredictionRank +FROM temp_ranked_prediction;", ct); + + await ExecuteAsync(c, $@" +UPDATE {CombinationDuckDbSchema.TableName} t +SET PredictedKld = r.PredictedKld, + PredictedSizeBytes = r.PredictedSizeBytes, + PredictionConfidence = r.PredictionConfidence, + PredictionRank = r.PredictionRank, + BaseRankSafeKld = r.PredictedKld, + AnomalyAdjustmentKld = 0.0, + FinalPredictedKld = r.PredictedKld +FROM temp_ranked_prediction_with_rank r +WHERE {CombinationDuckDbSchema.BuildSlotEqualityPredicate("t", "r")};", ct); + } + + private static async Task> LoadDistinctBaseQuantIdsAsync(DuckDBConnection c, CancellationToken ct) + { + var result = new List(); + using var cmd = c.CreateCommand(); + cmd.CommandText = $@" +SELECT DISTINCT BaseQuant +FROM {CombinationDuckDbSchema.TableName} +ORDER BY BaseQuant;"; + + using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + result.Add(ToByte(reader.GetValue(0))); + + return result; + } + + private static async Task> LoadDistinctStoredSlotsAsync( + DuckDBConnection c, + byte baseQuant, + string columnName, + CancellationToken ct) + { + var result = new List(); + using var cmd = c.CreateCommand(); + cmd.CommandText = $@" +SELECT DISTINCT {columnName} +FROM {CombinationDuckDbSchema.TableName} +WHERE BaseQuant = ? +ORDER BY {columnName};"; + cmd.Parameters.Add(new DuckDBParameter { Value = baseQuant }); + + using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + result.Add(ToByte(reader.GetValue(0))); + + return result; + } + + private static void PrintModelCoverageDiagnostics(RankSafeKldPredictionService.RankSafePredictionModel model) + { + string activeGroups = string.Join(", ", model.ActiveGroups.Select(x => $"{x.Name}:{x.UniqueId}")); + string baseOnly = FormatBaselineIds(model.BaseOnlySnapshotsByBaselineId.Keys.OrderBy(x => x).ToList()); + + AnsiConsole.MarkupLine($"[grey]Prediction model active groups:[/] {Markup.Escape(activeGroups)}"); + AnsiConsole.MarkupLine($"[grey]Prediction model base-only anchors:[/] [cyan]{model.BaseOnlySnapshotsByBaselineId.Count:N0}[/] ({Markup.Escape(baseOnly)})"); + AnsiConsole.MarkupLine($"[grey]Prediction model isolation anchors:[/] [cyan]{model.IsolationByGroupAndBaseline.Count:N0}[/]"); + + foreach (var group in model.ActiveGroups.OrderBy(x => x.UniqueId)) + { + var ids = model.IsolationByGroupAndBaseline.Keys + .Where(x => x.GroupId == group.UniqueId) + .Select(x => x.BaselineId) + .Distinct() + .OrderBy(x => x) + .ToList(); + + AnsiConsole.MarkupLine($"[grey] - isolation coverage {Markup.Escape(group.Name)}:[/] [cyan]{ids.Count:N0}[/] ({Markup.Escape(FormatBaselineIds(ids))})"); + } + } + + private static async Task PrintLookupDiagnosticsAsync( + DuckDBConnection c, + RankSafeKldPredictionService.RankSafePredictionModel model, + CancellationToken ct) + { + long totalRows = await ScalarLongAsync(c, $"SELECT COUNT(*) FROM {CombinationDuckDbSchema.TableName};", ct); + long baseLookupRows = await ScalarLongAsync(c, "SELECT COUNT(*) FROM temp_base_predicted_size;", ct); + long missingBaseJoin = await ScalarLongAsync(c, $@" +SELECT COUNT(*) +FROM {CombinationDuckDbSchema.TableName} t +LEFT JOIN temp_base_predicted_size b ON b.BaseQuant = t.BaseQuant +WHERE b.BaseQuant IS NULL;", ct); + + AnsiConsole.MarkupLine($"[grey]DuckDB prediction lookup rows:[/] total=[cyan]{totalRows:N0}[/] base-lookups=[cyan]{baseLookupRows:N0}[/] missing-base-join=[cyan]{missingBaseJoin:N0}[/]"); + + await PrintBaseLookupRowsAsync(c, ct); + await PrintIsolationSourceDiagnosticsAsync(c, ct); + await PrintMissingGroupLookupRowsAsync(c, model, ct); + } + + private static async Task PrintIsolationSourceDiagnosticsAsync(DuckDBConnection c, CancellationToken ct) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = @" +SELECT IsolationSource, COUNT(*) AS Rows +FROM temp_effective_group_prediction +GROUP BY IsolationSource +ORDER BY Rows DESC, IsolationSource;"; + + using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + string source = reader.GetValue(0)?.ToString() ?? "unknown"; + long rows = ToInt64(reader.GetValue(1)); + AnsiConsole.MarkupLine($"[grey] - isolation source {Markup.Escape(source)}:[/] rows={rows:N0}"); + } + } + + private static async Task PrintBaseLookupRowsAsync(DuckDBConnection c, CancellationToken ct) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = @" +SELECT BaseQuant, BaseSizeBytes, IsSizePredictable +FROM temp_base_predicted_size +ORDER BY BaseQuant;"; + + using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + byte baseId = ToByte(reader.GetValue(0)); + ulong bytes = ToUInt64(reader.GetValue(1)); + bool predictable = ToBool(reader.GetValue(2)); + AnsiConsole.MarkupLine($"[grey] - base lookup {Markup.Escape(FormatBaselineId(baseId))}:[/] size={bytes:N0} predictable={predictable}"); + } + } + + private static async Task PrintMissingGroupLookupRowsAsync( + DuckDBConnection c, + RankSafeKldPredictionService.RankSafePredictionModel model, + CancellationToken ct) + { + var active = model.ActiveGroups + .Select(g => GroupSlots.First(x => x.Group.UniqueId == g.UniqueId)) + .OrderBy(x => x.Group.UniqueId) + .ToList(); + + foreach (var slot in active) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = $@" +SELECT t.BaseQuant, t.{slot.ColumnName}, COUNT(*) AS MissingRows +FROM {CombinationDuckDbSchema.TableName} t +LEFT JOIN temp_effective_group_prediction e + ON e.BaseQuant = t.BaseQuant + AND e.GroupName = '{slot.ColumnName}' + AND e.StoredSlot = t.{slot.ColumnName} +WHERE e.BaseQuant IS NULL +GROUP BY t.BaseQuant, t.{slot.ColumnName} +ORDER BY MissingRows DESC +LIMIT 5;"; + + using var reader = await cmd.ExecuteReaderAsync(ct); + bool wroteHeader = false; + while (await reader.ReadAsync(ct)) + { + if (!wroteHeader) + { + AnsiConsole.MarkupLine($"[yellow]Missing effective lookup rows for group {Markup.Escape(slot.ColumnName)}:[/]"); + wroteHeader = true; + } + + byte baseId = ToByte(reader.GetValue(0)); + byte storedSlot = ToByte(reader.GetValue(1)); + long count = ToInt64(reader.GetValue(2)); + AnsiConsole.MarkupLine($"[yellow] - base={Markup.Escape(FormatBaselineId(baseId))} stored={Markup.Escape(FormatStoredSlot(storedSlot))} rows={count:N0}[/]"); + } + } + } + + private static async Task PrintPredictionWorkDiagnosticsAsync(DuckDBConnection c, CancellationToken ct) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = @" +SELECT + COUNT(*) AS WorkRows, + COALESCE(SUM(CASE WHEN IsKldPredictable THEN 1 ELSE 0 END), 0) AS KldPredictableRows, + COALESCE(SUM(CASE WHEN IsSizePredictable THEN 1 ELSE 0 END), 0) AS SizePredictableRows, + COALESCE(SUM(CASE WHEN IsKldPredictable AND IsSizePredictable THEN 1 ELSE 0 END), 0) AS ProjectableRows +FROM temp_prediction_work;"; + + using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + await reader.ReadAsync(ct); + long workRows = ToInt64(reader.GetValue(0)); + long kldRows = ToInt64(reader.GetValue(1)); + long sizeRows = ToInt64(reader.GetValue(2)); + long projectableRows = ToInt64(reader.GetValue(3)); + AnsiConsole.MarkupLine($"[grey]DuckDB prediction work rows:[/] work=[cyan]{workRows:N0}[/] kld-ok=[cyan]{kldRows:N0}[/] size-ok=[cyan]{sizeRows:N0}[/] projectable=[cyan]{projectableRows:N0}[/]"); + } + + await PrintPredictionFailureBreakdownAsync(c, ct); + + long projected = await ScalarLongAsync(c, "SELECT COUNT(*) FROM temp_projection;", ct); + long ordered = await ScalarLongAsync(c, "SELECT COUNT(*) FROM temp_prediction_order;", ct); + AnsiConsole.MarkupLine($"[grey]DuckDB projection rows:[/] projection=[cyan]{projected:N0}[/] ordered=[cyan]{ordered:N0}[/]"); + } + + private static async Task PrintPredictionFailureBreakdownAsync(DuckDBConnection c, CancellationToken ct) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = @" +SELECT + BaseQuant, + COUNT(*) AS Rows, + COALESCE(SUM(CASE WHEN NOT IsKldPredictable THEN 1 ELSE 0 END), 0) AS KldMissing, + COALESCE(SUM(CASE WHEN NOT IsSizePredictable THEN 1 ELSE 0 END), 0) AS SizeMissing +FROM temp_prediction_work +GROUP BY BaseQuant +ORDER BY BaseQuant;"; + + using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + byte baseId = ToByte(reader.GetValue(0)); + long rows = ToInt64(reader.GetValue(1)); + long kldMissing = ToInt64(reader.GetValue(2)); + long sizeMissing = ToInt64(reader.GetValue(3)); + AnsiConsole.MarkupLine($"[grey] - work {Markup.Escape(FormatBaselineId(baseId))}:[/] rows={rows:N0} missing-kld={kldMissing:N0} missing-size={sizeMissing:N0}"); + } + } + + private static async Task PrintFinalMaterializationDiagnosticsAsync( + DuckDBConnection c, + PredictionMaterializationStatus status, + CancellationToken ct) + { + long rankedRows = await ScalarLongAsync(c, "SELECT COUNT(*) FROM temp_ranked_prediction_with_rank;", ct); + AnsiConsole.MarkupLine($"[grey]DuckDB final materialized prediction rows:[/] predicted=[cyan]{status.PredictedRows:N0}[/] / {status.TotalRows:N0}, ranked-temp=[cyan]{rankedRows:N0}[/]"); + } + + private static async Task ScalarLongAsync(DuckDBConnection c, string sql, CancellationToken ct) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = sql; + return ToInt64(await cmd.ExecuteScalarAsync(ct) ?? 0L); + } + + private static string FormatBaselineIds(IReadOnlyCollection ids) + { + if (ids.Count == 0) + return "none"; + + return string.Join(", ", ids.Select(FormatBaselineId)); + } + + private static string FormatBaselineId(byte id) + { + try + { + var baseline = BaselineQuants.FromId(id); + return $"{baseline.Names[0]}:{id}"; + } + catch + { + return $"unknown:{id}"; + } + } + + private static string FormatStoredSlot(byte storedSlot) + { + if (BaselineQuants.IsNullTensorConfigGroupSlot(storedSlot)) + return $"base/null:{storedSlot}"; + + byte decoded = BaselineQuants.DecodeTensorConfigGroupSlotToBaselineId(storedSlot); + return $"{FormatBaselineId(decoded)} stored:{storedSlot}"; + } + + private static double ComputeBaseConfidence(RankSafePredictionFit fit) + { + if (fit.UsedFallback) + return 0.65d; + + double denom = Math.Max(Config.PredictionMinimumFitRows * 4.0d, 1.0d); + return Math.Clamp(fit.FitRowCount / denom, 0.35d, 1.0d); + } + + private static byte GetEffectiveBaselineId(byte baseQuant, byte storedSlot) + { + return BaselineQuants.IsNullTensorConfigGroupSlot(storedSlot) + ? baseQuant + : BaselineQuants.DecodeTensorConfigGroupSlotToBaselineId(storedSlot); + } + + private static bool IsZeroDamageAlias(byte baselineId) + { + // Only native exact aliases are zero-reference states. Q8_0 is intentionally + // excluded: it has measured isolation KLD and must be scored like every other + // quant baseline in prediction space. + return BaselineQuants.IsNativeExactAlias(baselineId); + } + + private static double GetBitRange(byte baselineId) + { + if (IsZeroDamageAlias(baselineId)) + return 99d; + + return BaselineQuants.FromId(baselineId).BitRange; + } + + private static long ToInt64(object? value) + { + if (value is null or DBNull) + return 0L; + + return value switch + { + long x => x, + int x => x, + short x => x, + sbyte x => x, + byte x => x, + uint x => checked((long)x), + ulong x => checked((long)x), + BigInteger x => checked((long)x), + decimal x => checked((long)x), + double x => checked((long)x), + float x => checked((long)x), + IConvertible x => x.ToInt64(CultureInfo.InvariantCulture), + _ => long.Parse(value.ToString() ?? "0", CultureInfo.InvariantCulture) + }; + } + + private static ulong ToUInt64(object? value) + { + if (value is null or DBNull) + return 0UL; + + return value switch + { + ulong x => x, + long x => checked((ulong)x), + int x => checked((ulong)x), + short x => checked((ulong)x), + sbyte x => checked((ulong)x), + byte x => x, + uint x => x, + BigInteger x => checked((ulong)x), + decimal x => checked((ulong)x), + double x => checked((ulong)x), + float x => checked((ulong)x), + IConvertible x => x.ToUInt64(CultureInfo.InvariantCulture), + _ => ulong.Parse(value.ToString() ?? "0", CultureInfo.InvariantCulture) + }; + } + + private static byte ToByte(object? value) + { + if (value is null or DBNull) + return 0; + + return value switch + { + byte x => x, + sbyte x => checked((byte)x), + short x => checked((byte)x), + int x => checked((byte)x), + long x => checked((byte)x), + ushort x => checked((byte)x), + uint x => checked((byte)x), + ulong x => checked((byte)x), + BigInteger x => checked((byte)x), + IConvertible x => x.ToByte(CultureInfo.InvariantCulture), + _ => byte.Parse(value.ToString() ?? "0", CultureInfo.InvariantCulture) + }; + } + + private static double ToDouble(object? value) + { + if (value is null or DBNull) + return 0d; + + return value switch + { + double x => x, + float x => x, + decimal x => (double)x, + BigInteger x => (double)x, + IConvertible x => x.ToDouble(CultureInfo.InvariantCulture), + _ => double.Parse(value.ToString() ?? "0", CultureInfo.InvariantCulture) + }; + } + + private static bool ToBool(object? value) + { + if (value is null or DBNull) + return false; + + return value switch + { + bool x => x, + byte x => x != 0, + sbyte x => x != 0, + short x => x != 0, + int x => x != 0, + long x => x != 0, + ushort x => x != 0, + uint x => x != 0, + ulong x => x != 0, + BigInteger x => x != BigInteger.Zero, + IConvertible x => x.ToBoolean(CultureInfo.InvariantCulture), + _ => bool.Parse(value.ToString() ?? "false") + }; + } + + private static async Task ConfigureSessionAsync(DuckDBConnection connection, CancellationToken ct) + { + await ExecuteAsync(connection, "SET preserve_insertion_order = false;", ct); + await ExecuteAsync(connection, $"SET threads = {Math.Max(1, Environment.ProcessorCount)};", ct); + } + + private static async Task ExecuteAsync(DuckDBConnection c, string sql, CancellationToken ct) + { + using var cmd = c.CreateCommand(); + cmd.CommandText = sql; + await cmd.ExecuteNonQueryAsync(ct); + } + + private static string SqlDouble(double value) + { + if (double.IsNaN(value) || double.IsInfinity(value)) + return "0.0"; + + return value.ToString("R", CultureInfo.InvariantCulture); + } + + private static string SqlString(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + return "NULL"; + + return $"'{value.Replace("'", "''")}'"; + } + + private static string SqlULong(ulong value) => value.ToString(CultureInfo.InvariantCulture); + private static string SqlBool(bool value) => value ? "TRUE" : "FALSE"; + + private readonly record struct GroupSlot(TensorGroup Group, string ColumnName); + + private struct PavaBlock + { + public ulong StartOrdinal; + public ulong EndOrdinal; + public double Sum; + public ulong Count; + public double Mean => Count == 0 ? 0d : Sum / Count; + } +} + +public sealed class PredictionMaterializationStatus +{ + public long TotalRows { get; init; } + public long PredictedRows { get; init; } + public long MissingPredictionRows { get; init; } + public long RankedRows { get; init; } + public double? MinPredictedKld { get; init; } + public double? MaxPredictedKld { get; init; } + public ulong? MinPredictedSizeBytes { get; init; } + public ulong? MaxPredictedSizeBytes { get; init; } +} diff --git a/src/MagicQuant/Services/EffectiveCandidateStateResolverService.cs b/src/MagicQuant/Services/EffectiveCandidateStateResolverService.cs new file mode 100644 index 0000000..1f027c8 --- /dev/null +++ b/src/MagicQuant/Services/EffectiveCandidateStateResolverService.cs @@ -0,0 +1,153 @@ +using MagicQuant.Helpers; +using System.Text; +using MagicQuant.Models; +using MQ.DB.Models; + +namespace MagicQuant.Services; + +public sealed class EffectiveCandidateStateResolverService +{ + private readonly HybridBenchmarkRepository _repository; + + public EffectiveCandidateStateResolverService(HybridBenchmarkRepository repository) + { + _repository = repository; + } + + public async Task ResolveAsync(TensorConfig config, CancellationToken ct = default) + { + return await ResolveAsync((HybridQuant)config, config, ct); + } + + public async Task ResolveAsync(HybridQuant quant, CancellationToken ct = default) + { + return await ResolveAsync(quant, (TensorConfig)quant, ct); + } + + private async Task ResolveAsync(HybridQuant quant, TensorConfig config, CancellationToken ct) + { + var warnings = new List(); + var groupStates = new Dictionary(StringComparer.Ordinal); + + string baseState = await ResolveBaseStateAsync(quant.BaseQuant, warnings, ct); + + foreach (var (group, storedValue) in TensorConfigIdentity.EnumerateGroupSlots(config)) + { + string state; + if (BaselineQuants.IsNullTensorConfigGroupSlot(storedValue)) + { + state = "base"; + } + else + { + byte baselineId = BaselineQuants.DecodeTensorConfigGroupSlotToBaselineId(storedValue); + + if (BaselineQuants.IsNativeExactAlias(baselineId)) + { + var exactScheme = BaselineQuants.ResolveExactOverrideScheme(baselineId); + state = $"exact:{exactScheme.Names[0]}"; + } + else + { + var baseline = BaselineQuants.FromId(baselineId); + state = await ResolveGroupStateAsync(baseline, group, warnings, ct); + } + } + + groupStates[group.Name] = state; + } + + var keyBuilder = new StringBuilder(); + keyBuilder.Append("base=").Append(baseState); + foreach (var kv in groupStates.OrderBy(x => x.Key, StringComparer.Ordinal)) + keyBuilder.Append('|').Append(kv.Key).Append('=').Append(kv.Value); + + return new EffectiveStateResolutionResult + { + Config = config, + EffectiveStateKey = keyBuilder.ToString(), + HasUnknownMappings = warnings.Count > 0, + Warnings = warnings, + GroupStates = groupStates, + BaseState = baseState + }; + } + + private async Task ResolveBaseStateAsync(BaselineQuants baseline, List warnings, CancellationToken ct) + { + if (!baseline.IsExternalRepositoryBaseline) + return baseline.CanonicalKey; + + var blanket = await _repository.LoadLearnedTensorMappingsAsync( + canonicalBaselineKey: baseline.CanonicalKey, + groupId: null, + preferredSourceScheme: baseline.DefaultTensorScheme, + allowDominantFallback: true, + ct: ct); + + if (blanket.Count == 0) + { + warnings.Add($"No learned blanket mapping found for external baseline '{baseline.Names[0]}'. Falling back to canonical key."); + return baseline.CanonicalKey; + } + + var payload = string.Join("|", blanket.OrderBy(x => x.Key, StringComparer.Ordinal).Select(x => $"{x.Key}={NormalizeOrPreserveRaw(x.Value, warnings)}")); + return $"{baseline.CanonicalKey}:{TensorConfigIdentity.StableHash(payload)}"; + } + + private async Task ResolveGroupStateAsync( + BaselineQuants baseline, + TensorGroup group, + List warnings, + CancellationToken ct) + { + var mappings = await _repository.LoadLearnedTensorMappingsAsync( + canonicalBaselineKey: baseline.CanonicalKey, + groupId: group.UniqueId, + preferredSourceScheme: baseline.DefaultTensorScheme, + allowDominantFallback: true, + ct: ct); + + if (mappings.Count == 0) + { + warnings.Add($"No learned mapping found for group '{group.Name}' baseline '{baseline.Names[0]}'. Falling back to requested baseline identity."); + return $"requested:{baseline.CanonicalKey}"; + } + + var normalized = mappings + .OrderBy(x => x.Key, StringComparer.Ordinal) + .Select(x => $"{x.Key}={NormalizeOrPreserveRaw(x.Value, warnings)}") + .ToList(); + + if (normalized.Select(x => x.Split('=')[1]).Distinct(StringComparer.Ordinal).Count() == 1) + return $"effective:{normalized[0].Split('=')[1]}"; + + return $"effective-map:{TensorConfigIdentity.StableHash(string.Join("|", normalized))}"; + } + + private static string NormalizeOrPreserveRaw(string raw, List warnings) + { + string normalizedRaw = (raw ?? string.Empty).Trim(); + if (string.IsNullOrWhiteSpace(normalizedRaw)) + { + warnings.Add("Encountered an empty learned tensor state and preserved it as unknown metadata."); + return "unknown:"; + } + + var resolved = TensorWeightScheme.All.FirstOrDefault(x => + x.Names.Any(n => string.Equals(n, normalizedRaw, StringComparison.OrdinalIgnoreCase))); + if (resolved != null) + return resolved.Names[0]; + + var nativeResolved = NativePrecisionNormalization.ResolveSchemeIdsForLearnedFinalQuantType(normalizedRaw); + if (nativeResolved.Count > 0) + { + var nativeScheme = TensorWeightScheme.All.FirstOrDefault(x => x.UniqueId == nativeResolved.First()); + if (nativeScheme != null) + return nativeScheme.Names[0]; + } + + warnings.Add($"Unknown or partially unmapped learned tensor state '{normalizedRaw}' was preserved as raw metadata."); + return $"unknown:{normalizedRaw}"; + } +} diff --git a/src/MagicQuant/Services/ExternalBaselineCacheCleanupService.cs b/src/MagicQuant/Services/ExternalBaselineCacheCleanupService.cs new file mode 100644 index 0000000..b735a94 --- /dev/null +++ b/src/MagicQuant/Services/ExternalBaselineCacheCleanupService.cs @@ -0,0 +1,89 @@ +using MagicQuant.Helpers; +using MQ.DB; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class ExternalBaselineCacheCleanupService +{ + public async Task CleanupStaleArtifactsAsync(CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(Cache.ExternalBaselineCacheDirectory)) + return false; + + if (string.IsNullOrWhiteSpace(Cache.ModelMagicQuantDirectory)) + throw new InvalidOperationException( + "ModelMagicQuantDirectory must be set before cleaning external baseline artifacts."); + + string cacheRoot = Path.GetFullPath(Cache.ExternalBaselineCacheDirectory); + string modelMagicQuantRoot = Path.GetFullPath(Cache.ModelMagicQuantDirectory); + ValidateCleanupRoot(cacheRoot, modelMagicQuantRoot); + + if (!Directory.Exists(cacheRoot)) + return false; + + bool preserveResumableDownloads = Config.Current.Baselines.CustomRepositories + .Any(x => x.Enabled && x.ResumeOrRetryDownloads); + + if (preserveResumableDownloads) + { + int removedTransientFiles = await CleanupTransientTopLevelFilesAsync(cacheRoot, ct); + AnsiConsole.MarkupLine( + $"[green]Preserved resumable external-baseline downloads:[/] {Markup.Escape(cacheRoot)} " + + $"[grey](removed transient files={removedTransientFiles:N0})[/]"); + return removedTransientFiles > 0; + } + + await HardDeleteHelper.DeleteDirectoryIfExistsAsync(cacheRoot, ct); + AnsiConsole.MarkupLine( + $"[green]Cleaned abandoned external-baseline artifacts:[/] {Markup.Escape(cacheRoot)}"); + return true; + } + + private static async Task CleanupTransientTopLevelFilesAsync(string cacheRoot, CancellationToken ct) + { + int removed = 0; + foreach (string file in Directory.EnumerateFiles(cacheRoot, "*", SearchOption.TopDirectoryOnly)) + { + ct.ThrowIfCancellationRequested(); + + string fileName = Path.GetFileName(file); + bool isCompletedGguf = fileName.EndsWith(".gguf", StringComparison.OrdinalIgnoreCase); + if (isCompletedGguf) + continue; + + await HardDeleteHelper.DeleteFileIfExistsAsync(file); + removed++; + } + + return removed; + } + + internal static void ValidateCleanupRoot(string cacheRoot, string modelMagicQuantRoot) + { + string fullCacheRoot = Path.GetFullPath(cacheRoot) + .TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + string fullModelRoot = Path.GetFullPath(modelMagicQuantRoot) + .TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + string relative = Path.GetRelativePath(fullModelRoot, fullCacheRoot); + + bool escapesModelRoot = Path.IsPathRooted(relative) || + relative.Equals("..", StringComparison.Ordinal) || + relative.StartsWith(".." + Path.DirectorySeparatorChar, StringComparison.Ordinal) || + relative.StartsWith(".." + Path.AltDirectorySeparatorChar, StringComparison.Ordinal); + + if (relative.Equals(".", StringComparison.Ordinal) || escapesModelRoot) + { + throw new InvalidOperationException( + $"Refusing to clean unsafe external baseline cache path '{fullCacheRoot}'. " + + $"It must be a child directory of '{fullModelRoot}'."); + } + + if (Directory.Exists(fullCacheRoot) && + (File.GetAttributes(fullCacheRoot) & FileAttributes.ReparsePoint) != 0) + { + throw new InvalidOperationException( + $"Refusing to recursively clean external baseline cache symlink/reparse point '{fullCacheRoot}'."); + } + } +} diff --git a/src/MagicQuant/Services/ExternalBaselineTensorParity.cs b/src/MagicQuant/Services/ExternalBaselineTensorParity.cs new file mode 100644 index 0000000..3459cd2 --- /dev/null +++ b/src/MagicQuant/Services/ExternalBaselineTensorParity.cs @@ -0,0 +1,147 @@ +namespace MagicQuant.Services; + +internal sealed class GgufTensorReadResult +{ + public string? Error { get; set; } + public string? Architecture { get; set; } + public int? BlockCount { get; set; } + public int? NextnPredictLayers { get; set; } + public List TensorNames { get; set; } = new(); + public Dictionary TensorTypes { get; set; } = new(StringComparer.Ordinal); +} + +internal sealed class ExternalBaselineTensorParityResult +{ + public required GgufTensorReadResult NativeMetadata { get; init; } + public required GgufTensorReadResult ExternalMetadata { get; init; } + public IReadOnlyList InheritedOptionalTensorNames { get; init; } = []; + public int OmittedNextnLayerCount { get; init; } +} + +internal static class ExternalBaselineTensorParity +{ + public static ExternalBaselineTensorParityResult ValidateOrThrow( + GgufTensorReadResult nativeMetadata, + GgufTensorReadResult externalMetadata) + { + ArgumentNullException.ThrowIfNull(nativeMetadata); + ArgumentNullException.ThrowIfNull(externalMetadata); + + var nativeNames = nativeMetadata.TensorNames + .OrderBy(x => x, StringComparer.Ordinal) + .ToList(); + var externalNames = externalMetadata.TensorNames + .OrderBy(x => x, StringComparer.Ordinal) + .ToList(); + + var missing = nativeNames + .Except(externalNames, StringComparer.Ordinal) + .OrderBy(x => x, StringComparer.Ordinal) + .ToList(); + var unexpected = externalNames + .Except(nativeNames, StringComparer.Ordinal) + .OrderBy(x => x, StringComparer.Ordinal) + .ToList(); + + if (missing.Count == 0 && unexpected.Count == 0 && nativeNames.Count == externalNames.Count) + { + return new ExternalBaselineTensorParityResult + { + NativeMetadata = nativeMetadata, + ExternalMetadata = externalMetadata + }; + } + + if (unexpected.Count == 0 && + TryValidateDeclaredOptionalNextnOmission( + nativeMetadata, + externalMetadata, + missing, + out int omittedLayerCount)) + { + return new ExternalBaselineTensorParityResult + { + NativeMetadata = nativeMetadata, + ExternalMetadata = externalMetadata, + InheritedOptionalTensorNames = missing, + OmittedNextnLayerCount = omittedLayerCount + }; + } + + throw new InvalidOperationException( + $"External/custom baseline tensor mismatch detected. " + + $"Missing=[{string.Join(", ", missing.Take(20))}] " + + $"Unexpected=[{string.Join(", ", unexpected.Take(20))}]. " + + "MagicQuant only permits missing tensors when GGUF metadata explicitly declares fewer trailing NextN/MTP layers; " + + "all model-trunk tensors must exactly match the source model."); + } + + private static bool TryValidateDeclaredOptionalNextnOmission( + GgufTensorReadResult nativeMetadata, + GgufTensorReadResult externalMetadata, + IReadOnlyCollection missing, + out int omittedLayerCount) + { + omittedLayerCount = 0; + if (missing.Count == 0 || + string.IsNullOrWhiteSpace(nativeMetadata.Architecture) || + !string.Equals(nativeMetadata.Architecture, externalMetadata.Architecture, + StringComparison.Ordinal) || + nativeMetadata.BlockCount is not > 0 || + externalMetadata.BlockCount is not > 0 || + nativeMetadata.NextnPredictLayers is not >= 0 || + externalMetadata.NextnPredictLayers is not >= 0) + { + return false; + } + + int nativeBlockCount = nativeMetadata.BlockCount.Value; + int externalBlockCount = externalMetadata.BlockCount.Value; + int nativeNextnLayers = nativeMetadata.NextnPredictLayers.Value; + int externalNextnLayers = externalMetadata.NextnPredictLayers.Value; + int nativeTrunkBlockCount = nativeBlockCount - nativeNextnLayers; + int externalTrunkBlockCount = externalBlockCount - externalNextnLayers; + int omittedNextnLayers = nativeNextnLayers - externalNextnLayers; + + // Qwen GGUF block_count includes its trailing NextN layers. Therefore 65/1 and + // 64/0 describe the same 64-block model trunk. Both the trunk size and the exact + // block-count reduction must agree with the declared NextN reduction. + if (omittedNextnLayers <= 0 || + nativeTrunkBlockCount <= 0 || + externalTrunkBlockCount != nativeTrunkBlockCount || + nativeBlockCount - externalBlockCount != omittedNextnLayers) + { + return false; + } + + int firstOmittedBlock = externalBlockCount; + int endExclusive = nativeBlockCount; + + bool IsInOmittedRange(string tensorName) => + TryGetBlockIndex(tensorName, out int blockIndex) && + blockIndex >= firstOmittedBlock && + blockIndex < endExclusive; + + // A file declaring fewer NextN layers must omit those layers completely. A partial + // block is still malformed and must not be accepted as an optional-layer omission. + if (externalMetadata.TensorNames.Any(IsInOmittedRange) || missing.Any(x => !IsInOmittedRange(x))) + return false; + + omittedLayerCount = omittedNextnLayers; + return true; + } + + private static bool TryGetBlockIndex(string tensorName, out int blockIndex) + { + blockIndex = default; + const string prefix = "blk."; + if (!tensorName.StartsWith(prefix, StringComparison.Ordinal)) + return false; + + int separator = tensorName.IndexOf('.', prefix.Length); + if (separator <= prefix.Length) + return false; + + return int.TryParse(tensorName.AsSpan(prefix.Length, separator - prefix.Length), out blockIndex); + } +} diff --git a/src/MagicQuant/Services/FinalArtifactNamingService.cs b/src/MagicQuant/Services/FinalArtifactNamingService.cs new file mode 100644 index 0000000..255ccde --- /dev/null +++ b/src/MagicQuant/Services/FinalArtifactNamingService.cs @@ -0,0 +1,517 @@ +using System.Text.RegularExpressions; +using MagicQuant.Models; +using MQ.DB.Models; + +namespace MagicQuant.Services; + +/// +/// Centralizes the public naming rules used by exported GGUF files, README rows, +/// CLI previews, links, and diagnostic logs. Internal tensor-combo display names stay internal. +/// +public sealed class FinalArtifactNamingService +{ + private static readonly Regex UnsafeFileChars = new(@"[^A-Za-z0-9._-]+", RegexOptions.Compiled); + + public FinalArtifactNamingContext CreateContext( + IReadOnlyCollection pureBaselineSnapshots) + { + var ranges = BuildRanges(pureBaselineSnapshots); + return new FinalArtifactNamingContext(ranges); + } + + public FinalArtifactName BuildName( + BenchmarkSnapshotRecord snapshot, + FinalArtifactNamingContext context, + ISet? reservedFileNames = null) + { + string prefix = ResolveModelPrefix(); + + string tag; + string providerToken; + string quantFamily; + + if (snapshot.IsHybrid) + { + providerToken = "MQ"; + quantFamily = ResolveHybridRangeFamily(snapshot, context); + int ordinal = context.NextHybridOrdinal(quantFamily); + tag = $"{providerToken}-{SanitizeToken(quantFamily)}_{ordinal}"; + } + else if (HybridBenchmarkRepository.ResolveSourceBaselineForProvider(snapshot.Quant).IsExternalRepositoryBaseline) + { + var sourceBaseline = HybridBenchmarkRepository.ResolveSourceBaselineForProvider(snapshot.Quant); + string externalProviderToken = ResolveExternalProviderToken(sourceBaseline); + string externalFamily = NormalizeExternalDisplayName(sourceBaseline.Names[0], externalProviderToken); + + providerToken = externalProviderToken; + // Rebuilt/materialized external baselines are still external baselines. + // MagicQuant may rebuild/benchmark/export the file, but the public name must + // not imply MagicQuant invented the quant recipe (no MQ-UD-* labels). + quantFamily = SanitizeToken(externalFamily); + tag = quantFamily; + } + else + { + providerToken = "LM"; + quantFamily = snapshot.Quant.BaseQuant.Names[0]; + tag = $"{providerToken}-{SanitizeToken(quantFamily)}"; + } + + string stem = $"{prefix}-{tag}"; + string fileName = MakeUniqueFileName($"{stem}.gguf", reservedFileNames); + + return new FinalArtifactName + { + FileName = fileName, + DisplayName = Path.GetFileNameWithoutExtension(fileName), + ShortDisplayName = ToShortDisplayName(Path.GetFileNameWithoutExtension(fileName)), + ProviderToken = providerToken, + QuantFamilyOrBaseline = quantFamily + }; + } + + public string BuildDisplayLabel( + BenchmarkSnapshotRecord snapshot, + FinalArtifactNamingContext context) + { + string prefix = ResolveModelPrefix(); + + if (snapshot.IsHybrid) + return $"{prefix}-MQ-{SanitizeToken(ResolveHybridRangeFamily(snapshot, context))}"; + + var sourceBaseline = HybridBenchmarkRepository.ResolveSourceBaselineForProvider(snapshot.Quant); + if (sourceBaseline.IsExternalRepositoryBaseline) + { + string providerToken = ResolveExternalProviderToken(sourceBaseline); + string family = SanitizeToken(NormalizeExternalDisplayName(sourceBaseline.Names[0], providerToken)); + return $"{prefix}-{family}"; + } + + return $"{prefix}-LM-{SanitizeToken(snapshot.Quant.BaseQuant.Names[0])}"; + } + + public string ToShortDisplayName(string displayNameOrFileName) + { + if (string.IsNullOrWhiteSpace(displayNameOrFileName)) + return string.Empty; + + string value = displayNameOrFileName.Trim(); + + // Only strip directories. Do NOT blindly call GetFileNameWithoutExtension on + // extensionless display names like Qwen3.6-35B-A3B-LM-Q8_0, because .NET will + // treat ".6-35B-A3B-LM-Q8_0" as the extension and return only "Qwen3". + value = Path.GetFileName(value); + + // Only remove the extension when it is a real GGUF artifact filename. + if (value.EndsWith(".gguf", StringComparison.OrdinalIgnoreCase)) + value = value[..^".gguf".Length]; + + string prefix = ResolveModelPrefix(); + string fullPrefix = prefix + "-"; + + if (value.StartsWith(fullPrefix, StringComparison.OrdinalIgnoreCase)) + return value[fullPrefix.Length..]; + + return value; + } + + public string ToPublicArtifactShortName( + string? displayNameOrFileName, + string? fileName = null, + string? providerName = null, + string? quantFamily = null, + BenchmarkSnapshotRecord? snapshot = null, + FinalArtifactNamingContext? context = null) + { + string prefix = ResolveModelPrefix(); + string? preferred = !string.IsNullOrWhiteSpace(fileName) ? fileName : displayNameOrFileName; + + if (!string.IsNullOrWhiteSpace(preferred)) + { + string candidate = Path.GetFileNameWithoutExtension(preferred.Trim()); + string fullPrefix = prefix + "-"; + + if (candidate.StartsWith(fullPrefix, StringComparison.OrdinalIgnoreCase)) + candidate = candidate[fullPrefix.Length..]; + + if (!LooksLikeModelOnlyLabel(candidate, prefix)) + return candidate; + } + + return BuildProviderQuantFallback(fileName, providerName, quantFamily, snapshot, context); + } + + public IReadOnlyList BuildProviderCredits( + IReadOnlyCollection artifacts) + { + var result = new Dictionary(StringComparer.OrdinalIgnoreCase); + + result["llama.cpp"] = new ProviderCredit + { + Name = "llama.cpp", + Url = "https://github.com/ggml-org/llama.cpp", + Note = "Baseline quantization formats and llama.cpp tooling." + }; + + foreach (var artifact in artifacts) + { + foreach (var baseline in EnumerateBaselinesUsedBy(artifact.Snapshot.Quant)) + { + if (!baseline.IsExternalRepositoryBaseline) + continue; + + string providerName = string.IsNullOrWhiteSpace(baseline.ShortSourceName) + ? "External provider" + : baseline.ShortSourceName!; + + string providerToken = ResolveExternalProviderToken(baseline); + if (string.Equals(providerName, "Unsloth", StringComparison.OrdinalIgnoreCase)) + providerName = "Unsloth"; + + string? url = HybridBenchmarkRepository.BuildExternalRepositoryUrl(baseline); + if (string.IsNullOrWhiteSpace(url) && !string.IsNullOrWhiteSpace(baseline.SourceRepository)) + url = $"https://huggingface.co/{baseline.SourceRepository}"; + + string key = !string.IsNullOrWhiteSpace(url) ? url : providerName; + result[key] = new ProviderCredit + { + Name = providerName, + Url = url ?? string.Empty, + Note = $"External learned baseline source ({providerToken})." + }; + } + } + + return result.Values + .OrderBy(x => x.Name, StringComparer.OrdinalIgnoreCase) + .ThenBy(x => x.Url, StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + public static string ReasonCode(string reason) + { + if (reason.Contains("strict", StringComparison.OrdinalIgnoreCase)) + return "STRICT_DOMINANCE"; + if (reason.Contains("near-baseline", StringComparison.OrdinalIgnoreCase) || + reason.Contains("size premium", StringComparison.OrdinalIgnoreCase)) + return "NEAR_BASELINE_PREMIUM"; + if (reason.Contains("interior", StringComparison.OrdinalIgnoreCase)) + return "INTERIOR_DISCOVERY"; + if (reason.Contains("spacing", StringComparison.OrdinalIgnoreCase) || + reason.Contains("collapse", StringComparison.OrdinalIgnoreCase)) + return "SPACING_COLLAPSE"; + if (reason.Contains("dominance", StringComparison.OrdinalIgnoreCase)) + return "FINAL_DOMINANCE"; + + return "VALIDATED_REPLACEMENT"; + } + + public static string ReasonDescription(string code) + { + return code switch + { + "STRICT_DOMINANCE" => "The winner was no larger and had lower real KLD than the removed anchor.", + "NEAR_BASELINE_PREMIUM" => "The winner used only the configured near-baseline size premium and beat the real linear KLD trade line.", + "INTERIOR_DISCOVERY" => "The winner was selected as a useful interior point inside a size/KLD gap between anchors.", + "SPACING_COLLAPSE" => "Two candidates were too close in practical output space; the stronger one was kept.", + "FINAL_DOMINANCE" => "A later validated survivor dominated this artifact in final real benchmark comparison.", + _ => "A validated survivor replaced or made this artifact redundant." + }; + } + + private static IEnumerable EnumerateBaselinesUsedBy(HybridQuant quant) + { + yield return quant.BaseQuant; + + foreach (var tensor in quant.Tensors) + { + if (tensor.OverrideMode == HybridTensorOverrideMode.LearnedBaselineCandidate && tensor.CandidateBaseline != null) + yield return tensor.CandidateBaseline; + } + } + + private static IReadOnlyList BuildRanges(IReadOnlyCollection pureBaselineSnapshots) + { + var anchors = pureBaselineSnapshots + .Where(x => !x.IsHybrid) + .OrderBy(x => x.Kld) + .ThenBy(x => x.SizeBytes) + .ToList(); + + var ranges = new List(); + + for (int i = 0; i < anchors.Count - 1; i++) + { + var lowerDamageLarger = anchors[i]; + var higherDamageSmaller = anchors[i + 1]; + + if (higherDamageSmaller.SizeBytes >= lowerDamageLarger.SizeBytes) + continue; + + ranges.Add(new FinalArtifactRange + { + MinSizeBytes = higherDamageSmaller.SizeBytes, + MaxSizeBytes = lowerDamageLarger.SizeBytes, + RangeFamily = NormalizeRangeFamily(higherDamageSmaller.Quant.BaseQuant), + SmallerHigherDamageAnchor = higherDamageSmaller, + LargerLowerDamageAnchor = lowerDamageLarger + }); + } + + return ranges + .OrderBy(x => x.MinSizeBytes) + .ThenBy(x => x.MaxSizeBytes) + .ToList(); + } + + private static string ResolveHybridRangeFamily( + BenchmarkSnapshotRecord snapshot, + FinalArtifactNamingContext context) + { + var match = context.Ranges + .Where(x => snapshot.SizeBytes >= x.MinSizeBytes && snapshot.SizeBytes <= x.MaxSizeBytes) + .OrderBy(x => x.MaxSizeBytes - x.MinSizeBytes) + .FirstOrDefault(); + + if (match != null) + return match.RangeFamily; + + var nearestSmallerAnchor = context.Ranges + .Select(x => x.SmallerHigherDamageAnchor) + .OrderBy(x => Distance(snapshot.SizeBytes, x.SizeBytes)) + .FirstOrDefault(); + + if (nearestSmallerAnchor != null) + return NormalizeRangeFamily(nearestSmallerAnchor.Quant.BaseQuant); + + return NormalizeRangeFamily(snapshot.Quant.BaseQuant); + } + + private static string NormalizeRangeFamily(BaselineQuants baseline) + { + if (!string.IsNullOrWhiteSpace(baseline.QuantizeBaseArgumentName)) + return baseline.QuantizeBaseArgumentName; + + return !baseline.Names.IsDefaultOrEmpty ? baseline.Names[0] : "Unknown"; + } + + private static string ResolveExternalProviderToken(BaselineQuants baseline) + { + string joined = $"{baseline.ShortSourceName} {baseline.SourceOwner} {baseline.SourceRepository} {baseline.Names[0]}"; + if (joined.Contains("unsloth", StringComparison.OrdinalIgnoreCase)) + return "UD"; + + if (!string.IsNullOrWhiteSpace(baseline.ShortSourceName)) + return SanitizeToken(baseline.ShortSourceName!); + + return "EXT"; + } + + private static string NormalizeExternalDisplayName(string displayName, string providerToken) + { + if (string.IsNullOrWhiteSpace(displayName)) + return providerToken; + + var value = displayName.Trim(); + + if (value.StartsWith("Unsloth_", StringComparison.OrdinalIgnoreCase)) + value = providerToken + value["Unsloth".Length..]; + + if (!value.StartsWith(providerToken + "_", StringComparison.OrdinalIgnoreCase) && + !value.StartsWith(providerToken + "-", StringComparison.OrdinalIgnoreCase)) + { + value = $"{providerToken}-{value}"; + } + + value = SanitizeToken(value); + if (value.StartsWith(providerToken + "_", StringComparison.OrdinalIgnoreCase)) + value = providerToken + "-" + value[(providerToken.Length + 1)..]; + + return value; + } + + private static string MakeUniqueFileName(string desiredFileName, ISet? reservedFileNames) + { + if (reservedFileNames == null) + return desiredFileName; + + string candidate = desiredFileName; + string stem = Path.GetFileNameWithoutExtension(desiredFileName); + string ext = Path.GetExtension(desiredFileName); + int i = 1; + + while (!reservedFileNames.Add(candidate)) + { + i++; + candidate = $"{stem}_{i}{ext}"; + } + + return candidate; + } + + private static string ResolveModelPrefix() + { + string prefix = SanitizeToken(Config.OutputNamePrefix); + return string.IsNullOrWhiteSpace(prefix) ? "Model" : prefix; + } + + public static string SanitizeToken(string value) + { + if (string.IsNullOrWhiteSpace(value)) + return string.Empty; + + value = value.Trim().Replace(' ', '_'); + value = UnsafeFileChars.Replace(value, "_"); + while (value.Contains("__", StringComparison.Ordinal)) + value = value.Replace("__", "_", StringComparison.Ordinal); + return value.Trim('_', '-'); + } + + private static ulong Distance(ulong left, ulong right) => left >= right ? left - right : right - left; + + private static bool LooksLikeModelOnlyLabel(string value, string modelPrefix) + { + if (string.IsNullOrWhiteSpace(value)) + return true; + + string normalized = value.Trim(); + if (string.Equals(normalized, modelPrefix, StringComparison.OrdinalIgnoreCase)) + return true; + + string alnum = new(normalized.Where(char.IsLetterOrDigit).ToArray()); + if (string.IsNullOrWhiteSpace(alnum)) + return true; + + return Regex.IsMatch(alnum, @"^[A-Za-z]+\d*(?:\d)?$", RegexOptions.IgnoreCase); + } + + private string BuildProviderQuantFallback( + string? fileName, + string? providerName, + string? quantFamily, + BenchmarkSnapshotRecord? snapshot, + FinalArtifactNamingContext? context) + { + string resolvedProvider = providerName ?? (snapshot != null + ? (snapshot.IsHybrid ? "MagicQuant" : HybridBenchmarkRepository.ResolveProviderName(snapshot.Quant, exportNaming: false)) + : string.Empty); + + string? resolvedFamily = quantFamily; + if (string.IsNullOrWhiteSpace(resolvedFamily) && snapshot != null) + resolvedFamily = snapshot.BaselineFamily; + if (string.IsNullOrWhiteSpace(resolvedFamily) && snapshot != null && context != null) + resolvedFamily = ResolveHybridRangeFamily(snapshot, context); + + string sanitizedFamily = SanitizeToken(resolvedFamily ?? string.Empty); + if (string.IsNullOrWhiteSpace(sanitizedFamily)) + return string.Empty; + + if (sanitizedFamily.StartsWith("MQ-", StringComparison.OrdinalIgnoreCase)) + { + if (snapshot?.IsExternalRebuiltBaseline == true || snapshot?.IsExternalPureBaseline == true || + string.Equals(resolvedProvider, "Unsloth", StringComparison.OrdinalIgnoreCase)) + return StripMagicQuantExternalPrefix(sanitizedFamily); + + return sanitizedFamily; + } + + if (snapshot?.IsHybrid == true) + { + string ordinal = ExtractOrdinalFromFileName(fileName); + string baseName = sanitizedFamily.StartsWith("MQ-", StringComparison.OrdinalIgnoreCase) + ? sanitizedFamily + : $"MQ-{sanitizedFamily}"; + return string.IsNullOrWhiteSpace(ordinal) ? baseName : $"{baseName}_{ordinal}"; + } + + if (string.Equals(resolvedProvider, "MagicQuant", StringComparison.OrdinalIgnoreCase)) + { + if (snapshot?.IsExternalRebuiltBaseline == true || snapshot?.IsExternalPureBaseline == true) + return StripMagicQuantExternalPrefix(sanitizedFamily); + + return sanitizedFamily.StartsWith("MQ-", StringComparison.OrdinalIgnoreCase) ? sanitizedFamily : $"MQ-{sanitizedFamily}"; + } + + if (string.Equals(resolvedProvider, "llama.cpp", StringComparison.OrdinalIgnoreCase)) + return sanitizedFamily.StartsWith("LM-", StringComparison.OrdinalIgnoreCase) ? sanitizedFamily : $"LM-{sanitizedFamily}"; + + if (string.Equals(resolvedProvider, "Unsloth", StringComparison.OrdinalIgnoreCase)) + { + if (sanitizedFamily.StartsWith("UD_", StringComparison.OrdinalIgnoreCase)) + return "UD-" + sanitizedFamily[3..]; + + if (sanitizedFamily.StartsWith("UD-", StringComparison.OrdinalIgnoreCase) || + sanitizedFamily.StartsWith("Unsloth", StringComparison.OrdinalIgnoreCase)) + return sanitizedFamily; + + return $"UD-{sanitizedFamily}"; + } + + return sanitizedFamily; + } + + private static string StripMagicQuantExternalPrefix(string value) + { + if (value.StartsWith("MQ-UD-", StringComparison.OrdinalIgnoreCase)) + return value[3..]; + if (value.StartsWith("MQ-Unsloth", StringComparison.OrdinalIgnoreCase)) + return value[3..]; + return value; + } + + private static string ExtractOrdinalFromFileName(string? fileName) + { + if (string.IsNullOrWhiteSpace(fileName)) + return string.Empty; + + string stem = Path.GetFileNameWithoutExtension(fileName.Trim()); + var match = Regex.Match(stem, @"_(\d+)$", RegexOptions.CultureInvariant); + return match.Success ? match.Groups[1].Value : string.Empty; + } +} + +public sealed class FinalArtifactNamingContext +{ + private readonly Dictionary _hybridOrdinalByFamily = new(StringComparer.OrdinalIgnoreCase); + + public FinalArtifactNamingContext(IReadOnlyList ranges) + { + Ranges = ranges; + } + + public IReadOnlyList Ranges { get; } + + public int NextHybridOrdinal(string rangeFamily) + { + string key = string.IsNullOrWhiteSpace(rangeFamily) ? "Unknown" : rangeFamily; + _hybridOrdinalByFamily.TryGetValue(key, out int current); + current++; + _hybridOrdinalByFamily[key] = current; + return current; + } +} + +public sealed class FinalArtifactRange +{ + public ulong MinSizeBytes { get; init; } + public ulong MaxSizeBytes { get; init; } + public string RangeFamily { get; init; } = string.Empty; + public BenchmarkSnapshotRecord SmallerHigherDamageAnchor { get; init; } = default!; + public BenchmarkSnapshotRecord LargerLowerDamageAnchor { get; init; } = default!; +} + +public sealed class FinalArtifactName +{ + public string FileName { get; init; } = string.Empty; + public string DisplayName { get; init; } = string.Empty; + public string ShortDisplayName { get; init; } = string.Empty; + public string ProviderToken { get; init; } = string.Empty; + public string QuantFamilyOrBaseline { get; init; } = string.Empty; +} + +public sealed class ProviderCredit +{ + public string Name { get; init; } = string.Empty; + public string Url { get; init; } = string.Empty; + public string Note { get; init; } = string.Empty; +} diff --git a/src/MagicQuant/Services/FinalRealBenchmarkEliminationService.cs b/src/MagicQuant/Services/FinalRealBenchmarkEliminationService.cs new file mode 100644 index 0000000..6042547 --- /dev/null +++ b/src/MagicQuant/Services/FinalRealBenchmarkEliminationService.cs @@ -0,0 +1,129 @@ +using MagicQuant.Helpers; +using MagicQuant.Models; +using MQ.DB.Models; + +namespace MagicQuant.Services; + +public sealed class FinalRealBenchmarkEliminationService +{ + public FinalRealEliminationResult Eliminate(IReadOnlyCollection snapshots) + { + var uniqueByConfig = snapshots + .DistinctBy(x => TensorConfigIdentity.ToKey(x.Config)) + .OrderBy(x => x.Kld) + .ThenBy(x => x.SizeBytes) + .ThenBy(x => x.Ppl) + .ThenBy(x => x.DisplayName, StringComparer.Ordinal) + .ToList(); + + var eliminated = new List(); + var collapsed = CollapseEquivalentTruths(uniqueByConfig, eliminated); + + var survivors = new List(); + + for (int i = 0; i < collapsed.Count; i++) + { + var current = collapsed[i]; + bool dominated = collapsed + .Where((_, index) => index != i) + .Any(other => Dominates(other, current)); + + if (dominated) + eliminated.Add(current); + else + survivors.Add(current); + } + + return new FinalRealEliminationResult + { + Survivors = survivors + .DistinctBy(x => TensorConfigIdentity.ToKey(x.Config)) + .OrderBy(x => x.Kld) + .ThenBy(x => x.SizeBytes) + .ThenBy(x => x.Ppl) + .ThenByDescending(x => x.Quant.BaseQuant.BitRange) + .ThenByDescending(x => x.Quant.BaseQuant.ExplicitCandidateSortOrder) + .ThenBy(x => x.IsHybrid) + .ThenBy(x => x.IsExternalPureBaseline) + .ThenBy(x => x.DisplayName, StringComparer.Ordinal) + .ToList(), + Eliminated = eliminated + .DistinctBy(x => TensorConfigIdentity.ToKey(x.Config)) + .ToList() + }; + } + + private static List CollapseEquivalentTruths( + IReadOnlyList ordered, + List eliminated) + { + var kept = new List(); + var used = new bool[ordered.Count]; + + for (int i = 0; i < ordered.Count; i++) + { + if (used[i]) + continue; + + var seed = ordered[i]; + var tied = new List { seed }; + used[i] = true; + + for (int j = i + 1; j < ordered.Count; j++) + { + if (used[j]) + continue; + + if (!EquivalentTruthSelectionHelper.AreEquivalentTruths( + seed.SizeBytes, + seed.Kld, + seed.Ppl, + ordered[j].SizeBytes, + ordered[j].Kld, + ordered[j].Ppl)) + continue; + + tied.Add(ordered[j]); + used[j] = true; + } + + if (tied.Count == 1) + { + kept.Add(seed); + continue; + } + + var representative = tied + .OrderByDescending(x => EquivalentTruthSelectionHelper.GetBaselineSafetyRank( + x.Quant.BaseQuant, + isHybrid: x.IsHybrid, + isExternalPureBaseline: x.IsExternalPureBaseline)) + .ThenBy(x => x.IsHybrid) + .ThenBy(x => x.IsExternalPureBaseline) + .ThenBy(x => x.ProviderName, StringComparer.Ordinal) + .ThenBy(x => x.DisplayName, StringComparer.Ordinal) + .First(); + + kept.Add(representative); + + foreach (var loser in tied) + { + if (!ReferenceEquals(loser, representative)) + eliminated.Add(loser); + } + } + + return kept; + } + + private static bool Dominates(BenchmarkSnapshotRecord better, BenchmarkSnapshotRecord worse) + { + bool sameOrSmaller = better.SizeBytes <= worse.SizeBytes; + bool strictlyBetterKld = better.Kld + IsolationPruningConfig.FloatingPointEpsilon < worse.Kld; + + // Final dominance intentionally follows the new survival rule: + // size must be same-or-smaller and KLD must be lower. PPL remains displayed + // and available for manual judgment, but it no longer prevents a KLD/size win. + return sameOrSmaller && strictlyBetterKld; + } +} \ No newline at end of file diff --git a/src/MagicQuant/Services/FinalReleaseMetadataService.cs b/src/MagicQuant/Services/FinalReleaseMetadataService.cs new file mode 100644 index 0000000..95f7574 --- /dev/null +++ b/src/MagicQuant/Services/FinalReleaseMetadataService.cs @@ -0,0 +1,320 @@ +using System.Text.Json; +using MagicQuant.Models; +using MQ.DB; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class FinalReleaseMetadataService +{ + public const string FinalSurvivorsFileName = MagicQuantManifestPathService.FinalSurvivorsFileName; + public const string ReplacementsFileName = MagicQuantManifestPathService.ReplacementsFileName; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + WriteIndented = true + }; + + private readonly FinalArtifactNamingService _namingService = new(); + + public async Task GenerateAsync( + string outputDirectory, + IReadOnlyCollection exportedArtifacts, + IReadOnlyCollection eliminations, + IReadOnlyCollection pureBaselineSnapshots, + BenchmarkSnapshotRecord? pplReference = null, + CancellationToken ct = default) + { + string manifestDirectory = MagicQuantManifestPathService.EnsureManifestDirectory(outputDirectory); + + double? referencePpl = ResolveReferencePpl(pplReference, pureBaselineSnapshots, exportedArtifacts.Select(x => x.Snapshot).ToList()); + var namingContext = _namingService.CreateContext(pureBaselineSnapshots); + var exportedByKey = exportedArtifacts + .GroupBy(x => TensorConfigIdentity.ToKey(x.Snapshot.Config), StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); + + var replacementMap = BuildReplacementMap(eliminations); + + string finalPath = Path.Combine(manifestDirectory, FinalSurvivorsFileName); + var survivors = exportedArtifacts + .OrderBy(x => x.Snapshot.Kld) + .ThenBy(x => x.Snapshot.SizeBytes) + .Select(x => ToSurvivorJson(x, referencePpl, replacementMap, namingContext)) + .ToList(); + await File.WriteAllTextAsync(finalPath, JsonSerializer.Serialize(survivors, JsonOptions), ct); + + string replacementsPath = Path.Combine(manifestDirectory, ReplacementsFileName); + var replacements = eliminations + .DistinctBy(x => $"{TensorConfigIdentity.ToKey(x.Eliminated.Config)}::{TensorConfigIdentity.ToKey(x.Eliminator.Config)}::{x.Reason}") + .OrderBy(x => x.Eliminated.Kld) + .ThenBy(x => x.Eliminated.SizeBytes) + .Select(x => ToReplacementJson(x, exportedByKey, namingContext, referencePpl)) + .ToList(); + await File.WriteAllTextAsync(replacementsPath, JsonSerializer.Serialize(replacements, JsonOptions), ct); + + AnsiConsole.MarkupLine($"[green]Final survivor metrics JSON generated:[/] {Markup.Escape(finalPath)}"); + AnsiConsole.MarkupLine($"[green]Replacement detail JSON generated:[/] {Markup.Escape(replacementsPath)}"); + } + + private object ToSurvivorJson( + ExportedArtifactRecord artifact, + double? referencePpl, + IReadOnlyDictionary> replacementMap, + FinalArtifactNamingContext namingContext) + { + string key = TensorConfigIdentity.ToKey(artifact.Snapshot.Config); + var replacements = ResolveTransitiveReplacements(key, replacementMap) + .Select(x => new + { + key = TensorConfigIdentity.ToKey(x.Eliminated.Config), + shortName = ToSnapshotShortName(x.Eliminated, null, namingContext), + internalDisplayName = x.Eliminated.DisplayName, + kld = x.Eliminated.Kld, + ppl = x.Eliminated.Ppl, + pplDeltaPercent = CalculatePplDeltaPercent(x.Eliminated.Ppl, referencePpl), + sizeBytes = x.Eliminated.SizeBytes, + sizeGB = ToGBNumber(x.Eliminated.SizeBytes), + sizeGiB = ToGiBNumber(x.Eliminated.SizeBytes), + reasonCode = FinalArtifactNamingService.ReasonCode(x.Reason), + reason = x.Reason + }) + .ToList(); + + return new + { + key, + fileName = artifact.IsExternalReference + ? EnsureGgufExtension(artifact.DisplayName) + : artifact.FileName, + displayName = artifact.DisplayName, + shortName = _namingService.ToPublicArtifactShortName( + artifact.DisplayName, + artifact.FileName, + artifact.ProviderName, + artifact.BaselineFamily, + artifact.Snapshot, + namingContext), + provider = artifact.ProviderName, + quantFamily = artifact.BaselineFamily, + isHybrid = artifact.Snapshot.IsHybrid, + isExternalPureBaseline = artifact.Snapshot.IsExternalPureBaseline, + isExternalRebuiltBaseline = artifact.Snapshot.IsExternalRebuiltBaseline, + isMaterializedTensorMapped = artifact.Snapshot.IsMaterializedTensorMapped, + isExternalReference = artifact.IsExternalReference, + downloadTarget = artifact.DownloadTarget, + kld = artifact.Snapshot.Kld, + ppl = artifact.Snapshot.Ppl, + pplDeltaPercent = CalculatePplDeltaPercent(artifact.Snapshot.Ppl, referencePpl), + sizeBytes = artifact.Snapshot.SizeBytes, + sizeGB = ToGBNumber(artifact.Snapshot.SizeBytes), + sizeGiB = ToGiBNumber(artifact.Snapshot.SizeBytes), + expectedSizeBytes = artifact.ExpectedSizeBytes, + actualSizeBytes = artifact.ActualSizeBytes, + usedImatrix = Cache.UseImatrix && Cache.IsImatrixAvailable, + replacedArtifacts = replacements + }; + } + + private object ToReplacementJson( + BaselineEliminationRecord row, + IReadOnlyDictionary exportedByKey, + FinalArtifactNamingContext namingContext, + double? referencePpl) + { + string reasonCode = FinalArtifactNamingService.ReasonCode(row.Reason); + double kldDelta = row.Eliminated.Kld - row.Eliminator.Kld; + long sizeDeltaBytes = (long)row.Eliminated.SizeBytes - (long)row.Eliminator.SizeBytes; + double? pplDeltaPercentRemoved = CalculatePplDeltaPercent(row.Eliminated.Ppl, referencePpl); + double? pplDeltaPercentWinner = CalculatePplDeltaPercent(row.Eliminator.Ppl, referencePpl); + double? pplDeltaPercentImprovement = pplDeltaPercentRemoved.HasValue && pplDeltaPercentWinner.HasValue + ? pplDeltaPercentRemoved.Value - pplDeltaPercentWinner.Value + : null; + + return new + { + reasonCode, + reasonDescription = FinalArtifactNamingService.ReasonDescription(reasonCode), + rawReason = row.Reason, + removed = ToReplacementSideJson(row.Eliminated, exportedByKey, namingContext, referencePpl), + winner = ToReplacementSideJson(row.Eliminator, exportedByKey, namingContext, referencePpl), + deltas = new + { + kld = kldDelta, + sizeBytes = sizeDeltaBytes, + sizeGB = ToGBNumber(sizeDeltaBytes), + sizeGiB = ToGiBNumber(sizeDeltaBytes), + removedPplDeltaPercent = pplDeltaPercentRemoved, + winnerPplDeltaPercent = pplDeltaPercentWinner, + pplDeltaPercentImprovement = pplDeltaPercentImprovement + } + }; + } + + private object ToReplacementSideJson( + BenchmarkSnapshotRecord snapshot, + IReadOnlyDictionary exportedByKey, + FinalArtifactNamingContext namingContext, + double? referencePpl) + { + string key = TensorConfigIdentity.ToKey(snapshot.Config); + string displayName; + string fileName; + string shortName; + string provider; + string quantFamily; + + if (exportedByKey.TryGetValue(key, out var artifact)) + { + displayName = artifact.DisplayName; + shortName = _namingService.ToPublicArtifactShortName( + artifact.DisplayName, + artifact.FileName, + artifact.ProviderName, + artifact.BaselineFamily, + artifact.Snapshot, + namingContext); + fileName = artifact.IsExternalReference ? EnsureGgufExtension(artifact.DisplayName) : artifact.FileName ?? EnsureGgufExtension(artifact.DisplayName); + provider = artifact.ProviderName; + quantFamily = artifact.BaselineFamily; + } + else + { + displayName = _namingService.BuildDisplayLabel(snapshot, namingContext); + provider = snapshot.IsHybrid ? "MagicQuant" : HybridBenchmarkRepository.ResolveProviderName(snapshot.Quant, exportNaming: false); + quantFamily = snapshot.BaselineFamily; + shortName = _namingService.ToPublicArtifactShortName( + displayName, + null, + provider, + quantFamily, + snapshot, + namingContext); + fileName = EnsureGgufExtension(displayName); + } + + return new + { + key, + fileName, + displayName, + shortName, + provider, + quantFamily, + isHybrid = snapshot.IsHybrid, + isExternalPureBaseline = snapshot.IsExternalPureBaseline, + isExternalRebuiltBaseline = snapshot.IsExternalRebuiltBaseline, + isMaterializedTensorMapped = snapshot.IsMaterializedTensorMapped, + kld = snapshot.Kld, + ppl = snapshot.Ppl, + pplDeltaPercent = CalculatePplDeltaPercent(snapshot.Ppl, referencePpl), + sizeBytes = snapshot.SizeBytes, + sizeGB = ToGBNumber(snapshot.SizeBytes), + sizeGiB = ToGiBNumber(snapshot.SizeBytes) + }; + } + + public static IReadOnlyDictionary> BuildReplacementMap( + IReadOnlyCollection eliminations) + { + return eliminations + .GroupBy(x => TensorConfigIdentity.ToKey(x.Eliminator.Config), StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.ToList(), StringComparer.Ordinal); + } + + public static IReadOnlyList ResolveTransitiveReplacements( + string finalWinnerKey, + IReadOnlyDictionary> replacementMap) + { + var output = new List(); + var visited = new HashSet(StringComparer.Ordinal); + + void Visit(string winnerKey) + { + if (!visited.Add(winnerKey)) + return; + + if (!replacementMap.TryGetValue(winnerKey, out var direct)) + return; + + foreach (var row in direct) + { + output.Add(row); + Visit(TensorConfigIdentity.ToKey(row.Eliminated.Config)); + } + } + + Visit(finalWinnerKey); + + return output + .DistinctBy(x => TensorConfigIdentity.ToKey(x.Eliminated.Config), StringComparer.Ordinal) + .ToList(); + } + + private static double? ResolveReferencePpl( + BenchmarkSnapshotRecord? pplReference, + IReadOnlyCollection pureBaselineSnapshots, + IReadOnlyCollection snapshots) + { + if (pplReference is { Ppl: > 0d }) + return pplReference.Ppl; + + var bestPure = pureBaselineSnapshots + .Where(x => x.Ppl > 0d) + .OrderBy(x => x.Kld) + .ThenByDescending(x => x.SizeBytes) + .FirstOrDefault(); + + if (bestPure != null) + return bestPure.Ppl; + + return snapshots + .Where(x => x.Ppl > 0d) + .OrderBy(x => x.Kld) + .FirstOrDefault() + ?.Ppl; + } + + public static double? CalculatePplDeltaPercent(double ppl, double? referencePpl) + { + if (referencePpl is null or <= 0d || ppl <= 0d) + return null; + + return ((ppl - referencePpl.Value) / referencePpl.Value) * 100d; + } + + private static string EnsureGgufExtension(string value) + { + if (string.IsNullOrWhiteSpace(value)) + return string.Empty; + + return value.EndsWith(".gguf", StringComparison.OrdinalIgnoreCase) + ? value + : value + ".gguf"; + } + + private string ToSnapshotShortName( + BenchmarkSnapshotRecord snapshot, + ExportedArtifactRecord? artifact, + FinalArtifactNamingContext namingContext) + { + if (artifact != null) + { + return _namingService.ToPublicArtifactShortName( + artifact.DisplayName, + artifact.FileName, + artifact.ProviderName, + artifact.BaselineFamily, + artifact.Snapshot, + namingContext); + } + + string provider = snapshot.IsHybrid ? "MagicQuant" : HybridBenchmarkRepository.ResolveProviderName(snapshot.Quant, exportNaming: false); + string display = _namingService.BuildDisplayLabel(snapshot, namingContext); + return _namingService.ToPublicArtifactShortName(display, null, provider, snapshot.BaselineFamily, snapshot, namingContext); + } + + private static double ToGBNumber(ulong bytes) => bytes / 1000d / 1000d / 1000d; + private static double ToGiBNumber(ulong bytes) => bytes / 1024d / 1024d / 1024d; + private static double ToGBNumber(long bytes) => bytes / 1000d / 1000d / 1000d; + private static double ToGiBNumber(long bytes) => bytes / 1024d / 1024d / 1024d; +} \ No newline at end of file diff --git a/src/MagicQuant/Services/FinalSurvivorSelectionCliService.cs b/src/MagicQuant/Services/FinalSurvivorSelectionCliService.cs new file mode 100644 index 0000000..2de4486 --- /dev/null +++ b/src/MagicQuant/Services/FinalSurvivorSelectionCliService.cs @@ -0,0 +1,172 @@ +using MagicQuant.Models; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class FinalSurvivorSelectionCliService +{ + private readonly FinalArtifactNamingService _namingService = new(); + + public IReadOnlyList Prompt( + IReadOnlyCollection survivors, + IReadOnlyCollection pureBaselineSnapshots, + BenchmarkSnapshotRecord? pplReference = null) + { + var namingContext = _namingService.CreateContext(pureBaselineSnapshots); + var reservedFileNames = new HashSet(StringComparer.OrdinalIgnoreCase); + double? referencePpl = ResolveReferencePpl(pplReference, pureBaselineSnapshots, survivors); + + var rows = survivors + .OrderBy(x => x.Kld) + .ThenBy(x => x.SizeBytes) + .Select((snapshot, index) => + { + var name = _namingService.BuildName(snapshot, namingContext, reservedFileNames); + return new FinalSelectionRow + { + Id = index + 1, + Enabled = true, + Snapshot = snapshot, + PlannedFileName = name.FileName, + // Show the same normalized public short label used by README/manifest tables. + // The full GGUF filename remains PlannedFileName. + PlannedDisplayName = string.IsNullOrWhiteSpace(name.ShortDisplayName) ? name.DisplayName : name.ShortDisplayName, + PlannedProviderName = ResolveProviderName(snapshot, name), + PlannedQuantFamily = name.QuantFamilyOrBaseline + }; + }) + .ToList(); + + if (rows.Count == 0) + { + AnsiConsole.MarkupLine("[yellow]No survivors remained after brutal real-truth elimination.[/]"); + return rows; + } + + while (true) + { + Render(rows, referencePpl); + + string input = AnsiConsole.Prompt( + new TextPrompt("Toggle [cyan]row number[/], or type [green]ready[/] to continue") + .AllowEmpty()) + .Trim(); + + if (string.IsNullOrWhiteSpace(input) || + string.Equals(input, "ready", StringComparison.OrdinalIgnoreCase) || + string.Equals(input, "continue", StringComparison.OrdinalIgnoreCase) || + string.Equals(input, "done", StringComparison.OrdinalIgnoreCase)) + { + if (rows.Any(x => x.Enabled)) + return rows; + + AnsiConsole.MarkupLine("[red]At least one survivor must remain enabled.[/]"); + continue; + } + + if (!int.TryParse(input, out var id)) + { + AnsiConsole.MarkupLine($"[yellow]Unknown selection command:[/] {Markup.Escape(input)}"); + continue; + } + + var row = rows.FirstOrDefault(x => x.Id == id); + if (row == null) + { + AnsiConsole.MarkupLine($"[yellow]No row exists with ID {id}.[/]"); + continue; + } + + row.Enabled = !row.Enabled; + } + } + + private static void Render(IReadOnlyCollection rows, double? referencePpl) + { + AnsiConsole.Clear(); + AnsiConsole.Write(new Rule("[yellow]Final Survivor Selection[/]") { Justification = Justify.Left }); + + var table = new Table().Border(TableBorder.Rounded).Expand(); + table.AddColumn("ID"); + table.AddColumn("State"); + table.AddColumn("Display / Model"); + table.AddColumn("Provider"); + table.AddColumn("Quant Family"); + table.AddColumn("KLD"); + table.AddColumn("PPL Δ %"); + table.AddColumn("Size (GB)"); + + foreach (var row in rows) + { + var snap = row.Snapshot; + string state = row.Enabled ? "[green]ENABLED[/]" : "[red]DISABLED[/]"; + string displayValue = string.IsNullOrWhiteSpace(row.PlannedDisplayName) ? snap.DisplayName : row.PlannedDisplayName; + string providerValue = string.IsNullOrWhiteSpace(row.PlannedProviderName) ? snap.ProviderName : row.PlannedProviderName; + string familyValue = string.IsNullOrWhiteSpace(row.PlannedQuantFamily) ? snap.BaselineFamily : row.PlannedQuantFamily; + + string display = row.Enabled ? Markup.Escape(displayValue) : $"[grey]{Markup.Escape(displayValue)}[/]"; + string provider = row.Enabled ? Markup.Escape(providerValue) : $"[grey]{Markup.Escape(providerValue)}[/]"; + string family = row.Enabled ? Markup.Escape(familyValue) : $"[grey]{Markup.Escape(familyValue)}[/]"; + string kld = row.Enabled ? $"[cyan]{snap.Kld:0.000000}[/]" : $"[grey]{snap.Kld:0.000000}[/]"; + string pplDelta = FormatPplDeltaPercent(snap.Ppl, referencePpl); + string ppl = row.Enabled ? $"[cyan]{pplDelta}[/]" : $"[grey]{pplDelta}[/]"; + string sizeGb = (snap.SizeBytes / 1000d / 1000d / 1000d).ToString("0.00"); + + table.AddRow( + row.Id.ToString(), + state, + display, + provider, + family, + kld, + ppl, + row.Enabled ? $"[cyan]{sizeGb}[/]" : $"[grey]{sizeGb}[/]"); + } + + AnsiConsole.Write(table); + AnsiConsole.MarkupLine("[grey]PPL Δ % is measured against the native/reference PPL when available. Negative is better; larger positive values are worse.[/]"); + } + + private static string ResolveProviderName(BenchmarkSnapshotRecord snapshot, FinalArtifactName name) + { + if (snapshot.IsHybrid) + return "MagicQuant"; + + // MQ-* in the planned artifact name can mean "rebuilt by MagicQuant". + // It must not overwrite the semantic upstream provider for rebuilt Unsloth/custom baselines. + return HybridBenchmarkRepository.ResolveProviderName(snapshot.Quant, exportNaming: false); + } + + private static double? ResolveReferencePpl( + BenchmarkSnapshotRecord? pplReference, + IReadOnlyCollection pureBaselineSnapshots, + IReadOnlyCollection survivors) + { + if (pplReference is { Ppl: > 0d }) + return pplReference.Ppl; + + var bestPure = pureBaselineSnapshots + .Where(x => x.Ppl > 0d) + .OrderBy(x => x.Kld) + .ThenByDescending(x => x.SizeBytes) + .FirstOrDefault(); + + if (bestPure != null) + return bestPure.Ppl; + + return survivors + .Where(x => x.Ppl > 0d) + .OrderBy(x => x.Kld) + .FirstOrDefault() + ?.Ppl; + } + + private static string FormatPplDeltaPercent(double ppl, double? referencePpl) + { + if (referencePpl is null or <= 0d || ppl <= 0d) + return "n/a"; + + double delta = ((ppl - referencePpl.Value) / referencePpl.Value) * 100d; + return $"{delta:0.000}%"; + } +} \ No newline at end of file diff --git a/src/MagicQuant/Services/GgufMetadataReader.cs b/src/MagicQuant/Services/GgufMetadataReader.cs new file mode 100644 index 0000000..2ecd3c7 --- /dev/null +++ b/src/MagicQuant/Services/GgufMetadataReader.cs @@ -0,0 +1,121 @@ +using System.Text.Json; +using MagicQuant.Helpers; + +namespace MagicQuant.Services; + +internal sealed class GgufMetadataReader +{ + private readonly PythonManager _python; + + public GgufMetadataReader(PythonManager python) + { + _python = python ?? throw new ArgumentNullException(nameof(python)); + } + + public async Task ReadAsync( + string ggufPath, + string workingDirectory, + CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(ggufPath) || !File.Exists(ggufPath)) + throw new FileNotFoundException("GGUF metadata source was not found.", ggufPath); + + Directory.CreateDirectory(workingDirectory); + + string unique = Guid.NewGuid().ToString("N"); + string payloadPath = Path.Combine(workingDirectory, $"read_gguf_tensors_{unique}.json"); + string resultPath = Path.Combine(workingDirectory, $"read_gguf_tensors_result_{unique}.json"); + string scriptPath = Path.Combine(workingDirectory, $"read_gguf_tensors_{unique}.py"); + + try + { + await File.WriteAllTextAsync( + payloadPath, + JsonSerializer.Serialize(new { gguf_path = ggufPath, output_path = resultPath }), + ct); + + const string py = """ + import json + import sys + + payload_path = sys.argv[1] + with open(payload_path, "r", encoding="utf-8") as f: + payload = json.load(f) + + output_path = payload["output_path"] + + def resolve_type_name(t): + for attr in ["type_name", "tensor_type", "type"]: + v = getattr(t, attr, None) + if v is None: + continue + if hasattr(v, "name"): + return str(v.name) + return str(v) + return "UNKNOWN" + + try: + import gguf + reader = gguf.GGUFReader(payload["gguf_path"]) + tensor_names = [t.name for t in reader.tensors] + tensor_types = {t.name: resolve_type_name(t) for t in reader.tensors} + + def read_scalar(key): + field = reader.fields.get(key) + if field is None: + return None + value = field.contents() + return value.item() if hasattr(value, "item") else value + + architecture = read_scalar("general.architecture") + architecture_key = str(architecture) if architecture is not None else None + block_count = read_scalar(f"{architecture_key}.block_count") if architecture_key else None + nextn_layers = read_scalar(f"{architecture_key}.nextn_predict_layers") if architecture_key else None + result = { + "Architecture": architecture_key, + "BlockCount": int(block_count) if block_count is not None else None, + "NextnPredictLayers": int(nextn_layers) if nextn_layers is not None else None, + "TensorNames": tensor_names, + "TensorTypes": tensor_types + } + except Exception as e: + result = {"Error": str(e), "TensorNames": [], "TensorTypes": {}} + + with open(output_path, "w", encoding="utf-8") as f: + json.dump(result, f, indent=2) + """; + + await File.WriteAllTextAsync(scriptPath, py, ct); + await _python.RunPythonScriptAsync(scriptPath, [payloadPath], ct: ct); + + var result = JsonSerializer.Deserialize( + await File.ReadAllTextAsync(resultPath, ct)); + + if (result == null) + throw new InvalidOperationException("Failed to parse GGUF metadata result."); + if (!string.IsNullOrWhiteSpace(result.Error)) + throw new InvalidOperationException($"Failed to read GGUF metadata: {result.Error}"); + + return result; + } + finally + { + TryDelete(payloadPath); + TryDelete(resultPath); + TryDelete(scriptPath); + } + } + + private static void TryDelete(string path) + { + try + { + if (File.Exists(path)) + File.Delete(path); + } + catch + { + // Best-effort cleanup; metadata success/failure is the authoritative result. + } + } +} diff --git a/src/MagicQuant/Services/HuggingFaceBaselineService.cs b/src/MagicQuant/Services/HuggingFaceBaselineService.cs new file mode 100644 index 0000000..0d8e901 --- /dev/null +++ b/src/MagicQuant/Services/HuggingFaceBaselineService.cs @@ -0,0 +1,778 @@ +using Microsoft.EntityFrameworkCore; +using MQ.DB.Data; +using System.Text.Json; +using MagicQuant.Configuration; +using MagicQuant.Helpers; +using MQ.DB; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class HuggingFaceBaselineService +{ + private readonly PythonManager _python; + + public HuggingFaceBaselineService(PythonManager python) + { + _python = python ?? throw new ArgumentNullException(nameof(python)); + } + + + public async Task> PrecheckAndRegisterConfiguredBaselinesAsync(CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + await EnsureHubSupportAsync(); + + int architectureFamilyId = Cache.CurrentArchitectureFamilyId + ?? throw new InvalidOperationException("Custom baseline sync requires the architecture family to be resolved first."); + + var enabledRepos = Config.Current.Baselines.CustomRepositories.Where(x => x.Enabled).ToList(); + var resolved = new List(); + BaselineQuants.ResetDynamicCustomBaselines(); + + AnsiConsole.Write(new Rule("[yellow]Custom Baseline DB Sync[/]") { Justification = Justify.Left }); + AnsiConsole.MarkupLine($"[grey]Enabled custom repositories:[/] [cyan]{enabledRepos.Count:N0}[/]"); + + await using var db = new MagicQuantContext(); + var existingDefinitions = await db.BaselineQuantDefinitions + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && x.IsCustomBaseline) + .ToListAsync(ct); + + foreach (var definition in existingDefinitions) + { + definition.IsActiveInCurrentConfig = false; + definition.LastUpdatedUtc = DateTime.UtcNow; + } + + RegisterHistoricalDefinitions(existingDefinitions); + + var existingDynamicIdsByCanonicalKey = existingDefinitions + .Where(x => !string.IsNullOrWhiteSpace(x.NormalizedCanonicalKey)) + .GroupBy(x => x.NormalizedCanonicalKey, StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.OrderBy(x => x.RuntimeBaselineId).First().RuntimeBaselineId, StringComparer.Ordinal); + + var reservedIds = BaselineQuants.GetAllRecognizedBaselines().Select(x => x.UniqueId).ToHashSet(); + foreach (var persistedId in existingDefinitions.Select(x => x.RuntimeBaselineId)) + reservedIds.Add(persistedId); + + byte nextId = BaselineQuants.GetFirstAvailableDynamicBaselineId(); + var now = DateTime.UtcNow; + + foreach (var repo in enabledRepos) + { + if (string.IsNullOrWhiteSpace(repo.RepoId)) + throw new InvalidOperationException("Custom baseline repository entry is missing repo_id."); + + if (repo.Includes.Count == 0) + throw new InvalidOperationException($"Custom baseline repository '{repo.RepoId}' is enabled but has zero include entries."); + + string revisionLabel = string.IsNullOrWhiteSpace(repo.Revision) + ? "main" + : repo.Revision!; + AnsiConsole.MarkupLine( + $"[cyan]Repo:[/] {Markup.Escape(repo.RepoId)} [grey](revision={Markup.Escape(revisionLabel)}, includes={repo.Includes.Count})[/]"); + + var repoFiles = await ListRepoFilesAsync(repo.RepoId, repo.Revision, ct); + if (repoFiles.Count == 0) + throw new InvalidOperationException($"No files were returned from Hugging Face repo '{repo.RepoId}'."); + + var ggufRepoFiles = repoFiles.Where(x => x.EndsWith(".gguf", StringComparison.OrdinalIgnoreCase)).ToList(); + AnsiConsole.MarkupLine($" [grey]GGUF files discovered:[/] [cyan]{ggufRepoFiles.Count:N0}[/]"); + + string shortSourceName = string.IsNullOrWhiteSpace(repo.ShortSourceName) + ? DeriveShortSourceName(repo.RepoId) + : repo.ShortSourceName!.Trim(); + + foreach (var include in repo.Includes) + { + if (string.IsNullOrWhiteSpace(include.BaselineFamily)) + throw new InvalidOperationException($"Repo '{repo.RepoId}' has an include entry missing baseline_family."); + + var standardFamily = BaselineQuants.ResolveBuiltInStandardBaseline(include.BaselineFamily) + ?? throw new InvalidOperationException( + $"Custom baseline include '{include.BaselineFamily}' in repo '{repo.RepoId}' could not be matched to a built-in baseline family."); + + string resolvedFileName = ResolveRepoFileName(repoFiles, include, standardFamily); + string normalizedRepo = BaselineDefinitionResolver.NormalizeRepoId(repo.RepoId); + string normalizedFile = BaselineDefinitionResolver.NormalizeFileName(resolvedFileName); + string canonicalKey = BuildCanonicalKey(Cache.CurrentArchitectureFamilyName, repo.RepoId, resolvedFileName); + string normalizedCanonicalKey = BaselineDefinitionResolver.NormalizeCanonicalKey(canonicalKey); + + string displayName = string.IsNullOrWhiteSpace(include.DisplayName) + ? $"{shortSourceName}-{standardFamily.Names[0]}" + : include.DisplayName!.Trim(); + + bool requiresImatrix = include.RequiresImatrix ?? standardFamily.RequiresImatrix; + bool allowAsLearning = include.AllowAsLearningBaseline ?? repo.AllowAsLearningBaseline; + bool allowAsCarrier = include.AllowAsCombinationCarrier ?? repo.AllowAsCombinationCarrier; + bool allowAsExplicit = include.AllowAsExplicitGroupCandidate ?? repo.AllowAsExplicitGroupCandidate; + string quantizeBaseName = string.IsNullOrWhiteSpace(include.QuantizeBaseName) + ? standardFamily.Names[0] + : include.QuantizeBaseName!.Trim(); + + var bannedGroups = include.BannedGroupIds.Count > 0 + ? include.BannedGroupIds.ToArray() + : standardFamily.BannedGroupIds.ToArray(); + + var definition = existingDefinitions.FirstOrDefault(x => + string.Equals(x.NormalizedSourceRepository, normalizedRepo, StringComparison.Ordinal) && + string.Equals(x.NormalizedSourceFileName, normalizedFile, StringComparison.Ordinal)); + + if (definition != null && !string.Equals(definition.BaselineFamily, standardFamily.Names[0], StringComparison.Ordinal) && !include.ForceRelearn) + { + throw new InvalidOperationException( + $"Custom baseline semantic family changed for {repo.RepoId}/{resolvedFileName}: " + + $"DB has '{definition.BaselineFamily}', YAML now says '{standardFamily.Names[0]}'. " + + "This is destructive. Set this include's force_relearn: true so MagicQuant can plan and confirm targeted invalidation before resyncing the definition."); + } + + byte dynamicBaselineId = definition?.RuntimeBaselineId ?? ResolveDynamicBaselineId( + normalizedCanonicalKey, + existingDynamicIdsByCanonicalKey, + reservedIds, + ref nextId); + + var nextDefinition = new BaselineQuantDefinition + { + ArchitectureFamilyId = architectureFamilyId, + RuntimeBaselineId = dynamicBaselineId, + CanonicalKey = canonicalKey, + NormalizedCanonicalKey = normalizedCanonicalKey, + BaselineName = displayName, + DisplayName = displayName, + QuantizeBaseArgumentName = quantizeBaseName, + DefaultTensorSchemeId = standardFamily.PrimaryTensorWeightScheme.UniqueId, + DefaultTensorSchemeName = standardFamily.PrimaryTensorWeightScheme.Names[0], + SourceKind = repo.SourceKind, + SourceOwner = DeriveSourceOwner(repo.RepoId), + SourceRepository = repo.RepoId, + NormalizedSourceRepository = normalizedRepo, + SourceFileName = resolvedFileName, + NormalizedSourceFileName = normalizedFile, + ShortSourceName = shortSourceName, + BaselineFamily = standardFamily.Names[0], + IsCustomBaseline = true, + IsLearningBaseline = allowAsLearning, + IsCombinationCarrierCandidate = allowAsCarrier, + IsExplicitGroupCombinationCandidate = allowAsExplicit, + RequiresImatrix = requiresImatrix, + BitRange = standardFamily.BitRange, + ExplicitCandidateSortOrder = standardFamily.ExplicitCandidateSortOrder, + IsActiveInCurrentConfig = true, + FirstSeenUtc = definition?.FirstSeenUtc ?? now, + LastSeenUtc = now, + LastUpdatedUtc = now + }; + + if (definition == null) + { + definition = nextDefinition; + db.BaselineQuantDefinitions.Add(definition); + existingDefinitions.Add(definition); + } + else + { + MagicQuantContext.ApplyBaselineDefinitionUpdate(definition, nextDefinition, preserveFirstSeen: true); + } + + var dynamicBaseline = BaselineQuants.CreateDynamicCustomBaseline( + uniqueId: dynamicBaselineId, + displayName: displayName, + quantizeBaseArgumentName: quantizeBaseName, + sourceRepository: repo.RepoId, + sourceFileName: resolvedFileName, + shortSourceName: shortSourceName, + sourceOwner: DeriveSourceOwner(repo.RepoId), + sourceKind: repo.SourceKind, + canonicalKey: canonicalKey, + primaryTensorWeightScheme: standardFamily.PrimaryTensorWeightScheme, + learnedMatchTensorWeightSchemes: standardFamily.LearnedMatchTensorWeightSchemes, + bannedGroupIds: bannedGroups, + requiresImatrix: requiresImatrix, + isLearningBaseline: allowAsLearning, + isCombinationCarrierCandidate: allowAsCarrier, + isExplicitGroupCombinationCandidate: allowAsExplicit, + bitRange: standardFamily.BitRange, + explicitCandidateSortOrder: standardFamily.ExplicitCandidateSortOrder); + + BaselineQuants.RegisterDynamicCustomBaseline(dynamicBaseline); + + var spec = new ResolvedCustomBaselineSpec + { + DynamicBaselineId = dynamicBaseline.UniqueId, + BaselineQuantDefinitionId = definition.Id == 0 ? null : definition.Id, + CanonicalKey = dynamicBaseline.CanonicalKey, + DisplayName = dynamicBaseline.Names[0], + RepoId = repo.RepoId, + Revision = repo.Revision, + SourceOwner = dynamicBaseline.SourceOwner ?? string.Empty, + SourceFileName = dynamicBaseline.SourceFileName ?? string.Empty, + ShortSourceName = dynamicBaseline.ShortSourceName ?? shortSourceName, + BaselineFamily = standardFamily.Names[0], + QuantizeBaseName = dynamicBaseline.QuantizeBaseArgumentName, + RequiresImatrix = dynamicBaseline.RequiresImatrix, + AllowAsLearningBaseline = dynamicBaseline.IsLearningBaseline, + AllowAsCombinationCarrier = dynamicBaseline.IsCombinationCarrierCandidate, + AllowAsExplicitGroupCandidate = dynamicBaseline.IsExplicitGroupCombinationCandidate, + ForceRelearn = include.ForceRelearn, + IsActiveInCurrentConfig = true, + BannedGroupIds = dynamicBaseline.BannedGroupIds + }; + + resolved.Add(spec); + AnsiConsole.MarkupLine( + $" [green]Resolved:[/] id=[cyan]{dynamicBaseline.UniqueId}[/] family=[yellow]{Markup.Escape(standardFamily.Names[0])}[/] file=[blue]{Markup.Escape(resolvedFileName)}[/] learning={allowAsLearning} carrier={allowAsCarrier} explicit={allowAsExplicit} relearn={include.ForceRelearn}"); + } + } + + await db.SaveChangesAsync(ct); + + foreach (var spec in resolved.Where(x => x.BaselineQuantDefinitionId == null)) + { + var definition = await db.BaselineQuantDefinitions.AsNoTracking().FirstAsync(x => + x.ArchitectureFamilyId == architectureFamilyId && + x.RuntimeBaselineId == spec.DynamicBaselineId, ct); + spec.BaselineQuantDefinitionId = definition.Id; + } + + RegisterHistoricalDefinitions(existingDefinitions.Where(x => !x.IsActiveInCurrentConfig)); + + Config.SetResolvedCustomBaselines(resolved); + BaselineQuants.ValidateIntegrityOrThrow(); + + AnsiConsole.MarkupLine($"[green]Custom baseline sync complete:[/] [cyan]{resolved.Count:N0}[/] active custom baseline(s); [cyan]{existingDefinitions.Count(x => !x.IsActiveInCurrentConfig):N0}[/] inactive historical definition(s) retained."); + return resolved; + } + + private static void RegisterHistoricalDefinitions(IEnumerable definitions) + { + foreach (var definition in definitions.Where(x => x.IsCustomBaseline)) + { + try + { + var runtime = BaselineDefinitionResolver.ToRuntimeBaseline(definition, forceInactiveRegistration: true); + BaselineQuants.RegisterDynamicCustomBaseline(runtime); + } + catch + { + // A bad historical row should not prevent active YAML from being resolved. + // It simply will not be available for runtime TensorConfig hydration until fixed. + } + } + } + + private static byte ResolveDynamicBaselineId( + string normalizedCanonicalKey, + IReadOnlyDictionary existingDynamicIdsByCanonicalKey, + HashSet reservedIds, + ref byte nextId) + { + if (!string.IsNullOrWhiteSpace(normalizedCanonicalKey) && + existingDynamicIdsByCanonicalKey.TryGetValue(normalizedCanonicalKey, out var existingId)) + { + reservedIds.Add(existingId); + return existingId; + } + + while (reservedIds.Contains(nextId)) + { + if (nextId >= 199) + throw new InvalidOperationException("No free dynamic baseline ids remain in the configured range."); + + nextId++; + } + + var allocated = nextId; + reservedIds.Add(allocated); + + if (nextId < 199) + nextId++; + + return allocated; + } + + public async Task DownloadBaselineAsync(BaselineQuants baseline, string destinationPath, bool forceRedownload = false, CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (!baseline.IsExternalRepositoryBaseline) + throw new InvalidOperationException($"Baseline '{baseline.Names[0]}' is not an external repository baseline."); + + await EnsureHubSupportAsync(); + + var spec = Config.GetResolvedCustomBaseline(baseline.CanonicalKey) + ?? throw new InvalidOperationException($"No resolved custom baseline spec exists for canonical key '{baseline.CanonicalKey}'."); + + Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!); + + string payloadPath = Path.Combine(Path.GetDirectoryName(destinationPath)!, $"hf_download_{Guid.NewGuid():N}.json"); + string scriptPath = Path.Combine(Path.GetDirectoryName(destinationPath)!, $"hf_download_{Guid.NewGuid():N}.py"); + string resultPath = Path.Combine(Path.GetDirectoryName(destinationPath)!, $"hf_download_result_{Guid.NewGuid():N}.json"); + string atomicStagingPath = destinationPath + $".partial.{Guid.NewGuid():N}"; + + try + { + await File.WriteAllTextAsync(payloadPath, JsonSerializer.Serialize(new + { + repo_id = spec.RepoId, + revision = spec.Revision, + file_name = spec.SourceFileName, + target_path = destinationPath, + force_redownload = forceRedownload, + result_path = resultPath + }), ct); + + const string py = """ + import json + import os + import sys + from huggingface_hub import hf_hub_download + + payload_path = sys.argv[1] + with open(payload_path, 'r', encoding='utf-8') as f: + payload = json.load(f) + + target_path = payload['target_path'] + result_path = payload['result_path'] + os.makedirs(os.path.dirname(target_path), exist_ok=True) + + try: + downloaded = hf_hub_download( + repo_id=payload['repo_id'], + revision=payload.get('revision') or None, + filename=payload['file_name'], + local_dir=os.path.dirname(target_path), + force_download=payload.get('force_redownload', False), + ) + + result = { + 'ok': True, + 'downloaded_path': downloaded, + 'size_bytes': os.path.getsize(downloaded) if os.path.exists(downloaded) else 0, + } + except Exception as ex: + result = { + 'ok': False, + 'error': str(ex), + } + + with open(result_path, 'w', encoding='utf-8') as f: + json.dump(result, f) + """; + + await File.WriteAllTextAsync(scriptPath, py, ct); + await _python.RunPythonScriptAsync(scriptPath, $"\"{payloadPath}\""); + + var json = JsonDocument.Parse(await File.ReadAllTextAsync(resultPath, ct)).RootElement; + if (!json.GetProperty("ok").GetBoolean()) + throw new InvalidOperationException($"External baseline download failed: {json.GetProperty("error").GetString()}"); + + string downloadedPath = json.GetProperty("downloaded_path").GetString() + ?? throw new InvalidOperationException("External baseline download did not return a source path."); + if (!File.Exists(downloadedPath) || new FileInfo(downloadedPath).Length == 0) + throw new InvalidOperationException($"External baseline download completed but produced no file: {downloadedPath}"); + if (!HasGgufMagic(downloadedPath)) + throw new InvalidOperationException($"Downloaded external baseline is not a GGUF file: {downloadedPath}"); + + bool reused = CanReuseDownloadedFile(downloadedPath, destinationPath); + if (!reused && !PathsReferToSameLocation(downloadedPath, destinationPath)) + { + await CopyDownloadedFileAtomicallyAsync(downloadedPath, atomicStagingPath, destinationPath, ct); + } + + if (!File.Exists(destinationPath) || new FileInfo(destinationPath).Length == 0 || !HasGgufMagic(destinationPath)) + throw new InvalidOperationException($"External baseline staging produced no valid GGUF file: {destinationPath}"); + + if (!PathsReferToSameLocation(downloadedPath, destinationPath)) + { + string destinationDirectory = Path.GetDirectoryName(Path.GetFullPath(destinationPath))!; + if (!IsPathInsideDirectory(downloadedPath, destinationDirectory)) + { + throw new InvalidOperationException( + $"Refusing to delete Hugging Face staging artifact outside the external baseline cache: {downloadedPath}"); + } + + await HardDeleteHelper.DeleteFileIfExistsAsync(downloadedPath); + } + + AnsiConsole.MarkupLine(reused + ? $"[grey]Reusing verified cached external baseline:[/] {Markup.Escape(destinationPath)}" + : $"[green]Downloaded and atomically staged external baseline:[/] {Markup.Escape(destinationPath)}"); + return destinationPath; + } + finally + { + TryDelete(payloadPath); + TryDelete(scriptPath); + TryDelete(resultPath); + TryDelete(atomicStagingPath); + } + } + + internal static bool CanReuseDownloadedFile(string downloadedPath, string destinationPath) + { + if (!File.Exists(downloadedPath) || !File.Exists(destinationPath) || + !HasGgufMagic(downloadedPath) || !HasGgufMagic(destinationPath)) + { + return false; + } + + var downloaded = new FileInfo(downloadedPath); + var destination = new FileInfo(destinationPath); + return downloaded.Length > 0 && + downloaded.Length == destination.Length && + downloaded.LastWriteTimeUtc == destination.LastWriteTimeUtc; + } + + internal static bool IsPathInsideDirectory(string childPath, string parentDirectory) + { + string child = ResolvePhysicalPath(childPath) + .TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + string parent = ResolvePhysicalPath(parentDirectory) + .TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + + return child.StartsWith(parent + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase); + } + + internal static bool PathsReferToSameLocation(string firstPath, string secondPath) + { + return string.Equals( + ResolvePhysicalPath(firstPath).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), + ResolvePhysicalPath(secondPath).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar), + StringComparison.OrdinalIgnoreCase); + } + + private static string ResolvePhysicalPath(string path) + { + string fullPath = Path.GetFullPath(path); + string root = Path.GetPathRoot(fullPath) + ?? throw new InvalidOperationException($"Path '{path}' has no filesystem root."); + string current = root; + string relative = Path.GetRelativePath(root, fullPath); + + foreach (string component in relative.Split( + [Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar], + StringSplitOptions.RemoveEmptyEntries)) + { + current = Path.Combine(current, component); + + FileSystemInfo info = Directory.Exists(current) + ? new DirectoryInfo(current) + : new FileInfo(current); + + if (!info.Exists || string.IsNullOrWhiteSpace(info.LinkTarget)) + continue; + + current = info.ResolveLinkTarget(returnFinalTarget: true)?.FullName ?? current; + } + + return Path.GetFullPath(current); + } + + private static bool HasGgufMagic(string path) + { + try + { + Span magic = stackalloc byte[4]; + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); + return stream.Read(magic) == magic.Length && magic.SequenceEqual("GGUF"u8); + } + catch + { + return false; + } + } + + private static async Task CopyDownloadedFileAtomicallyAsync( + string sourcePath, + string stagingPath, + string destinationPath, + CancellationToken ct) + { + try + { + await using (var source = new FileStream( + sourcePath, FileMode.Open, FileAccess.Read, FileShare.Read, + bufferSize: 1024 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan)) + await using (var staging = new FileStream( + stagingPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, + bufferSize: 1024 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan)) + { + await source.CopyToAsync(staging, 1024 * 1024, ct); + await staging.FlushAsync(ct); + } + + File.SetLastWriteTimeUtc(stagingPath, File.GetLastWriteTimeUtc(sourcePath)); + File.Move(stagingPath, destinationPath, overwrite: true); + } + catch + { + TryDelete(stagingPath); + throw; + } + } + + + public async Task DownloadRepositoryFileAsync( + string repoId, + string fileName, + string destinationPath, + bool forceRedownload = true, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + await EnsureHubSupportAsync(); + + if (string.IsNullOrWhiteSpace(repoId)) + throw new ArgumentException("Hugging Face repo id is required.", nameof(repoId)); + if (string.IsNullOrWhiteSpace(fileName)) + throw new ArgumentException("Hugging Face file name is required.", nameof(fileName)); + if (string.IsNullOrWhiteSpace(destinationPath)) + throw new ArgumentException("Destination path is required.", nameof(destinationPath)); + + Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!); + + string tempDir = Cache.ExternalBaselineCacheDirectory ?? Cache.MagicQuantDirectory ?? AppContext.BaseDirectory; + Directory.CreateDirectory(tempDir); + + string payloadPath = Path.Combine(tempDir, $"hf_download_file_{Guid.NewGuid():N}.json"); + string resultPath = Path.Combine(tempDir, $"hf_download_file_result_{Guid.NewGuid():N}.json"); + string scriptPath = Path.Combine(tempDir, $"hf_download_file_{Guid.NewGuid():N}.py"); + + try + { + await File.WriteAllTextAsync(payloadPath, JsonSerializer.Serialize(new + { + repo_id = repoId, + file_name = fileName, + destination_path = destinationPath, + force_redownload = forceRedownload + }), ct); + + const string py = """ + import json + import os + import shutil + import sys + from huggingface_hub import hf_hub_download + + payload_path = sys.argv[1] + with open(payload_path, 'r', encoding='utf-8') as f: + payload = json.load(f) + + target_path = payload['destination_path'] + result_path = target_path + '.download_result.json' + os.makedirs(os.path.dirname(target_path), exist_ok=True) + + try: + downloaded = hf_hub_download( + repo_id=payload['repo_id'], + filename=payload['file_name'], + local_dir=os.path.dirname(target_path), + force_download=payload.get('force_redownload', True), + ) + + if os.path.abspath(downloaded) != os.path.abspath(target_path): + if os.path.exists(target_path): + os.remove(target_path) + shutil.copy2(downloaded, target_path) + + result = {'ok': True, 'downloaded_path': target_path, 'size_bytes': os.path.getsize(target_path)} + except Exception as ex: + result = {'ok': False, 'error': str(ex)} + + with open(result_path, 'w', encoding='utf-8') as f: + json.dump(result, f) + """; + + await File.WriteAllTextAsync(scriptPath, py, ct); + await _python.RunPythonScriptAsync(scriptPath, $"\"{payloadPath}\""); + + string pythonResultPath = destinationPath + ".download_result.json"; + File.Move(pythonResultPath, resultPath, overwrite: true); + + var json = JsonDocument.Parse(await File.ReadAllTextAsync(resultPath, ct)).RootElement; + if (!json.GetProperty("ok").GetBoolean()) + throw new InvalidOperationException($"Hugging Face file download failed: {json.GetProperty("error").GetString()}"); + + if (!File.Exists(destinationPath) || new FileInfo(destinationPath).Length == 0) + throw new InvalidOperationException($"Hugging Face file download completed but produced no file: {destinationPath}"); + + AnsiConsole.MarkupLine($"[green]Downloaded repository file:[/] {Markup.Escape(repoId)}/{Markup.Escape(fileName)} -> {Markup.Escape(destinationPath)}"); + return destinationPath; + } + finally + { + TryDelete(payloadPath); + TryDelete(scriptPath); + TryDelete(resultPath); + TryDelete(destinationPath + ".download_result.json"); + } + } + + private async Task EnsureHubSupportAsync() + { + string? version = await _python.GetInstalledVersionAsync("huggingface_hub"); + if (version != null) + return; + + AnsiConsole.MarkupLine("[cyan]Installing huggingface_hub (includes Hub download/CLI support)...[/]"); + await _python.RunPipInstallAsync("--upgrade huggingface_hub"); + } + + private async Task> ListRepoFilesAsync( + string repoId, + string? revision, + CancellationToken ct) + { + string tempDir = Cache.ExternalBaselineCacheDirectory ?? Cache.MagicQuantDirectory ?? AppContext.BaseDirectory; + Directory.CreateDirectory(tempDir); + + string payloadPath = Path.Combine(tempDir, $"hf_repo_list_{Guid.NewGuid():N}.json"); + string resultPath = Path.Combine(tempDir, $"hf_repo_list_result_{Guid.NewGuid():N}.json"); + string scriptPath = Path.Combine(tempDir, $"hf_repo_list_{Guid.NewGuid():N}.py"); + + try + { + await File.WriteAllTextAsync(payloadPath, JsonSerializer.Serialize(new + { + repo_id = repoId, + revision, + result_path = resultPath + }), ct); + + const string py = """ + import json + import sys + from huggingface_hub import HfApi + + payload_path = sys.argv[1] + with open(payload_path, 'r', encoding='utf-8') as f: + payload = json.load(f) + + result_path = payload['result_path'] + try: + files = HfApi().list_repo_files( + repo_id=payload['repo_id'], + revision=payload.get('revision') or None, + ) + result = {'success': True, 'files': files} + except Exception as ex: + result = {'success': False, 'error': str(ex), 'files': []} + + with open(result_path, 'w', encoding='utf-8') as f: + json.dump(result, f) + """; + + await File.WriteAllTextAsync(scriptPath, py, ct); + await _python.RunPythonScriptAsync(scriptPath, $"\"{payloadPath}\""); + + using var doc = JsonDocument.Parse(await File.ReadAllTextAsync(resultPath, ct)); + if (!doc.RootElement.TryGetProperty("success", out var successProp) || !successProp.GetBoolean()) + { + string error = doc.RootElement.TryGetProperty("error", out var errProp) ? errProp.GetString() ?? "unknown error" : "unknown error"; + string revisionSuffix = string.IsNullOrWhiteSpace(revision) ? string.Empty : $" at revision '{revision}'"; + throw new InvalidOperationException($"Failed listing files for Hugging Face repo '{repoId}'{revisionSuffix}: {error}"); + } + + return doc.RootElement.GetProperty("files") + .EnumerateArray() + .Select(x => x.GetString()) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Cast() + .ToList(); + } + finally + { + TryDelete(payloadPath); + TryDelete(scriptPath); + TryDelete(resultPath); + } + } + + private static string ResolveRepoFileName(IReadOnlyList repoFiles, CustomBaselineIncludeConfig include, BaselineQuants standardFamily) + { + if (!string.IsNullOrWhiteSpace(include.FileName)) + { + string explicitName = include.FileName!.Trim(); + var match = repoFiles.FirstOrDefault(x => string.Equals(x, explicitName, StringComparison.OrdinalIgnoreCase)); + if (match == null) + throw new InvalidOperationException($"Configured file '{explicitName}' was not found in the configured custom baseline repository."); + + if (!match.EndsWith(".gguf", StringComparison.OrdinalIgnoreCase)) + throw new InvalidOperationException($"Configured file '{explicitName}' is not a GGUF file."); + + return match; + } + + string family = NormalizeSuffixToken(standardFamily.Names[0]); + var matches = repoFiles + .Where(x => x.EndsWith(".gguf", StringComparison.OrdinalIgnoreCase)) + .Where(x => NormalizeSuffixToken(Path.GetFileNameWithoutExtension(x)).EndsWith(family, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + if (matches.Count == 0) + { + throw new InvalidOperationException( + $"Could not auto-match a GGUF file for baseline family '{standardFamily.Names[0]}'. " + + "Specify file_name explicitly in the YAML."); + } + + if (matches.Count > 1) + { + throw new InvalidOperationException( + $"Auto-match for baseline family '{standardFamily.Names[0]}' returned multiple files: {string.Join(", ", matches)}. " + + "Specify file_name explicitly in the YAML."); + } + + return matches[0]; + } + + private static string BuildCanonicalKey(string architectureFamilyName, string repoId, string fileName) + => BaselineDefinitionResolver.BuildCustomCanonicalKey(architectureFamilyName, repoId, fileName); + + private static string DeriveShortSourceName(string repoId) + { + var owner = DeriveSourceOwner(repoId); + if (string.IsNullOrWhiteSpace(owner)) + return "Custom"; + + return char.ToUpperInvariant(owner[0]) + owner[1..]; + } + + private static string DeriveSourceOwner(string repoId) + { + if (string.IsNullOrWhiteSpace(repoId)) + return string.Empty; + + var parts = repoId.Split('/', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries); + return parts.Length > 0 ? parts[0] : repoId; + } + + private static string NormalizeSuffixToken(string value) + { + return value.Trim() + .Replace("-", "_") + .Replace(" ", string.Empty) + .ToUpperInvariant(); + } + + private static void TryDelete(string path) + { + try + { + if (File.Exists(path)) + File.Delete(path); + } + catch + { + } + } +} diff --git a/src/MagicQuant/Services/HybridArtifactExportService.cs b/src/MagicQuant/Services/HybridArtifactExportService.cs new file mode 100644 index 0000000..648ba0b --- /dev/null +++ b/src/MagicQuant/Services/HybridArtifactExportService.cs @@ -0,0 +1,331 @@ +using MagicQuant.Helpers; +using MagicQuant.Models; +using MagicQuant.Services.Progress; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class HybridArtifactExportService +{ + private static readonly string[] ModelAdjacentFiles = + [ + "generation_config.json", + "config.json", + "chat_template.jinja", + "added_tokenizer.json", + "LICENSE", + "merges.txt", + "model.safetensors.index.json", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json" + ]; + + private readonly QuantizationService _quantizationService; + private readonly EffectiveCandidateStateResolverService _effectiveResolver; + private readonly FinalArtifactNamingService _namingService; + private readonly ModelSidecarArtifactService _sidecarService; + + public HybridArtifactExportService( + QuantizationService quantizationService, + EffectiveCandidateStateResolverService effectiveResolver, + ModelSidecarArtifactService sidecarService) + { + _quantizationService = quantizationService; + _effectiveResolver = effectiveResolver; + _namingService = new FinalArtifactNamingService(); + _sidecarService = sidecarService; + } + + public async Task> ExportAsync( + IReadOnlyCollection selectedRows, + IReadOnlyCollection pureBaselineSnapshots, + CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(Cache.OutputDirectory)) + throw new InvalidOperationException("Cache.OutputDirectory is not set."); + + Directory.CreateDirectory(Cache.OutputDirectory); + await CleanOutputDirectoryAsync(Cache.OutputDirectory!, Config.ReuseExistingFinalArtifacts, ct); + + var output = new List(); + var reservedFileNames = new HashSet(StringComparer.OrdinalIgnoreCase); + var namingContext = _namingService.CreateContext(pureBaselineSnapshots); + + var enabledRows = selectedRows + .Where(x => x.Enabled) + .OrderBy(x => x.Snapshot.Kld) + .ThenBy(x => x.Snapshot.SizeBytes) + .ToList(); + + var localBuilds = new List<(ExportedArtifactRecord Record, HybridQuant Quant, string FullPath, ulong ExpectedBytes)>(); + + foreach (var row in enabledRows) + { + var snap = row.Snapshot; + bool isHybrid = snap.IsHybrid; + bool exportLocally = isHybrid || !snap.IsExternalPureBaseline || Config.ExportExternalLearnedBaselines; + var name = ResolvePlannedOrBuildName(row, snap, namingContext, reservedFileNames); + string provider = ResolveReadmeProviderName(snap, name); + + if (!exportLocally) + { + AnsiConsole.MarkupLine($"[grey]Skipping local export for external learned baseline by default:[/] {Markup.Escape(name.DisplayName)} [grey](enable with --export-external-learned-baselines or output.export_external_learned_baselines: true)[/]"); + + output.Add(new ExportedArtifactRecord + { + Snapshot = snap, + DisplayName = name.DisplayName, + ProviderName = provider, + BaselineFamily = name.QuantFamilyOrBaseline, + IsExternalReference = true, + FileName = name.FileName, + FullPath = null, + DownloadTarget = snap.ExternalRepositoryUrl ?? string.Empty, + ExpectedSizeBytes = snap.SizeBytes, + EffectiveState = await _effectiveResolver.ResolveAsync(snap.Config, ct) + }); + + continue; + } + + if (snap.IsExternalPureBaseline && !snap.IsHybrid) + AnsiConsole.MarkupLine($"[yellow]Local export enabled for external learned baseline:[/] {Markup.Escape(name.DisplayName)}"); + + string fullPath = Path.Combine(Cache.OutputDirectory!, name.FileName); + var record = new ExportedArtifactRecord + { + Snapshot = snap, + DisplayName = name.DisplayName, + ProviderName = provider, + BaselineFamily = name.QuantFamilyOrBaseline, + IsExternalReference = false, + FileName = name.FileName, + FullPath = fullPath, + DownloadTarget = $"./../../resolve/main/{name.FileName}?download=true", + ExpectedSizeBytes = snap.SizeBytes, + EffectiveState = await _effectiveResolver.ResolveAsync(snap.Config, ct) + }; + + if (Config.ReuseExistingFinalArtifacts && + TryReuseExistingFinalArtifact(Cache.OutputDirectory!, row, name.FileName, out var existingFullPath, out var actualSizeBytes)) + { + record.ActualSizeBytes = actualSizeBytes; + AnsiConsole.MarkupLine($"[green]Reused existing final GGUF:[/] {Markup.Escape(existingFullPath)} [grey]({actualSizeBytes:N0} bytes matched benchmark truth)[/]"); + output.Add(record); + continue; + } + + output.Add(record); + localBuilds.Add((record, snap.Quant, fullPath, snap.SizeBytes)); + } + + StageProgressTracker? exportProgress = localBuilds.Count > 0 + ? new StageProgressTracker(new StageProgressOptions + { + StageName = "Final artifact export", + Total = localBuilds.Count, + ShowEta = false, + MinimumPrintInterval = TimeSpan.FromSeconds(5), + UnitLabel = "local GGUF outputs built" + }) + : null; + + // Kick off all exports together. QuantizationService owns the real concurrency gates, + // so this trusts that service to self-regulate CPU/GPU/process pressure. + var buildTasks = localBuilds.Select(async item => + { + string fileName = Path.GetFileName(item.FullPath); + try + { + await _quantizationService.BuildExportArtifactAsync(item.Quant, item.FullPath, forceRebuild: true, ct: ct); + + ulong actualBytes = File.Exists(item.FullPath) ? (ulong)new FileInfo(item.FullPath).Length : 0UL; + item.Record.ActualSizeBytes = actualBytes; + + if (actualBytes != item.ExpectedBytes) + { + AnsiConsole.MarkupLine($"[yellow]Export byte validation warning:[/] expected [cyan]{item.ExpectedBytes:N0}[/] but got [cyan]{actualBytes:N0}[/] for {Markup.Escape(fileName)}"); + } + + exportProgress?.ReportFinished(SampleProcessState.Completed, fileName); + } + catch + { + exportProgress?.ReportFinished(SampleProcessState.Failed, fileName); + throw; + } + }); + + await Task.WhenAll(buildTasks); + + try + { + await CopyModelAdjacentFilesAsync(Cache.OutputDirectory!, ct); + await CopyImatrixArtifactsAsync(Cache.OutputDirectory!, ct); + await _sidecarService.CopyMmprojArtifactsAsync(Cache.OutputDirectory!, ct); + } + finally + { + await CleanExportSidecarsAsync(Cache.OutputDirectory!, ct); + } + + return output; + } + + private FinalArtifactName ResolvePlannedOrBuildName( + FinalSelectionRow row, + BenchmarkSnapshotRecord snapshot, + FinalArtifactNamingContext namingContext, + ISet reservedFileNames) + { + if (!string.IsNullOrWhiteSpace(row.PlannedFileName) && + !string.IsNullOrWhiteSpace(row.PlannedDisplayName)) + { + reservedFileNames.Add(row.PlannedFileName); + return new FinalArtifactName + { + FileName = row.PlannedFileName, + DisplayName = row.PlannedDisplayName, + ShortDisplayName = _namingService.ToShortDisplayName(row.PlannedDisplayName), + ProviderToken = row.PlannedProviderName, + QuantFamilyOrBaseline = row.PlannedQuantFamily + }; + } + + return _namingService.BuildName(snapshot, namingContext, reservedFileNames); + } + + private static string ResolveReadmeProviderName(BenchmarkSnapshotRecord snapshot, FinalArtifactName name) + { + if (snapshot.IsHybrid) + return "MagicQuant"; + + // The artifact filename may contain MQ-* when MagicQuant rebuilt an external + // baseline for equal-footing export. The provider remains the upstream source. + return HybridBenchmarkRepository.ResolveProviderName(snapshot.Quant, exportNaming: false); + } + + private static bool TryReuseExistingFinalArtifact( + string outputDirectory, + FinalSelectionRow row, + string plannedFileName, + out string fullPath, + out ulong actualSizeBytes) + { + actualSizeBytes = 0UL; + fullPath = Path.Combine(outputDirectory, plannedFileName); + + if (string.IsNullOrWhiteSpace(plannedFileName)) + return false; + + string expectedFileName = string.IsNullOrWhiteSpace(row.PlannedFileName) + ? plannedFileName + : row.PlannedFileName.Trim(); + + if (!string.Equals(Path.GetFileName(fullPath), expectedFileName, StringComparison.OrdinalIgnoreCase)) + return false; + + if (!File.Exists(fullPath)) + return false; + + var info = new FileInfo(fullPath); + if (info.Length <= 0) + return false; + + actualSizeBytes = (ulong)info.Length; + return actualSizeBytes == row.Snapshot.SizeBytes; + } + + private static async Task CleanOutputDirectoryAsync(string outputDirectory, bool preserveReusableGgufs, CancellationToken ct) + { + if (!Directory.Exists(outputDirectory)) + { + Directory.CreateDirectory(outputDirectory); + return; + } + + foreach (var file in Directory.EnumerateFiles(outputDirectory, "*", SearchOption.TopDirectoryOnly)) + { + ct.ThrowIfCancellationRequested(); + + if (preserveReusableGgufs && + string.Equals(Path.GetExtension(file), ".gguf", StringComparison.OrdinalIgnoreCase) && + new FileInfo(file).Length > 0) + { + continue; + } + + await HardDeleteHelper.DeleteFileIfExistsAsync(file); + } + + foreach (var directory in Directory.EnumerateDirectories(outputDirectory, "*", SearchOption.TopDirectoryOnly)) + { + ct.ThrowIfCancellationRequested(); + await HardDeleteHelper.DeleteDirectoryIfExistsAsync(directory, ct); + } + + AnsiConsole.MarkupLine(preserveReusableGgufs + ? $"[grey]Cleaned final export directory metadata/non-GGUF files; preserved existing non-empty GGUFs for reuse validation:[/] {Markup.Escape(outputDirectory)}" + : $"[grey]Cleaned final export directory:[/] {Markup.Escape(outputDirectory)}"); + } + + private static async Task CleanExportSidecarsAsync(string outputDirectory, CancellationToken ct) + { + string[] patterns = + [ + "*.success.json", + "*.quantize.log", + "*.convert.log", + "imatrix.success.json", + "imatrix.metadata.json", + "imatrix.build.log" + ]; + + foreach (var pattern in patterns) + { + foreach (var file in Directory.EnumerateFiles(outputDirectory, pattern, SearchOption.TopDirectoryOnly)) + { + ct.ThrowIfCancellationRequested(); + await HardDeleteHelper.DeleteFileIfExistsAsync(file); + } + } + } + + private static async Task CopyModelAdjacentFilesAsync(string outputDirectory, CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(Cache.ModelDirectory)) + return; + + foreach (var fileName in ModelAdjacentFiles) + { + string source = Path.Combine(Cache.ModelDirectory!, fileName); + string target = Path.Combine(outputDirectory, fileName); + + if (!File.Exists(source)) + { + AnsiConsole.MarkupLine($"[grey]Optional model-adjacent file missing:[/] {Markup.Escape(fileName)}"); + continue; + } + + File.Copy(source, target, overwrite: true); + await Task.Yield(); + AnsiConsole.MarkupLine($"[green]Copied model-adjacent file:[/] {Markup.Escape(fileName)}"); + } + } + + private static Task CopyImatrixArtifactsAsync(string outputDirectory, CancellationToken ct) + { + if (!Cache.IsImatrixAvailable || string.IsNullOrWhiteSpace(Cache.ActiveImatrixPath)) + return Task.CompletedTask; + + string source = Cache.ActiveImatrixPath!; + string target = Path.Combine(outputDirectory, "imatrix.dat"); + File.Copy(source, target, overwrite: true); + AnsiConsole.MarkupLine($"[green]Copied imatrix artifact:[/] {Markup.Escape(target)}"); + return Task.CompletedTask; + } + +} \ No newline at end of file diff --git a/src/MagicQuant/Services/HybridBenchmarkRepository.cs b/src/MagicQuant/Services/HybridBenchmarkRepository.cs new file mode 100644 index 0000000..2e56076 --- /dev/null +++ b/src/MagicQuant/Services/HybridBenchmarkRepository.cs @@ -0,0 +1,509 @@ +using MagicQuant.Helpers; +using MagicQuant.Models; +using Microsoft.EntityFrameworkCore; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; + +namespace MagicQuant.Services; + +public sealed class HybridBenchmarkRepository +{ + public async Task> LoadBenchmarkSnapshotsAsync( + IEnumerable configs, + CancellationToken ct = default) + { + var result = new Dictionary(StringComparer.Ordinal); + + foreach (var config in configs.DistinctBy(TensorConfigIdentity.ToKey)) + { + var snapshot = await LoadBenchmarkSnapshotAsync(config, ct); + if (snapshot != null) + result[TensorConfigIdentity.ToKey(config)] = snapshot; + } + + return result; + } + + public async Task LoadBenchmarkSnapshotAsync( + TensorConfig config, + CancellationToken ct = default) + { + await using var db = new MagicQuantContext(); + var scopedAiModelHashId = await ArchitectureFamilyService.ResolveScopedAiModelHashIdOrNullAsync(db, ct); + if (scopedAiModelHashId == null) + return null; + + int? activeImatrixId = await ResolveActiveImatrixIdAsync(db, scopedAiModelHashId.Value, ct); + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + + var query = db.AiBenchmarks + .AsNoTracking() + .Include(x => x.TensorCombo) + .Include(x => x.CategorBenchmarks) + .Where(x => x.ArchitectureFamilyId == architectureFamilyId) + .Where(x => x.TensorGroupProfileId == tensorGroupProfileId) + .Where(x => x.AiModelHashId == scopedAiModelHashId.Value) + .Where(x => x.TensorCombo.BaseQuant == config.BaseQuant) + .Where(x => x.TensorCombo.Embeddings == config.Embeddings) + .Where(x => x.TensorCombo.LmHead == config.LmHead) + .Where(x => x.TensorCombo.AttnQ == config.AttnQ) + .Where(x => x.TensorCombo.AttnKV == config.AttnKV) + .Where(x => x.TensorCombo.AttnOutput == config.AttnOutput) + .Where(x => x.TensorCombo.FfnUpGate == config.FfnUpGate) + .Where(x => x.TensorCombo.FfnDown == config.FfnDown) + .Where(x => x.TensorCombo.MoeExperts == config.MoeExperts) + .Where(x => x.TensorCombo.MoeRouter == config.MoeRouter); + + var rows = await query.ToListAsync(ct); + if (rows.Count == 0) + return null; + + AiBenchmark chosen = rows + .OrderByDescending(x => activeImatrixId != null && x.ImatrixDefinitionId == activeImatrixId.Value) + .ThenByDescending(x => x.ImatrixDefinitionId != null) + .ThenBy(x => x.Id) + .First(); + + var general = chosen.CategorBenchmarks.FirstOrDefault(x => x.Category == (byte)BenchmarkCategory.General) + ?? chosen.CategorBenchmarks.OrderBy(x => x.Category).FirstOrDefault(); + + if (general == null) + return null; + + var quant = (HybridQuant)config; + var sourceBaseline = ResolveSourceBaselineForProvider(quant); + + return new BenchmarkSnapshotRecord + { + Config = config, + Quant = quant, + DisplayName = BuildDisplayName(quant), + ProviderName = ResolveProviderName(quant, exportNaming: false), + BaselineFamily = ResolveBaselineFamily(quant), + IsHybrid = IsTrueMagicQuantHybrid(quant), + IsExternalPureBaseline = quant.Tensors.Count == 0 && sourceBaseline.IsExternalRepositoryBaseline, + IsExternalRebuiltBaseline = IsExternalRebuiltBaseline(quant), + IsMaterializedTensorMapped = quant.Tensors.Count > 0, + SizeBytes = chosen.SizeBytes, + Kld = general.Kld, + Ppl = general.Ppl, + OutputModelPath = await FindLatestSuccessfulOutputPathAsync(config, ct), + ExternalRepositoryUrl = BuildExternalRepositoryUrl(sourceBaseline) + }; + } + + public async Task> LoadPureBaselineSnapshotsAsync(CancellationToken ct = default) + { + var result = new List(); + + foreach (var baseline in BaselineQuants.GetAllRecognizedBaselines()) + { + var snapshot = await LoadBenchmarkSnapshotAsync((TensorConfig)HybridQuant.CreatePureBaseline(baseline), ct); + if (snapshot != null) + result.Add(snapshot); + } + + return result + .GroupBy(x => TensorConfigIdentity.ToKey(x.Config), StringComparer.Ordinal) + .Select(g => g.First()) + .OrderBy(x => x.Kld) + .ThenBy(x => x.SizeBytes) + .ToList(); + } + + + public async Task> LoadBaseOnlyCarrierSnapshotsAsync(CancellationToken ct = default) + { + var result = new List(); + var activeGroups = TReg.All + .Where(x => !Cache.UnusedTensorGroups.Any(u => u.UniqueId == x.UniqueId)) + .OrderBy(x => x.UniqueId) + .ToList(); + var nativeExactScheme = TensorWeightScheme.GetCurrentNativePrecisionScheme(); + + foreach (var baseline in RuntimeSearchSpace.GetActiveCombinationBaselines()) + { + var quant = HybridQuant.CreateExactBlanket( + baseQuant: baseline, + groups: activeGroups, + exactScheme: nativeExactScheme); + + var snapshot = await LoadBenchmarkSnapshotAsync((TensorConfig)quant, ct); + if (snapshot != null) + result.Add(snapshot); + } + + return result + .GroupBy(x => TensorConfigIdentity.ToKey(x.Config), StringComparer.Ordinal) + .Select(g => g.First()) + .OrderByDescending(x => x.Quant.BaseQuant.BitRange) + .ThenBy(x => x.Kld) + .ThenBy(x => x.SizeBytes) + .ToList(); + } + + public async Task FindLatestSuccessfulOutputPathAsync(TensorConfig config, CancellationToken ct = default) + { + await using var db = new MagicQuantContext(); + var scopedAiModelHashId = await ArchitectureFamilyService.ResolveScopedAiModelHashIdOrNullAsync(db, ct); + if (scopedAiModelHashId == null) + return null; + + var tensorComboId = await db.TensorCombos + .AsNoTracking() + .Where(x => x.BaseQuant == config.BaseQuant) + .Where(x => x.Embeddings == config.Embeddings) + .Where(x => x.LmHead == config.LmHead) + .Where(x => x.AttnQ == config.AttnQ) + .Where(x => x.AttnKV == config.AttnKV) + .Where(x => x.AttnOutput == config.AttnOutput) + .Where(x => x.FfnUpGate == config.FfnUpGate) + .Where(x => x.FfnDown == config.FfnDown) + .Where(x => x.MoeExperts == config.MoeExperts) + .Where(x => x.MoeRouter == config.MoeRouter) + .Select(x => (Guid?)x.Id) + .FirstOrDefaultAsync(ct); + + if (tensorComboId == null) + return null; + + int? activeImatrixId = await ResolveActiveImatrixIdAsync(db, scopedAiModelHashId.Value, ct); + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + + return await db.QuantizationRuns + .AsNoTracking() + .Where(x => x.ArchitectureFamilyId == architectureFamilyId) + .Where(x => x.TensorGroupProfileId == tensorGroupProfileId) + .Where(x => x.AiModelHashId == scopedAiModelHashId.Value) + .Where(x => x.ImatrixDefinitionId == activeImatrixId) + .Where(x => x.TensorComboId == tensorComboId.Value) + .Where(x => x.Succeeded) + .OrderByDescending(x => x.CompletedUtc) + .Select(x => x.OutputModelPath) + .FirstOrDefaultAsync(ct); + } + + public async Task> LoadLearnedTensorMappingsAsync( + string canonicalBaselineKey, + byte? groupId = null, + TensorWeightScheme? preferredSourceScheme = null, + bool allowDominantFallback = true, + CancellationToken ct = default) + { + await using var db = new MagicQuantContext(); + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + var normalizedCanonicalKey = BaselineDefinitionResolver.NormalizeCanonicalKey(canonicalBaselineKey); + var baselineDefinitionId = await db.BaselineQuantDefinitions + .AsNoTracking() + .Where(x => (x.ArchitectureFamilyId == architectureFamilyId || x.ArchitectureFamilyId == null) && + x.NormalizedCanonicalKey == normalizedCanonicalKey) + .OrderByDescending(x => x.ArchitectureFamilyId.HasValue) + .Select(x => (int?)x.Id) + .FirstOrDefaultAsync(ct); + + if (!baselineDefinitionId.HasValue) + return new Dictionary(StringComparer.Ordinal); + + var query = db.LearnedBaselineTensorQuants + .AsNoTracking() + .Where(x => x.ArchitectureFamilyId == architectureFamilyId) + .Where(x => x.TensorGroupProfileId == tensorGroupProfileId) + .Where(x => x.BaselineQuantDefinitionId == baselineDefinitionId.Value); + + if (groupId != null) + query = query.Where(x => x.TensorGroupId == groupId.Value); + + var allRows = await query.OrderBy(x => x.TensorName).ToListAsync(ct); + if (allRows.Count == 0) + return new Dictionary(StringComparer.Ordinal); + + var rows = allRows; + + if (preferredSourceScheme != null) + { + var preferred = allRows.Where(x => x.TensorWeightSchemeId == preferredSourceScheme.UniqueId).ToList(); + if (preferred.Count > 0) + rows = preferred; + else if (!allowDominantFallback) + return new Dictionary(StringComparer.Ordinal); + } + + if (rows.Select(x => x.TensorWeightSchemeId).Distinct().Count() > 1) + { + if (!allowDominantFallback) + return new Dictionary(StringComparer.Ordinal); + + var dominant = rows.GroupBy(x => x.TensorWeightSchemeId) + .OrderByDescending(g => g.Count()) + .ThenBy(g => g.Key) + .First() + .Key; + + rows = rows.Where(x => x.TensorWeightSchemeId == dominant).ToList(); + } + + return rows.ToDictionary( + x => x.TensorName, + x => + { + var normalized = NativePrecisionNormalization.NormalizeLearnedFinalQuantTypeForApplication(x.FinalQuantType); + return string.IsNullOrWhiteSpace(normalized) ? x.FinalQuantType : normalized; + }, + StringComparer.Ordinal); + } + + + public async Task> LoadAllBenchmarkSnapshotsForCurrentContextAsync( + byte category = (byte)BenchmarkCategory.General, + bool strictImatrixContext = true, + CancellationToken ct = default) + { + await using var db = new MagicQuantContext(); + var scopedAiModelHashId = await ArchitectureFamilyService.ResolveScopedAiModelHashIdOrNullAsync(db, ct); + if (scopedAiModelHashId == null) + return new List(); + + int? activeImatrixId = await ResolveActiveImatrixIdAsync(db, scopedAiModelHashId.Value, ct); + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + + var query = db.AiBenchmarks + .AsNoTracking() + .Include(x => x.TensorCombo) + .Include(x => x.CategorBenchmarks) + .Where(x => x.ArchitectureFamilyId == architectureFamilyId) + .Where(x => x.TensorGroupProfileId == tensorGroupProfileId) + .Where(x => x.AiModelHashId == scopedAiModelHashId.Value); + + if (strictImatrixContext) + query = query.Where(x => x.ImatrixDefinitionId == activeImatrixId); + + var rows = await query.ToListAsync(ct); + var result = new List(); + + foreach (var benchmark in rows) + { + var metric = benchmark.CategorBenchmarks.FirstOrDefault(x => x.Category == category) + ?? benchmark.CategorBenchmarks.FirstOrDefault(x => x.Category == (byte)BenchmarkCategory.General) + ?? benchmark.CategorBenchmarks.OrderBy(x => x.Category).FirstOrDefault(); + + if (metric == null) + continue; + + var combo = benchmark.TensorCombo; + var config = new TensorConfig( + baseQuant: combo.BaseQuant, + embeddings: combo.Embeddings, + lmHead: combo.LmHead, + attnQ: combo.AttnQ, + attnKV: combo.AttnKV, + attnOutput: combo.AttnOutput, + ffnUpGate: combo.FfnUpGate, + ffnDown: combo.FfnDown, + moeExperts: combo.MoeExperts, + moeRouter: combo.MoeRouter); + + var quant = (HybridQuant)config; + var sourceBaseline = ResolveSourceBaselineForProvider(quant); + + result.Add(new BenchmarkSnapshotRecord + { + Config = config, + Quant = quant, + DisplayName = BuildDisplayName(quant), + ProviderName = ResolveProviderName(quant, exportNaming: false), + BaselineFamily = ResolveBaselineFamily(quant), + IsHybrid = IsTrueMagicQuantHybrid(quant), + IsExternalPureBaseline = quant.Tensors.Count == 0 && sourceBaseline.IsExternalRepositoryBaseline, + IsExternalRebuiltBaseline = IsExternalRebuiltBaseline(quant), + IsMaterializedTensorMapped = quant.Tensors.Count > 0, + SizeBytes = benchmark.SizeBytes, + Kld = metric.Kld, + Ppl = metric.Ppl, + OutputModelPath = await FindLatestSuccessfulOutputPathAsync(config, ct), + ExternalRepositoryUrl = BuildExternalRepositoryUrl(sourceBaseline) + }); + } + + return result + .GroupBy(x => TensorConfigIdentity.ToKey(x.Config), StringComparer.Ordinal) + .Select(g => g.OrderBy(x => x.Kld).ThenBy(x => x.SizeBytes).First()) + .OrderBy(x => x.Kld) + .ThenBy(x => x.SizeBytes) + .ToList(); + } + + public static string ResolveProviderName(HybridQuant quant, bool exportNaming) + { + if (IsTrueMagicQuantHybrid(quant)) + return exportNaming ? "MQ" : "MagicQuant"; + + var baseline = ResolveSourceBaselineForProvider(quant); + if (baseline.IsExternalRepositoryBaseline) + return string.IsNullOrWhiteSpace(baseline.ShortSourceName) + ? "External" + : baseline.ShortSourceName!; + + return string.IsNullOrWhiteSpace(baseline.ShortSourceName) + ? "llama.cpp" + : baseline.ShortSourceName!; + } + + public static string ResolveBaselineFamily(HybridQuant quant) + { + if (IsTrueMagicQuantHybrid(quant)) + return quant.BaseQuant.Names[0]; + + return ResolveSourceBaselineForProvider(quant).Names[0]; + } + + public static BaselineQuants ResolveSourceBaselineForProvider(HybridQuant quant) + { + if (TryResolveUniformExternalLearnedBaseline(quant, out var externalBaseline)) + return externalBaseline; + + return quant.BaseQuant; + } + + public static bool IsExternalRebuiltBaseline(HybridQuant quant) + { + if (IsTrueMagicQuantHybrid(quant)) + return false; + + if (quant.BaseQuant.IsExternalRepositoryBaseline) + return quant.Tensors.Count > 0; + + return TryResolveUniformExternalLearnedBaseline(quant, out _); + } + + public static bool IsTrueMagicQuantHybrid(HybridQuant quant) + { + if (quant.Tensors.Count == 0) + return false; + + var activeTensors = GetActiveTensors(quant).ToList(); + if (activeTensors.Count == 0) + return false; + + if (activeTensors.All(x => x.OverrideMode == HybridTensorOverrideMode.ExactTensorScheme)) + return false; + + if (TryResolveUniformExternalLearnedBaseline(quant, out _)) + return false; + + return true; + } + + private static bool TryResolveUniformExternalLearnedBaseline(HybridQuant quant, out BaselineQuants externalBaseline) + { + externalBaseline = default!; + + var activeGroups = GetActiveGroups().ToList(); + if (activeGroups.Count == 0) + return false; + + var activeTensors = GetActiveTensors(quant).ToList(); + if (activeTensors.Count != activeGroups.Count) + return false; + + if (activeTensors.Any(x => x.OverrideMode != HybridTensorOverrideMode.LearnedBaselineCandidate || x.CandidateBaseline == null)) + return false; + + var candidates = activeTensors + .Select(x => x.CandidateBaseline!) + .ToList(); + + if (candidates.Any(x => !x.IsExternalRepositoryBaseline)) + return false; + + var first = candidates[0]; + bool allSame = candidates.All(x => + x.UniqueId == first.UniqueId || + string.Equals(x.CanonicalKey, first.CanonicalKey, StringComparison.OrdinalIgnoreCase)); + + if (!allSame) + return false; + + externalBaseline = first; + return true; + } + + private static IEnumerable GetActiveTensors(HybridQuant quant) + { + var activeGroupIds = GetActiveGroups() + .Select(x => x.UniqueId) + .ToHashSet(); + + return quant.Tensors + .Where(x => x?.TGroup != null && activeGroupIds.Contains(x.TGroup.UniqueId)); + } + + private static IReadOnlyList GetActiveGroups() + { + return TReg.All + .Where(x => !Cache.UnusedTensorGroups.Any(u => u.UniqueId == x.UniqueId)) + .OrderBy(x => x.UniqueId) + .ToList(); + } + + public static string BuildDisplayName(HybridQuant quant) + { + string modelName = string.IsNullOrWhiteSpace(Cache.ModelDirectory) + ? "model" + : new DirectoryInfo(Cache.ModelDirectory!).Name; + + if (quant.Tensors.Count == 0) + return $"{modelName}-{quant.BaseQuant.Names[0]}"; + + var parts = quant.Tensors + .Where(x => x?.TGroup != null) + .OrderBy(x => x.TGroup.ShortCode) + .Select(x => + { + if (x.OverrideMode == HybridTensorOverrideMode.ExactTensorScheme) + return $"{x.TGroup.ShortCode}-{x.ExactTensorScheme!.Names[0]}"; + + return $"{x.TGroup.ShortCode}-{x.CandidateBaseline!.Names[0]}"; + }); + + return $"{modelName}-{quant.BaseQuant.Names[0]}-{string.Join("-", parts)}"; + } + + public static string? BuildExternalRepositoryUrl(BaselineQuants baseline) + { + if (!baseline.IsExternalRepositoryBaseline) + return null; + + var resolved = Config.GetResolvedCustomBaseline(baseline.CanonicalKey); + if (resolved != null && !string.IsNullOrWhiteSpace(resolved.RepoId)) + { + string repositoryUrl = $"https://huggingface.co/{resolved.RepoId}"; + return string.IsNullOrWhiteSpace(resolved.Revision) + ? repositoryUrl + : $"{repositoryUrl}/tree/{Uri.EscapeDataString(resolved.Revision)}"; + } + + if (!string.IsNullOrWhiteSpace(baseline.SourceRepository)) + return $"https://huggingface.co/{baseline.SourceRepository}"; + + return null; + } + + private static async Task ResolveActiveImatrixIdAsync(MagicQuantContext db, uint scopedAiModelHashId, CancellationToken ct) + { + if (!Cache.IsImatrixAvailable || string.IsNullOrWhiteSpace(Cache.ActiveImatrixIdentityHash)) + return null; + + return await db.ImatrixDefinitions + .AsNoTracking() + .Where(x => x.AiModelHashId == scopedAiModelHashId) + .Where(x => x.IdentityHash == Cache.ActiveImatrixIdentityHash) + .Select(x => (int?)x.Id) + .FirstOrDefaultAsync(ct); + } +} diff --git a/src/MagicQuant/Services/HybridMapGenerationService.cs b/src/MagicQuant/Services/HybridMapGenerationService.cs new file mode 100644 index 0000000..3e82870 --- /dev/null +++ b/src/MagicQuant/Services/HybridMapGenerationService.cs @@ -0,0 +1,57 @@ +using System.Text.Json; +using MagicQuant.Models; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class HybridMapGenerationService +{ + public async Task GenerateAsync( + string outputDirectory, + IReadOnlyCollection exportedArtifacts, + CancellationToken ct = default) + { + string manifestDirectory = MagicQuantManifestPathService.EnsureManifestDirectory(outputDirectory); + string path = Path.Combine(manifestDirectory, MagicQuantManifestPathService.HybridMapFileName); + + var entries = exportedArtifacts + .Where(x => !x.IsExternalReference) + .Where(x => x.Snapshot.IsHybrid) + .Select(x => new HybridMapEntry + { + ExportedFileName = x.FileName ?? string.Empty, + DisplayName = x.DisplayName, + ProviderSource = x.ProviderName, + BaselineFamily = x.BaselineFamily, + OriginalReferenceBaseline = x.Snapshot.Quant.BaseQuant.Names[0], + TensorGroups = x.Snapshot.Quant.Tensors.ToDictionary( + t => t.TGroup.Name, + t => t.OverrideMode == HybridTensorOverrideMode.ExactTensorScheme + ? t.ExactTensorScheme!.Names[0] + : t.CandidateBaseline!.Names[0], + StringComparer.Ordinal), + EffectiveQuantStateKey = x.EffectiveState?.EffectiveStateKey ?? string.Empty, + HasUnknownMappings = x.EffectiveState?.HasUnknownMappings ?? false, + Warnings = x.EffectiveState?.Warnings.ToList() ?? new List(), + UsedImatrix = Cache.UseImatrix && Cache.IsImatrixAvailable, + ExpectedSizeBytes = x.ExpectedSizeBytes, + ExpectedSizeGB = ToGBNumber(x.ExpectedSizeBytes), + ExpectedSizeGiB = ToGiBNumber(x.ExpectedSizeBytes), + ActualSizeBytes = x.ActualSizeBytes, + ActualSizeGB = x.ActualSizeBytes.HasValue ? ToGBNumber(x.ActualSizeBytes.Value) : null, + ActualSizeGiB = x.ActualSizeBytes.HasValue ? ToGiBNumber(x.ActualSizeBytes.Value) : null, + OriginalExternalSource = HybridBenchmarkRepository.BuildExternalRepositoryUrl(x.Snapshot.Quant.BaseQuant) + }) + .ToList(); + + var json = JsonSerializer.Serialize(entries, new JsonSerializerOptions { WriteIndented = true }); + await File.WriteAllTextAsync(path, json, ct); + AnsiConsole.MarkupLine($"[green]Hybrid map JSON generated:[/] {Markup.Escape(path)}"); + return path; + } + + private static double ToGBNumber(ulong bytes) => bytes / 1000d / 1000d / 1000d; + private static double ToGiBNumber(ulong bytes) => bytes / 1024d / 1024d / 1024d; +} \ No newline at end of file diff --git a/src/MagicQuant/Services/ImatrixIdentityService.cs b/src/MagicQuant/Services/ImatrixIdentityService.cs new file mode 100644 index 0000000..d4b8f8a --- /dev/null +++ b/src/MagicQuant/Services/ImatrixIdentityService.cs @@ -0,0 +1,86 @@ +using System.Security.Cryptography; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models.DbModels; +using Microsoft.EntityFrameworkCore; + +namespace MagicQuant.Services; + +public static class ImatrixIdentityService +{ + public static async Task EnsureActiveImatrixIdentityHashAsync(CancellationToken ct = default) + { + if (!Cache.IsImatrixAvailable || string.IsNullOrWhiteSpace(Cache.ActiveImatrixPath)) + { + Cache.ActiveImatrixIdentityHash = null; + return null; + } + + if (!string.IsNullOrWhiteSpace(Cache.ActiveImatrixIdentityHash)) + return Cache.ActiveImatrixIdentityHash; + + await using var stream = File.OpenRead(Cache.ActiveImatrixPath); + var hash = await SHA256.HashDataAsync(stream, ct); + Cache.ActiveImatrixIdentityHash = Convert.ToHexString(hash).ToLowerInvariant(); + return Cache.ActiveImatrixIdentityHash; + } + + public static async Task ResolveCurrentImatrixDefinitionIdAsync( + MagicQuantContext db, + uint aiModelHashId, + bool createIfMissing, + CancellationToken ct = default) + { + var identityHash = await EnsureActiveImatrixIdentityHashAsync(ct); + if (string.IsNullOrWhiteSpace(identityHash)) + return null; + + var existing = await db.ImatrixDefinitions + .FirstOrDefaultAsync(x => x.AiModelHashId == aiModelHashId && x.IdentityHash == identityHash, ct); + + if (existing != null) + return existing.Id; + + if (!createIfMissing) + return null; + + var row = new ImatrixDefinition + { + AiModelHashId = aiModelHashId, + IdentityHash = identityHash, + CanonicalPath = Cache.ActiveImatrixPath, + SourceKind = "runtime-active", + MetadataJson = null, + BuildFingerprint = null, + CreatedUtc = DateTime.UtcNow + }; + + db.ImatrixDefinitions.Add(row); + await db.SaveChangesAsync(ct); + return row.Id; + } + + public static async Task ValidateOwnershipAsync( + MagicQuantContext db, + uint aiModelHashId, + int? imatrixDefinitionId, + CancellationToken ct = default) + { + if (!imatrixDefinitionId.HasValue) + return; + + var ownerHashId = await db.ImatrixDefinitions + .AsNoTracking() + .Where(x => x.Id == imatrixDefinitionId.Value) + .Select(x => (uint?)x.AiModelHashId) + .FirstOrDefaultAsync(ct); + + if (ownerHashId == null || ownerHashId.Value != aiModelHashId) + { + throw new InvalidOperationException( + $"ImatrixDefinitionId {imatrixDefinitionId.Value} does not belong to AiModelHashId {aiModelHashId}. " + + $"Owner AiModelHashId={(ownerHashId.HasValue ? ownerHashId.Value.ToString() : "missing")}. " + + "Imatrix identity is exact-model-hash scoped and must not be resolved through architecture family scope."); + } + } +} diff --git a/src/MagicQuant/Services/ImatrixService.cs b/src/MagicQuant/Services/ImatrixService.cs new file mode 100644 index 0000000..a88cf69 --- /dev/null +++ b/src/MagicQuant/Services/ImatrixService.cs @@ -0,0 +1,936 @@ +using System.Security.Cryptography; +using System.Text.Json; +using System.Text; +using System.Text.RegularExpressions; +using MagicQuant.Helpers; +using MagicQuant.Models; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class ImatrixService +{ + private readonly JsonSerializerOptions _json = new() + { + WriteIndented = true + }; + + public async Task EnsureImatrixAsync(ImatrixRequest request, CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + AnsiConsole.MarkupLine("[grey]Imatrix: starting ensure flow...[/]"); + + if (!request.UseImatrix) + { + AnsiConsole.MarkupLine("[grey]Imatrix: disabled by --use-imatrix flag (false).[/]"); + Cache.IsImatrixAvailable = false; + Cache.ActiveImatrixPath = null; + Cache.ActiveImatrixIdentityHash = null; + RuntimeSearchSpace.SetImatrixAvailability(false); + return new ImatrixEnsureResult { Enabled = false, Available = false }; + } + + ValidateRequest(request, out var sourceKind, out var sourceIdentity); + AnsiConsole.MarkupLine( + $"[grey]Imatrix: validated source mode:[/] [cyan]{Markup.Escape(ToSidecarSourceKind(sourceKind))}[/]"); + + string imatrixDir = Path.Combine(request.MagicQuantDirectory, "imatrix"); + Directory.CreateDirectory(imatrixDir); + AnsiConsole.MarkupLine($"[grey]Imatrix: using directory:[/] [cyan]{Markup.Escape(imatrixDir)}[/]"); + + string datPath = Path.Combine(imatrixDir, "imatrix.dat"); + string successPath = Path.Combine(imatrixDir, "imatrix.success.json"); + string metadataPath = Path.Combine(imatrixDir, "imatrix.metadata.json"); + string buildLogPath = Path.Combine(imatrixDir, "imatrix.build.log"); + + if (request.ForceRebuild) + { + AnsiConsole.MarkupLine("[yellow]Imatrix: force rebuild enabled, cleaning prior canonical artifacts...[/]"); + await CleanupArtifactsAsync(datPath, successPath, metadataPath, buildLogPath); + } + + bool shouldRebuild = await ShouldRebuildAsync(request, sourceKind, datPath, successPath, metadataPath); + + if (shouldRebuild) + { + AnsiConsole.MarkupLine("[grey]Imatrix: canonical artifacts missing/stale/mismatched; rebuilding now...[/]"); + await CleanupArtifactsAsync(datPath, successPath, metadataPath, buildLogPath); + await AcquireImatrixAsync(request, sourceKind, sourceIdentity, datPath, metadataPath, successPath, buildLogPath, ct); + + Cache.IsImatrixAvailable = true; + Cache.ActiveImatrixPath = datPath; + Cache.ActiveImatrixIdentityHash = await ComputeSha256Async(datPath, ct); + RuntimeSearchSpace.SetImatrixAvailability(true); + AnsiConsole.MarkupLine($"[green]Imatrix: ready (rebuilt).[/] [grey]{Markup.Escape(datPath)}[/]"); + + return new ImatrixEnsureResult + { + Enabled = true, + Available = true, + Rebuilt = true, + CanonicalImatrixPath = datPath, + SourceKind = sourceKind + }; + } + + Cache.IsImatrixAvailable = true; + Cache.ActiveImatrixPath = datPath; + Cache.ActiveImatrixIdentityHash = await ComputeSha256Async(datPath, ct); + RuntimeSearchSpace.SetImatrixAvailability(true); + AnsiConsole.MarkupLine($"[green]Imatrix: ready (reused existing trusted artifact).[/] [grey]{Markup.Escape(datPath)}[/]"); + + return new ImatrixEnsureResult + { + Enabled = true, + Available = true, + Rebuilt = false, + CanonicalImatrixPath = datPath, + SourceKind = sourceKind + }; + } + + public bool ShouldUseImatrixForQuant(HybridQuant quant) + { + if (!Cache.UseImatrix || !Cache.IsImatrixAvailable || string.IsNullOrWhiteSpace(Cache.ActiveImatrixPath)) + return false; + + if (quant.BaseQuant.UniqueId == BaselineQuants.NativeSourceUniqueId) + return false; + + var baseScheme = quant.BaseQuant.DefaultTensorScheme; + if (baseScheme != null && TensorWeightScheme.IsNativePrecisionScheme(baseScheme)) + return false; + + return true; + } + + public string GetCanonicalImatrixPath() + { + if (string.IsNullOrWhiteSpace(Cache.ModelMagicQuantDirectory)) + throw new InvalidOperationException("Cache.ModelMagicQuantDirectory is not set."); + + return Path.Combine(Cache.ModelMagicQuantDirectory, "imatrix", "imatrix.dat"); + } + + private static void ValidateRequest(ImatrixRequest request, out ImatrixSourceKind sourceKind, out string sourceIdentity) + { + int activeModes = 0; + + bool urlMode = !string.IsNullOrWhiteSpace(request.ImatrixUrl); + bool hfMode = !string.IsNullOrWhiteSpace(request.DatasetRepo); + bool localMode = !string.IsNullOrWhiteSpace(request.LocalDatasetFile); + + if (urlMode) activeModes++; + if (hfMode) activeModes++; + if (localMode) activeModes++; + + if (activeModes != 1) + { + throw new InvalidOperationException( + "When --use-imatrix is true, exactly one source mode must be provided: --imatrix-url OR --imatrix-dataset-repo OR --imatrix-dataset-local-file."); + } + + if (urlMode) + { + if (!Uri.TryCreate(request.ImatrixUrl, UriKind.Absolute, out var parsed) || + (parsed.Scheme != Uri.UriSchemeHttps && parsed.Scheme != Uri.UriSchemeHttp)) + { + throw new InvalidOperationException("--imatrix-url must be a valid http/https URL."); + } + + sourceKind = ImatrixSourceKind.Https; + sourceIdentity = request.ImatrixUrl!; + return; + } + + if (hfMode) + { + if (string.IsNullOrWhiteSpace(request.DatasetSplit)) + throw new InvalidOperationException("--imatrix-dataset-split is required with --imatrix-dataset-repo."); + + sourceKind = ImatrixSourceKind.HfDataset; + sourceIdentity = $"{request.DatasetRepo}:{request.DatasetConfig ?? "default"}:{request.DatasetSplit}"; + return; + } + + if (string.IsNullOrWhiteSpace(request.LocalDatasetFile)) + throw new InvalidOperationException("--imatrix-dataset-local-file cannot be empty."); + + string ext = Path.GetExtension(request.LocalDatasetFile).ToLowerInvariant(); + if (ext is not ".json" and not ".jsonl") + { + throw new InvalidOperationException( + $"Local dataset file mode supports only .json/.jsonl in MVP. Got '{ext}'. If this is a YAML recipe, add explicit recipe parsing support or use a raw JSON/JSONL corpus file."); + } + + sourceKind = ImatrixSourceKind.LocalDatasetFile; + sourceIdentity = Path.GetFullPath(request.LocalDatasetFile); + } + + private async Task ShouldRebuildAsync( + ImatrixRequest request, + ImatrixSourceKind sourceKind, + string datPath, + string successPath, + string metadataPath) + { + bool hasDat = File.Exists(datPath); + bool hasSuccess = File.Exists(successPath); + + if (hasDat && !hasSuccess) + return true; + + if (!hasDat && hasSuccess) + return true; + + if (!hasDat || !hasSuccess || !File.Exists(metadataPath)) + return true; + + var metadata = JsonSerializer.Deserialize(await File.ReadAllTextAsync(metadataPath), _json); + if (metadata == null) + return true; + + return !MetadataMatchesRequest(metadata, request, sourceKind); + } + + private static bool MetadataMatchesRequest(ImatrixMetadataSidecar metadata, ImatrixRequest request, ImatrixSourceKind sourceKind) + { + if (!string.Equals(metadata.SourceKind, ToSidecarSourceKind(sourceKind), StringComparison.OrdinalIgnoreCase)) + return false; + + string effectiveSplit = GetEffectiveSplitForMetadata(request, sourceKind); + + return sourceKind switch + { + ImatrixSourceKind.Https => string.Equals(metadata.OriginalUrl, request.ImatrixUrl, StringComparison.Ordinal), + ImatrixSourceKind.HfDataset => + string.Equals(metadata.DatasetRepo, request.DatasetRepo, StringComparison.Ordinal) && + string.Equals(metadata.DatasetConfig, request.DatasetConfig, StringComparison.Ordinal) && + string.Equals(metadata.DatasetSplit, effectiveSplit, StringComparison.Ordinal), + ImatrixSourceKind.LocalDatasetFile => + string.Equals(metadata.LocalDatasetFile, Path.GetFullPath(request.LocalDatasetFile!), StringComparison.Ordinal) && + string.Equals(metadata.DatasetSplit, effectiveSplit, StringComparison.Ordinal), + _ => false + }; + } + + private async Task AcquireImatrixAsync( + ImatrixRequest request, + ImatrixSourceKind sourceKind, + string sourceIdentity, + string datPath, + string metadataPath, + string successPath, + string buildLogPath, + CancellationToken ct) + { + AnsiConsole.MarkupLine( + $"[grey]Imatrix: acquiring from source:[/] [cyan]{Markup.Escape(ToSidecarSourceKind(sourceKind))}[/]"); + string effectiveSplit = GetEffectiveSplitForMetadata(request, sourceKind); + + switch (sourceKind) + { + case ImatrixSourceKind.Https: + await AcquireFromHttpsAsync(request, datPath, buildLogPath, ct); + break; + case ImatrixSourceKind.LocalDatasetFile: + await BuildFromLocalDatasetAsync(request, datPath, buildLogPath, ct); + break; + case ImatrixSourceKind.HfDataset: + await BuildFromHfDatasetAsync(request, datPath, buildLogPath, ct); + break; + default: + throw new InvalidOperationException($"Unknown imatrix source kind '{sourceKind}'."); + } + + var metadata = new ImatrixMetadataSidecar + { + SourceKind = ToSidecarSourceKind(sourceKind), + OriginalUrl = request.ImatrixUrl, + OriginalDownloadName = request.ImatrixUrl == null ? null : Path.GetFileName(new Uri(request.ImatrixUrl).AbsolutePath), + DatasetRepo = request.DatasetRepo, + DatasetConfig = request.DatasetConfig, + DatasetSplit = effectiveSplit, + LocalDatasetFile = string.IsNullOrWhiteSpace(request.LocalDatasetFile) ? null : Path.GetFullPath(request.LocalDatasetFile) + }; + + await File.WriteAllTextAsync(metadataPath, JsonSerializer.Serialize(metadata, _json), ct); + + var fileInfo = new FileInfo(datPath); + if (!fileInfo.Exists || fileInfo.Length == 0) + throw new InvalidOperationException("Imatrix acquisition completed but canonical imatrix.dat is missing or empty."); + + var success = new ImatrixSuccessSidecar + { + CompletedUtc = DateTime.UtcNow, + CanonicalPath = datPath, + SourceKind = ToSidecarSourceKind(sourceKind), + SourceIdentity = sourceIdentity, + Split = effectiveSplit, + Config = request.DatasetConfig, + Sha256 = await ComputeSha256Async(datPath, ct), + FileSizeBytes = fileInfo.Length + }; + + await File.WriteAllTextAsync(successPath, JsonSerializer.Serialize(success, _json), ct); + } + + private static async Task CleanupArtifactsAsync(params string[] paths) + { + foreach (var path in paths) + { + if (File.Exists(path)) + await HardDeleteHelper.DeleteFileIfExistsAsync(path); + } + } + + private static string ToSidecarSourceKind(ImatrixSourceKind kind) => kind switch + { + ImatrixSourceKind.Https => "https", + ImatrixSourceKind.HfDataset => "hf_dataset", + ImatrixSourceKind.LocalDatasetFile => "local_dataset_file", + _ => "unknown" + }; + + private static string GetEffectiveSplitForMetadata(ImatrixRequest request, ImatrixSourceKind sourceKind) + { + if (sourceKind == ImatrixSourceKind.LocalDatasetFile) + return string.IsNullOrWhiteSpace(request.DatasetSplit) ? "" : request.DatasetSplit.Trim(); + + return request.DatasetSplit ?? string.Empty; + } + + private static async Task ComputeSha256Async(string path, CancellationToken ct) + { + await using var stream = File.OpenRead(path); + var hash = await SHA256.HashDataAsync(stream, ct); + return Convert.ToHexString(hash).ToLowerInvariant(); + } + + private static async Task AcquireFromHttpsAsync(ImatrixRequest request, string datPath, string buildLogPath, CancellationToken ct) + { + string tempPath = datPath + ".source.tmp"; + AnsiConsole.MarkupLine($"[grey]Imatrix: downloading from URL:[/] [cyan]{Markup.Escape(request.ImatrixUrl ?? string.Empty)}[/]"); + + using var client = new HttpClient(); + await using (var sourceStream = await client.GetStreamAsync(request.ImatrixUrl!, ct)) + await using (var destinationStream = File.Create(tempPath)) + { + await sourceStream.CopyToAsync(destinationStream, ct); + } + + var tempInfo = new FileInfo(tempPath); + if (!tempInfo.Exists || tempInfo.Length == 0) + throw new InvalidOperationException("Downloaded imatrix file is empty."); + + if (File.Exists(datPath)) + File.Delete(datPath); + + File.Move(tempPath, datPath); + await File.WriteAllTextAsync(buildLogPath, $"Downloaded from {request.ImatrixUrl} at {DateTime.UtcNow:O}{Environment.NewLine}", ct); + + AnsiConsole.MarkupLine($"[green]Imatrix downloaded and normalized:[/] {Markup.Escape(datPath)}"); + } + + private async Task BuildFromLocalDatasetAsync(ImatrixRequest request, string datPath, string buildLogPath, CancellationToken ct) + { + string datasetPath = Path.GetFullPath(request.LocalDatasetFile!); + if (!File.Exists(datasetPath)) + throw new FileNotFoundException($"Local dataset file not found: {datasetPath}"); + + AnsiConsole.MarkupLine($"[grey]Imatrix: building from local dataset file:[/] [cyan]{Markup.Escape(datasetPath)}[/]"); + string exportedCorpusPath = Path.Combine(Path.GetDirectoryName(datPath)!, "local-dataset.export.txt"); + string? splitPropertyPath = string.IsNullOrWhiteSpace(request.DatasetSplit) ? null : request.DatasetSplit!.Trim(); + + if (splitPropertyPath != null) + { + AnsiConsole.MarkupLine( + $"[grey]Imatrix: local dataset split/property =[/] [cyan]{Markup.Escape(splitPropertyPath)}[/]"); + AnsiConsole.MarkupLine( + $"[grey]Imatrix: extracting property/path[/] [cyan]{Markup.Escape(splitPropertyPath)}[/] [grey]from local JSON rows.[/]"); + } + else + { + AnsiConsole.MarkupLine( + "[yellow]Imatrix warning:[/] no local split/property provided; using explicit fallback recursive text extraction mode."); + } + + var exportSummary = await ExportLocalDatasetToCorpusAsync( + datasetPath, + exportedCorpusPath, + splitPropertyPath, + buildLogPath, + ct); + + if (exportSummary.StructuredRows > 0) + { + AnsiConsole.MarkupLine( + $"[yellow]Imatrix warning:[/] input appears to be structured JSON rows " + + $"([cyan]{exportSummary.StructuredRows}[/]/[cyan]{exportSummary.TotalRows}[/]). " + + "Flattening extracted text into temporary corpus before llama-imatrix."); + } + + AnsiConsole.MarkupLine( + $"[grey]Imatrix: local dataset export complete.[/] rows=[cyan]{exportSummary.TotalRows}[/], " + + $"structured=[cyan]{exportSummary.StructuredRows}[/], missing_split=[cyan]{exportSummary.RowsMissingSplitProperty}[/], " + + $"extracted_text_blocks=[cyan]{exportSummary.ExtractedTextBlocks}[/], " + + $"corpus=[cyan]{Markup.Escape(exportedCorpusPath)}[/]"); + + await BuildImatrixFromDatasetTextAsync(exportedCorpusPath, datPath, buildLogPath, ct); + } + + private async Task BuildFromHfDatasetAsync(ImatrixRequest request, string datPath, string buildLogPath, CancellationToken ct) + { + string tempJsonl = Path.Combine(Path.GetDirectoryName(datPath)!, "hf-dataset.export.jsonl"); + string python = ResolvePythonExecutableOrThrow(); + string scriptPath = Path.Combine(Path.GetDirectoryName(datPath)!, "build_hf_imatrix_dataset.py"); + AnsiConsole.MarkupLine( + $"[grey]Imatrix: exporting Hugging Face dataset[/] [cyan]{Markup.Escape(request.DatasetRepo ?? string.Empty)}[/]" + + $"[grey] split=[/][cyan]{Markup.Escape(request.DatasetSplit ?? string.Empty)}[/]"); + + string script = """ +import json +from datasets import load_dataset +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('--repo', required=True) +parser.add_argument('--split', required=True) +parser.add_argument('--config', required=False) +parser.add_argument('--out', required=True) +args = parser.parse_args() + +if args.config: + ds = load_dataset(args.repo, args.config, split=args.split) +else: + ds = load_dataset(args.repo, split=args.split) + +with open(args.out, 'w', encoding='utf-8') as f: + for row in ds: + f.write(json.dumps(row, ensure_ascii=False) + '\n') +"""; + + await File.WriteAllTextAsync(scriptPath, script, ct); + + var psi = new System.Diagnostics.ProcessStartInfo + { + FileName = python, + Arguments = + $"\"{scriptPath}\" --repo \"{request.DatasetRepo}\" --split \"{request.DatasetSplit}\" " + + (string.IsNullOrWhiteSpace(request.DatasetConfig) ? string.Empty : $"--config \"{request.DatasetConfig}\" ") + + $"--out \"{tempJsonl}\"", + RedirectStandardError = true, + RedirectStandardOutput = true, + UseShellExecute = false + }; + + using var p = System.Diagnostics.Process.Start(psi) + ?? throw new InvalidOperationException("Failed to start Python process for HF dataset export."); + + string stdout = await p.StandardOutput.ReadToEndAsync(); + string stderr = await p.StandardError.ReadToEndAsync(); + await p.WaitForExitAsync(ct); + + await File.WriteAllTextAsync(buildLogPath, stdout + Environment.NewLine + stderr, ct); + + if (p.ExitCode != 0) + throw new InvalidOperationException("Failed to export HF dataset for imatrix generation. See imatrix.build.log."); + + await BuildImatrixFromDatasetTextAsync(tempJsonl, datPath, buildLogPath, ct); + } + + private static async Task BuildImatrixFromDatasetTextAsync(string datasetPath, string datPath, string buildLogPath, CancellationToken ct) + { + AnsiConsole.MarkupLine( + $"[grey]Imatrix: invoking llama-imatrix build from dataset:[/] [cyan]{Markup.Escape(datasetPath)}[/]"); + AnsiConsole.MarkupLine($"[grey]Imatrix: streaming llama-imatrix output to:[/] [cyan]{Markup.Escape(buildLogPath)}[/]"); + if (File.Exists(datasetPath)) + { + long corpusSizeBytes = new FileInfo(datasetPath).Length; + AnsiConsole.MarkupLine( + $"[grey]Imatrix: exported corpus ready.[/] [cyan]size={Markup.Escape(FormatBytes(corpusSizeBytes))}[/]"); + } + + string llamaBin = Cache.LlamaBin ?? throw new InvalidOperationException("Cache.LlamaBin not set."); + string binaryName = OperatingSystem.IsWindows() ? "llama-imatrix.exe" : "llama-imatrix"; + string imatrixBin = Path.Combine(llamaBin, binaryName); + + if (!File.Exists(imatrixBin)) + throw new InvalidOperationException($"Missing {binaryName}. Cannot build imatrix from dataset sources."); + + string modelName = new DirectoryInfo(Cache.ModelDirectory!).Name; + string torchType = (Cache.TorchType ?? Cache.MainTorchType.BF16).ToString(); + string baseModelPath = Path.Combine(Cache.ModelMagicQuantDirectory!, "GGUF", $"{modelName}-{torchType}.gguf"); + + if (!File.Exists(baseModelPath)) + throw new InvalidOperationException($"Base model GGUF is required before dataset-based imatrix build. Missing: {baseModelPath}"); + + /*var psi = new System.Diagnostics.ProcessStartInfo + { + FileName = imatrixBin, + Arguments = + $"--no-mmap " + + $"-m \"{baseModelPath}\" " + + $"-f \"{datasetPath}\" " + + $"-o \"{datPath}\" ", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + };*/ + + var psi = new System.Diagnostics.ProcessStartInfo + { + FileName = imatrixBin, + Arguments = + $"--no-mmap " + + //$"-ngl 45 " + + //$"--tensor-split 19,22 " + + $"-m \"{baseModelPath}\" " + + $"-f \"{datasetPath}\" " + + $"-o \"{datPath}\" " + + // $"-b 128 " + + // $"-ub 64 " + + $"-fa off", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + + psi.Environment["GGML_CUDA_DISABLE_GRAPHS"] = "1"; + + string launchedCommand = $"\"{imatrixBin}\" {psi.Arguments}"; + + AnsiConsole.MarkupLine($"[grey]Imatrix: launching command:[/] [cyan]{Markup.Escape(launchedCommand)}[/]"); + + using var p = System.Diagnostics.Process.Start(psi) + ?? throw new InvalidOperationException("Failed to start llama-imatrix process."); + + await using var buildLog = new StreamWriter(buildLogPath, append: true, Encoding.UTF8); + await buildLog.WriteLineAsync($"[{DateTime.UtcNow:O}] Launch: {launchedCommand}"); + await buildLog.FlushAsync(); + + using var writeLock = new SemaphoreSlim(1, 1); + var startedUtc = DateTime.UtcNow; + var maxRuntime = TimeSpan.FromHours(168); + int outputLineCount = 0; + bool datDetected = false; + long lastDatSize = -1; + + Task stdoutTask = PumpProcessStreamAsync(p.StandardOutput, "stdout", buildLog, writeLock, line => + { + outputLineCount++; + if (Cache.VerboseProcessOutput) + AnsiConsole.MarkupLine($"[grey]llama-imatrix stdout:[/] {Markup.Escape(line)}"); + }, ct); + + Task stderrTask = PumpProcessStreamAsync(p.StandardError, "stderr", buildLog, writeLock, line => + { + outputLineCount++; + if (Cache.VerboseProcessOutput) + AnsiConsole.MarkupLine($"[grey]llama-imatrix stderr:[/] {Markup.Escape(line)}"); + }, ct); + + while (!p.HasExited) + { + await Task.Delay(TimeSpan.FromSeconds(30), ct); + var elapsed = DateTime.UtcNow - startedUtc; + bool datExists = File.Exists(datPath); + string datSizeText = "n/a"; + if (datExists) + { + long datSize = new FileInfo(datPath).Length; + datSizeText = FormatBytes(datSize); + + if (!datDetected) + { + datDetected = true; + lastDatSize = datSize; + AnsiConsole.MarkupLine($"[green]Imatrix: output file detected:[/] [cyan]{Markup.Escape(datPath)}[/]"); + AnsiConsole.MarkupLine($"[green]Imatrix: output file size now[/] [cyan]{Markup.Escape(datSizeText)}[/]"); + } + else if (datSize != lastDatSize) + { + lastDatSize = datSize; + AnsiConsole.MarkupLine($"[grey]Imatrix: output file size now[/] [cyan]{Markup.Escape(datSizeText)}[/]"); + } + } + + string corpusSizeText = File.Exists(datasetPath) ? FormatBytes(new FileInfo(datasetPath).Length) : "n/a"; + + if (elapsed > maxRuntime) + { + await buildLog.WriteLineAsync($"[{DateTime.UtcNow:O}] Timeout after {elapsed}. Killing llama-imatrix."); + await buildLog.FlushAsync(); + p.Kill(entireProcessTree: true); + throw new TimeoutException( + $"llama-imatrix exceeded safeguard runtime of {maxRuntime}. Process was terminated. See imatrix.build.log."); + } + + AnsiConsole.MarkupLine( + $"[grey]Imatrix: llama-imatrix still running... elapsed[/] [cyan]{elapsed:hh\\:mm\\:ss}[/]" + + $"[grey], output lines[/] [cyan]{outputLineCount}[/]" + + $"[grey], dat_exists[/] [cyan]{datExists}[/]" + + $"[grey], dat_size[/] [cyan]{Markup.Escape(datSizeText)}[/]" + + $"[grey], corpus_size[/] [cyan]{Markup.Escape(corpusSizeText)}[/]" + + $"[grey], log[/] [cyan]{Markup.Escape(buildLogPath)}[/]"); + } + + await Task.WhenAll(stdoutTask, stderrTask); + await p.WaitForExitAsync(ct); + await buildLog.FlushAsync(); + + if (p.ExitCode != 0) + throw new InvalidOperationException("llama-imatrix failed. See imatrix.build.log."); + + AnsiConsole.MarkupLine($"[green]Imatrix: llama-imatrix completed successfully.[/] [grey]exit={p.ExitCode}[/]"); + } + + private static async Task ExportLocalDatasetToCorpusAsync( + string datasetPath, + string exportedCorpusPath, + string? splitPropertyPath, + string buildLogPath, + CancellationToken ct) + { + string ext = Path.GetExtension(datasetPath).ToLowerInvariant(); + if (ext is not ".json" and not ".jsonl") + throw new InvalidOperationException($"Unsupported local dataset extension '{ext}'."); + + int totalRows = 0; + int structuredRows = 0; + int extractedTextBlocks = 0; + int rowsWithMissingSplitProperty = 0; + bool usingSplitProperty = !string.IsNullOrWhiteSpace(splitPropertyPath); + + await using var writer = new StreamWriter(exportedCorpusPath, false, Encoding.UTF8); + + if (ext == ".jsonl") + { + using var reader = new StreamReader(datasetPath, Encoding.UTF8); + while (await reader.ReadLineAsync(ct) is { } line) + { + ct.ThrowIfCancellationRequested(); + if (string.IsNullOrWhiteSpace(line)) + continue; + + totalRows++; + string trimmed = line.TrimStart(); + bool looksStructured = trimmed.StartsWith("{", StringComparison.Ordinal) || trimmed.StartsWith("[", StringComparison.Ordinal); + if (looksStructured) + structuredRows++; + + bool rowMissingRequestedSplit; + foreach (string text in ExtractCorpusTextFromJsonPayload(line, splitPropertyPath, out rowMissingRequestedSplit)) + { + await writer.WriteLineAsync(text); + await writer.WriteLineAsync(); + extractedTextBlocks++; + } + + if (rowMissingRequestedSplit) + rowsWithMissingSplitProperty++; + } + } + else + { + string json = await File.ReadAllTextAsync(datasetPath, ct); + using var doc = JsonDocument.Parse(json); + + if (doc.RootElement.ValueKind is JsonValueKind.Object or JsonValueKind.Array) + structuredRows = 1; + + if (doc.RootElement.ValueKind == JsonValueKind.Array) + { + foreach (var row in doc.RootElement.EnumerateArray()) + { + totalRows++; + bool rowMissingRequestedSplit; + foreach (string text in ExtractCorpusTextFromElement(row, splitPropertyPath, out rowMissingRequestedSplit)) + { + await writer.WriteLineAsync(text); + await writer.WriteLineAsync(); + extractedTextBlocks++; + } + + if (rowMissingRequestedSplit) + rowsWithMissingSplitProperty++; + } + } + else + { + totalRows = 1; + bool rowMissingRequestedSplit; + foreach (string text in ExtractCorpusTextFromElement(doc.RootElement, splitPropertyPath, out rowMissingRequestedSplit)) + { + await writer.WriteLineAsync(text); + await writer.WriteLineAsync(); + extractedTextBlocks++; + } + + if (rowMissingRequestedSplit) + rowsWithMissingSplitProperty++; + } + } + + await writer.FlushAsync(); + + if (usingSplitProperty && extractedTextBlocks == 0) + { + throw new InvalidOperationException( + $"Local dataset split/property '{splitPropertyPath}' was requested but no text could be extracted from '{datasetPath}'. " + + "Verify the property/path exists in your JSON rows."); + } + + if (extractedTextBlocks == 0) + throw new InvalidOperationException( + $"No usable text content was extracted from local dataset file '{datasetPath}'."); + + await File.AppendAllTextAsync( + buildLogPath, + $"[{DateTime.UtcNow:O}] Local dataset export: src={datasetPath}, out={exportedCorpusPath}, " + + $"split_property={(splitPropertyPath ?? "")}, rows={totalRows}, structured_rows={structuredRows}, " + + $"rows_missing_split={rowsWithMissingSplitProperty}, extracted_text_blocks={extractedTextBlocks}{Environment.NewLine}", + ct); + + return new LocalDatasetExportSummary(totalRows, structuredRows, extractedTextBlocks, rowsWithMissingSplitProperty); + } + + private static IEnumerable ExtractCorpusTextFromJsonPayload(string payload, string? splitPropertyPath, out bool missingRequestedSplit) + { + missingRequestedSplit = false; + try + { + using var doc = JsonDocument.Parse(payload); + return ExtractCorpusTextFromElement(doc.RootElement, splitPropertyPath, out missingRequestedSplit).ToList(); + } + catch (JsonException) + { + if (splitPropertyPath == null && LooksLikeUsefulText(payload)) + return new[] { payload.Trim() }; + + if (splitPropertyPath != null) + missingRequestedSplit = true; + + return Array.Empty(); + } + } + + private static IEnumerable ExtractCorpusTextFromElement(JsonElement element, string? splitPropertyPath, out bool missingRequestedSplit) + { + missingRequestedSplit = false; + + if (!string.IsNullOrWhiteSpace(splitPropertyPath)) + { + if (!TryResolveJsonPath(element, splitPropertyPath!, out JsonElement resolved)) + { + missingRequestedSplit = true; + return Array.Empty(); + } + + var pathTexts = new List(); + CollectText(resolved, pathTexts); + return NormalizeDistinct(pathTexts); + } + + var texts = new List(); + CollectText(element, texts); + return NormalizeDistinct(texts); + } + + private static IEnumerable NormalizeDistinct(List texts) + { + // de-dup while preserving order + var seen = new HashSet(StringComparer.Ordinal); + foreach (var text in texts) + { + string normalized = Regex.Replace(text.Trim(), "\\s+", " "); + if (normalized.Length == 0) + continue; + + if (seen.Add(normalized)) + yield return normalized; + } + } + + private static bool TryResolveJsonPath(JsonElement row, string splitPropertyPath, out JsonElement resolved) + { + resolved = row; + foreach (string rawSegment in splitPropertyPath.Split('.', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) + { + if (resolved.ValueKind == JsonValueKind.Object) + { + if (!TryGetPropertyCaseInsensitive(resolved, rawSegment, out resolved)) + return false; + continue; + } + + if (resolved.ValueKind == JsonValueKind.Array) + { + if (int.TryParse(rawSegment, out int index)) + { + if (index < 0 || index >= resolved.GetArrayLength()) + return false; + + resolved = resolved[index]; + continue; + } + + // if segment points to a property on each array element, gather all hits + var hits = new List(); + foreach (var item in resolved.EnumerateArray()) + { + if (item.ValueKind == JsonValueKind.Object && TryGetPropertyCaseInsensitive(item, rawSegment, out JsonElement value)) + hits.Add(value); + } + + if (hits.Count == 0) + return false; + + using var hitsDoc = JsonDocument.Parse(JsonSerializer.Serialize(hits)); + resolved = hitsDoc.RootElement.Clone(); + continue; + } + + return false; + } + + return true; + } + + private static bool TryGetPropertyCaseInsensitive(JsonElement obj, string name, out JsonElement value) + { + foreach (var prop in obj.EnumerateObject()) + { + if (string.Equals(prop.Name, name, StringComparison.OrdinalIgnoreCase)) + { + value = prop.Value; + return true; + } + } + + value = default; + return false; + } + + private static void CollectText(JsonElement element, List sink) + { + switch (element.ValueKind) + { + case JsonValueKind.String: + string value = element.GetString() ?? string.Empty; + if (LooksLikeUsefulText(value)) + sink.Add(value); + break; + case JsonValueKind.Array: + foreach (var item in element.EnumerateArray()) + CollectText(item, sink); + break; + case JsonValueKind.Object: + foreach (var prop in element.EnumerateObject()) + { + if (prop.Value.ValueKind == JsonValueKind.String) + { + string text = prop.Value.GetString() ?? string.Empty; + if (IsLikelyTextFieldName(prop.Name) || LooksLikeUsefulText(text)) + sink.Add(text); + } + else + { + CollectText(prop.Value, sink); + } + } + break; + } + } + + private static bool LooksLikeUsefulText(string value) + { + string trimmed = value.Trim(); + if (trimmed.Length < 4) + return false; + + bool hasLetter = trimmed.Any(char.IsLetter); + bool hasWordBreak = trimmed.Contains(' ') || trimmed.Contains('\t') || trimmed.Contains('\n'); + return hasLetter && (hasWordBreak || trimmed.Length >= 20); + } + + private static bool IsLikelyTextFieldName(string fieldName) => + fieldName.Equals("text", StringComparison.OrdinalIgnoreCase) || + fieldName.Equals("content", StringComparison.OrdinalIgnoreCase) || + fieldName.Equals("prompt", StringComparison.OrdinalIgnoreCase) || + fieldName.Equals("completion", StringComparison.OrdinalIgnoreCase) || + fieldName.Equals("response", StringComparison.OrdinalIgnoreCase) || + fieldName.Equals("instruction", StringComparison.OrdinalIgnoreCase) || + fieldName.Equals("input", StringComparison.OrdinalIgnoreCase) || + fieldName.Equals("output", StringComparison.OrdinalIgnoreCase) || + fieldName.Equals("question", StringComparison.OrdinalIgnoreCase) || + fieldName.Equals("answer", StringComparison.OrdinalIgnoreCase) || + fieldName.Equals("value", StringComparison.OrdinalIgnoreCase) || + fieldName.Equals("body", StringComparison.OrdinalIgnoreCase) || + fieldName.Equals("message", StringComparison.OrdinalIgnoreCase) || + fieldName.Equals("messages", StringComparison.OrdinalIgnoreCase); + + private static async Task PumpProcessStreamAsync( + StreamReader reader, + string label, + StreamWriter buildLog, + SemaphoreSlim writeLock, + Action onLine, + CancellationToken ct) + { + while (true) + { + ct.ThrowIfCancellationRequested(); + string? line = await reader.ReadLineAsync(ct); + if (line == null) + break; + + onLine(line); + await writeLock.WaitAsync(ct); + try + { + await buildLog.WriteLineAsync($"[llama-imatrix {label}] {line}"); + await buildLog.FlushAsync(); + } + finally + { + writeLock.Release(); + } + } + } + + private static string FormatBytes(long sizeBytes) + { + string[] units = ["B", "KB", "MB", "GB", "TB"]; + double size = sizeBytes; + int unit = 0; + while (size >= 1024 && unit < units.Length - 1) + { + size /= 1024; + unit++; + } + + return $"{size:0.0} {units[unit]}"; + } + + private sealed record LocalDatasetExportSummary(int TotalRows, int StructuredRows, int ExtractedTextBlocks, int RowsMissingSplitProperty); + + private static string ResolvePythonExecutableOrThrow() + { + if (string.IsNullOrWhiteSpace(Cache.MagicQuantDirectory)) + throw new InvalidOperationException("Cache.MagicQuantDirectory is not set."); + + var py = new PythonManager(Cache.MagicQuantDirectory); + string pythonExe = py.GetPythonExecutable(); + + if (!File.Exists(pythonExe)) + throw new InvalidOperationException( + $"Python environment is missing or broken at '{pythonExe}'. Re-run initialize-llama-cpp."); + + return pythonExe; + } +} diff --git a/src/MagicQuant/Services/IsolationDiagnosticsManifestService.cs b/src/MagicQuant/Services/IsolationDiagnosticsManifestService.cs new file mode 100644 index 0000000..3a5a0b1 --- /dev/null +++ b/src/MagicQuant/Services/IsolationDiagnosticsManifestService.cs @@ -0,0 +1,281 @@ +using System.Text.Json; +using MagicQuant.Helpers; +using Microsoft.EntityFrameworkCore; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class IsolationDiagnosticsManifestService +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + WriteIndented = true + }; + + public async Task GenerateIsolationSamplesAsync( + string outputDirectory, + RequiredSampleGenerationResult samplePlan, + CancellationToken ct = default) + { + string manifestDirectory = MagicQuantManifestPathService.EnsureManifestDirectory(outputDirectory); + string path = Path.Combine(manifestDirectory, MagicQuantManifestPathService.IsolationSamplesFileName); + + var samples = await BuildIsolationSamplePayloadAsync(samplePlan, ct); + await File.WriteAllTextAsync(path, JsonSerializer.Serialize(samples, JsonOptions), ct); + AnsiConsole.MarkupLine($"[green]Isolation sample JSON generated:[/] {Markup.Escape(path)} samples={samples.Count:N0}"); + return path; + } + + public async Task GenerateBadTradesAsync( + string outputDirectory, + IsolationOptimizationResult isolationResult, + CancellationToken ct = default) + { + string manifestDirectory = MagicQuantManifestPathService.EnsureManifestDirectory(outputDirectory); + string path = Path.Combine(manifestDirectory, MagicQuantManifestPathService.BadTradesFileName); + + var payload = new + { + generatedUtc = DateTime.UtcNow, + modelId = Cache.CurrentModelId, + architectureFamily = Cache.CurrentArchitectureFamilyName, + summary = new + { + badTradeEliminations = isolationResult.BadTradeEliminations, + synergySecondChanceReinstatements = isolationResult.SynergySecondChanceReinstatements, + finalKldCleanupEliminations = isolationResult.FinalKldCleanupEliminations, + disabledBaselines = isolationResult.DisabledBaselines, + structuredBadTradeRows = isolationResult.BadTradeDetails.Count, + structuredSynergySecondChanceRows = isolationResult.SynergySecondChanceDetails.Count + }, + thresholds = new + { + maxSizeDeltaPercent = IsolationPruningConfig.BadTradeMaxSizeDeltaPercent, + kldMultiplier = IsolationPruningConfig.BadTradeKldMultiplier, + pplMultiplier = IsolationPruningConfig.BadTradePplMultiplier, + floatingPointEpsilon = IsolationPruningConfig.FloatingPointEpsilon + }, + badTrades = isolationResult.BadTradeDetails, + synergySecondChances = isolationResult.SynergySecondChanceDetails, + notes = isolationResult.Notes + .Where(x => x.Contains("bad trade", StringComparison.OrdinalIgnoreCase) || + x.Contains("synergy", StringComparison.OrdinalIgnoreCase) || + x.Contains("carrier anchor", StringComparison.OrdinalIgnoreCase) || + x.Contains("combination baseline", StringComparison.OrdinalIgnoreCase)) + .Distinct(StringComparer.Ordinal) + .ToList() + }; + + await File.WriteAllTextAsync(path, JsonSerializer.Serialize(payload, JsonOptions), ct); + AnsiConsole.MarkupLine($"[green]Bad trade JSON generated:[/] {Markup.Escape(path)} records={isolationResult.BadTradeDetails.Count:N0}"); + return path; + } + + private static async Task> BuildIsolationSamplePayloadAsync( + RequiredSampleGenerationResult samplePlan, + CancellationToken ct) + { + await using var db = new MagicQuantContext(); + + var exactAiModelHashId = await ArchitectureFamilyService.ResolveExactCurrentAiModelHashIdOrNullAsync(db, ct); + if (exactAiModelHashId == null) + throw new InvalidOperationException("Cannot export isolation sample manifest because the current exact AiModelHashId could not be resolved."); + + var imatrixDefinitionId = await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync( + db, + exactAiModelHashId.Value, + createIfMissing: false, + ct: ct); + + var nativeSnapshot = await LoadSnapshotAsync(db, exactAiModelHashId.Value, imatrixDefinitionId, (TensorConfig)HybridQuant.CreatePureBaseline(BaselineQuants.GetBF16Quant()), ct); + var nativeByCategory = nativeSnapshot?.Categories.ToDictionary(x => x.CategoryId) ?? new Dictionary(); + + var output = new List(); + + foreach (var plan in samplePlan.Plans + .Where(x => x.Kind is RequiredSampleKind.BaseOnlyIsolation or RequiredSampleKind.GroupIsolationProbe or RequiredSampleKind.GroupIsolationContinuation) + .OrderBy(x => x.Kind) + .ThenBy(x => x.TargetGroupId ?? 0) + .ThenBy(x => x.TestedBaselineId ?? 0) + .ThenBy(x => x.TestedCandidateId ?? 0) + .ThenBy(x => x.Key, StringComparer.Ordinal)) + { + ct.ThrowIfCancellationRequested(); + + var config = (TensorConfig)plan.Quant; + var snapshot = await LoadSnapshotAsync(db, exactAiModelHashId.Value, imatrixDefinitionId, config, ct); + var categories = snapshot?.Categories ?? new List(); + + double? kld = categories.Where(x => x.Kld.HasValue).Select(x => x.Kld!.Value).AverageOrNull(); + double? ppl = categories.Select(x => x.Ppl).AverageOrNull(); + double? pplDelta = CalculateAggregatePplDelta(categories, nativeByCategory); + + output.Add(new + { + key = plan.Key, + description = plan.Description, + kind = plan.Kind.ToString(), + tensorConfigKey = MagicQuant.Models.TensorConfigIdentity.ToKey(config), + group = ResolveGroupName(plan.TargetGroupId), + testedBaseline = ResolveBaselineName(plan.TestedBaselineId), + testedCandidate = ResolveBaselineName(plan.TestedCandidateId), + testedBaselineCanonicalKey = plan.TestedBaselineCanonicalKey, + testedCandidateCanonicalKey = plan.TestedCandidateCanonicalKey, + isSmallestProbe = plan.IsSmallestProbe, + sizeBytes = snapshot?.SizeBytes, + sizeGB = snapshot?.SizeBytes is { } bytes ? ToGBNumber(bytes) : (double?)null, + sizeGiB = snapshot?.SizeBytes is { } gibBytes ? ToGiBNumber(gibBytes) : (double?)null, + kld, + ppl, + pplDeltaPercent = pplDelta, + foundBenchmark = snapshot != null, + categories, + config = new + { + config.BaseQuant, + config.Embeddings, + config.LmHead, + config.AttnQ, + config.AttnKV, + config.AttnOutput, + config.FfnUpGate, + config.FfnDown, + config.MoeExperts, + config.MoeRouter + } + }); + } + + return output; + } + + private static async Task LoadSnapshotAsync( + MagicQuantContext db, + uint aiModelHashId, + int? imatrixDefinitionId, + TensorConfig lookup, + CancellationToken ct) + { + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + + var row = await db.AiBenchmarks + .AsNoTracking() + .Include(x => x.CategorBenchmarks) + .Join(db.TensorCombos, + b => b.TensorComboId, + c => c.Id, + (b, c) => new { b, c }) + .FirstOrDefaultAsync(x => + x.b.ArchitectureFamilyId == architectureFamilyId && + x.b.TensorGroupProfileId == tensorGroupProfileId && + x.b.AiModelHashId == aiModelHashId && + x.b.ImatrixDefinitionId == imatrixDefinitionId && + x.c.BaseQuant == lookup.BaseQuant && + x.c.Embeddings == lookup.Embeddings && + x.c.LmHead == lookup.LmHead && + x.c.AttnQ == lookup.AttnQ && + x.c.AttnKV == lookup.AttnKV && + x.c.AttnOutput == lookup.AttnOutput && + x.c.FfnUpGate == lookup.FfnUpGate && + x.c.FfnDown == lookup.FfnDown && + x.c.MoeExperts == lookup.MoeExperts && + x.c.MoeRouter == lookup.MoeRouter, + ct); + + if (row == null) + return null; + + return new BenchmarkPayload + { + SizeBytes = row.b.SizeBytes, + Categories = row.b.CategorBenchmarks + .OrderBy(x => x.Category) + .Select(x => new CategoryPayload + { + CategoryId = x.Category, + Category = Enum.IsDefined(typeof(BenchmarkCategory), (int)x.Category) + ? ((BenchmarkCategory)x.Category).ToString() + : x.Category.ToString(), + Kld = x.Kld, + Ppl = x.Ppl, + PplError = x.PplError + }) + .ToList() + }; + } + + private static double? CalculateAggregatePplDelta( + IReadOnlyCollection categories, + IReadOnlyDictionary nativeByCategory) + { + var deltas = new List(); + + foreach (var category in categories) + { + if (!nativeByCategory.TryGetValue(category.CategoryId, out var native)) + continue; + + if (native.Ppl <= 0d || category.Ppl <= 0d) + continue; + + deltas.Add(((category.Ppl - native.Ppl) / native.Ppl) * 100d); + } + + return deltas.Count == 0 ? null : deltas.Average(); + } + + private static string? ResolveGroupName(byte? groupId) + { + if (groupId == null) + return null; + + return TReg.All.FirstOrDefault(x => x.UniqueId == groupId.Value)?.Name ?? groupId.Value.ToString(); + } + + private static string? ResolveBaselineName(byte? baselineId) + { + if (baselineId == null) + return null; + + try + { + return BaselineQuants.FromId(baselineId.Value).Names[0]; + } + catch + { + return baselineId.Value.ToString(); + } + } + + private static double ToGBNumber(ulong bytes) => bytes / 1000d / 1000d / 1000d; + private static double ToGiBNumber(ulong bytes) => bytes / 1024d / 1024d / 1024d; + + private sealed class BenchmarkPayload + { + public ulong SizeBytes { get; init; } + public List Categories { get; init; } = new(); + } + + private sealed class CategoryPayload + { + public byte CategoryId { get; init; } + public string Category { get; init; } = string.Empty; + public double? Kld { get; init; } + public double Ppl { get; init; } + public double PplError { get; init; } + } +} + +internal static class MagicQuantEnumerableExtensions +{ + public static double? AverageOrNull(this IEnumerable values) + { + var list = values.Where(x => !double.IsNaN(x) && !double.IsInfinity(x)).ToList(); + return list.Count == 0 ? null : list.Average(); + } +} \ No newline at end of file diff --git a/src/MagicQuant/Services/IsolationOptimizationService.cs b/src/MagicQuant/Services/IsolationOptimizationService.cs new file mode 100644 index 0000000..f47fc04 --- /dev/null +++ b/src/MagicQuant/Services/IsolationOptimizationService.cs @@ -0,0 +1,1449 @@ +using MagicQuant.Helpers; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models; +using Microsoft.EntityFrameworkCore; + +namespace MagicQuant.Services; + +public sealed class IsolationOptimizationOptions +{ + public double MinMeaningfulGroupReductionRatio { get; set; } = + IsolationPruningConfig.MinimumIsolationReductionToContinueRatio; + + public double MinMeaningfulBaseOnlyReductionRatio { get; set; } = + IsolationPruningConfig.MinimumMeaningfulBaseOnlyReductionRatio; +} + +public sealed class IsolationGroupDecision +{ + public string GroupName { get; set; } = string.Empty; + public double BestReductionRatio { get; set; } + + public bool ExplicitQuantBanned { get; set; } + public bool Bf16Suppressed { get; set; } + + public string? WinningCandidate { get; set; } + public ulong? WinningSizeBytes { get; set; } + public double? WinningKld { get; set; } + public double? WinningPplDelta { get; set; } + + public List Candidates { get; set; } = new(); +} + +public sealed class InitialIsolationAnalysisResult +{ + public List GroupsToContinue { get; set; } = new(); + public List Notes { get; set; } = new(); + public List GroupDetails { get; set; } = new(); +} + +public sealed class IsolationOptimizationResult +{ + public int ExplicitQuantBannedGroups { get; set; } + public int DominatedGroupCandidatesBanned { get; set; } + public int HardDamageEliminations { get; set; } + public int BadTradeEliminations { get; set; } + public int FinalKldCleanupEliminations { get; set; } + public int DisabledBaselines { get; set; } + public int Bf16SuppressedGroups { get; set; } + + public int SynergySecondChanceReinstatements { get; set; } + + public List Notes { get; set; } = new(); + public List GroupDetails { get; set; } = new(); + public List BadTradeDetails { get; set; } = new(); + public List SynergySecondChanceDetails { get; set; } = new(); +} + +public sealed class IsolationBadTradeRecord +{ + public string Scope { get; set; } = string.Empty; + public string? GroupName { get; set; } + public string RemovedCandidate { get; set; } = string.Empty; + public string AcceptedAnchor { get; set; } = string.Empty; + public string Reason { get; set; } = string.Empty; + public ulong RemovedSizeBytes { get; set; } + public double RemovedSizeGB { get; set; } + public double RemovedSizeGiB { get; set; } + public double RemovedKld { get; set; } + public double RemovedPplDeltaPercent { get; set; } + public ulong AnchorSizeBytes { get; set; } + public double AnchorSizeGB { get; set; } + public double AnchorSizeGiB { get; set; } + public double AnchorKld { get; set; } + public double AnchorPplDeltaPercent { get; set; } + public double SizeDeltaPercent { get; set; } + public double KldRatio { get; set; } + public double PplAbsRatio { get; set; } +} + +public sealed class IsolationSynergySecondChanceRecord +{ + public string SynergyName { get; set; } = string.Empty; + public string GroupName { get; set; } = string.Empty; + public string PeerGroupName { get; set; } = string.Empty; + public string RestoredCandidate { get; set; } = string.Empty; + public string AcceptedAnchor { get; set; } = string.Empty; + public string OriginalBadTradeReason { get; set; } = string.Empty; + public string SecondChanceReason { get; set; } = string.Empty; + public ulong RestoredSizeBytes { get; set; } + public double RestoredKld { get; set; } + public double RestoredPplDeltaPercent { get; set; } + public ulong AnchorSizeBytes { get; set; } + public double AnchorKld { get; set; } + public double AnchorPplDeltaPercent { get; set; } +} + +public class IsolationOptimizationService +{ + public async Task AnalyzeInitialIsolationProbesAsync( + RequiredSampleGenerationResult plan, + IsolationOptimizationOptions? options = null, + CancellationToken ct = default) + { + options ??= new IsolationOptimizationOptions(); + + var result = new InitialIsolationAnalysisResult(); + + var nativeBaseline = await LoadSnapshotAsync( + HybridQuant.CreatePureBaseline(BaselineQuants.GetBF16Quant()), ct) + ?? throw new InvalidOperationException($"Required native exact baseline benchmark was not found for model '{Cache.CurrentModelId}'."); + + var carrierBaselineId = BaselineQuants.Q8_0.UniqueId; + + var carrierBaseOnlyPlan = plan.Plans.First(x => + x.Kind == RequiredSampleKind.BaseOnlyIsolation && + x.TestedBaselineId == carrierBaselineId && + x.Key.StartsWith("carrier-baseonly:", StringComparison.Ordinal)); + + var carrierBaseOnly = await LoadSnapshotAsync(carrierBaseOnlyPlan.Quant, ct) + ?? throw new InvalidOperationException($"Required carrier base-only benchmark was not found for model '{Cache.CurrentModelId}' and carrier '{BaselineQuants.Q8_0.Names[0]}'."); + + var groupPlans = plan.Plans + .Where(x => x.Kind == RequiredSampleKind.GroupIsolationProbe) + .Where(x => x.TestedBaselineId == carrierBaselineId) + .GroupBy(x => x.TargetGroupId!.Value) + .OrderBy(x => x.Key) + .ToList(); + + foreach (var groupSet in groupPlans) + { + var group = TReg.All.First(x => x.UniqueId == groupSet.Key); + var probe = groupSet.Single(); + + var snap = await LoadSnapshotAsync(probe.Quant, ct); + if (snap == null) + continue; + + var candidate = BaselineQuants.FromId(probe.TestedCandidateId!.Value); + var reduction = ComputeReductionRatio(carrierBaseOnly.SizeBytes, snap.SizeBytes); + var kld = GetAggregateKld(snap); + var pplDelta = GetAggregatePplDeltaPercent(snap, nativeBaseline); + + var decision = new IsolationGroupDecision + { + GroupName = group.Name, + BestReductionRatio = reduction, + WinningCandidate = candidate.Names[0], + WinningSizeBytes = snap.SizeBytes, + WinningKld = kld, + WinningPplDelta = pplDelta + }; + + decision.Candidates.Add( + $"{candidate.Names[0]} | size={(snap.SizeBytes / 1024.0 / 1024.0):F2}MB | savings={reduction:P2} | kld={kld:G6} | pplΔ={pplDelta:F4}%"); + + + if (reduction < options.MinMeaningfulGroupReductionRatio) + { + RuntimeSearchSpace.BanAllExplicitCombinationCandidatesForGroup(group, phase: "InitialProbe", reason: "smallest probe savings below threshold"); + decision.ExplicitQuantBanned = true; + + result.Notes.Add( + $"Early stop for '{group.Name}': smallest baseline-candidate probe '{candidate.Names[0]}' only saved {reduction:P2}, below {options.MinMeaningfulGroupReductionRatio:P2}. Explicit baseline-candidate exploration removed for this group."); + + result.GroupDetails.Add(decision); + continue; + } + + result.GroupsToContinue.Add(group.UniqueId); + + result.Notes.Add( + $"Continuation enabled for '{group.Name}': smallest baseline-candidate probe '{candidate.Names[0]}' saved {reduction:P2}. Candidate-level continuation will honor current runtime bans/prunes."); + + if (reduction >= IsolationPruningConfig.MinimumIsolationReductionToSuppressBf16Ratio) + { + RuntimeSearchSpace.SuppressBf16TensorChoice(group, phase: "InitialProbe", reason: "smallest probe savings exceeded BF16 suppression threshold"); + decision.Bf16Suppressed = true; + + result.Notes.Add( + $"Suppressed BF16 explicit candidate for '{group.Name}' because smallest baseline-candidate probe already saved {reduction:P2}."); + } + + result.Notes.Add($"Early learned-scheme continuation pruning is disabled for '{group.Name}'. All compatible candidates remain available for later pipeline stages."); + + result.GroupDetails.Add(decision); + } + + return result; + } + + public async Task AnalyzeAndApplyFinalAsync( + RequiredSampleGenerationResult fullPlan, + IsolationOptimizationOptions? options = null, + CancellationToken ct = default) + { + options ??= new IsolationOptimizationOptions(); + + var result = new IsolationOptimizationResult(); + + var nativeBaseline = await LoadSnapshotAsync( + HybridQuant.CreatePureBaseline(BaselineQuants.GetBF16Quant()), ct) + ?? throw new InvalidOperationException($"Required native exact baseline benchmark was not found for model '{Cache.CurrentModelId}'."); + + var carrierBaselineId = BaselineQuants.Q8_0.UniqueId; + + var carrierBaseOnlyPlan = fullPlan.Plans.First(x => + x.Kind == RequiredSampleKind.BaseOnlyIsolation && + x.TestedBaselineId == carrierBaselineId && + x.Key.StartsWith("carrier-baseonly:", StringComparison.Ordinal)); + + var carrierBaseOnly = await LoadSnapshotAsync(carrierBaseOnlyPlan.Quant, ct) + ?? throw new InvalidOperationException($"Required carrier base-only benchmark was not found for model '{Cache.CurrentModelId}' and carrier '{BaselineQuants.Q8_0.Names[0]}'."); + + var groupPlans = fullPlan.Plans + .Where(x => x.Kind == RequiredSampleKind.GroupIsolationProbe || + x.Kind == RequiredSampleKind.GroupIsolationContinuation) + .Where(x => x.TestedBaselineId == carrierBaselineId) + .GroupBy(x => x.TargetGroupId!.Value) + .OrderBy(x => x.Key) + .ToList(); + + var groupWorkItems = new List(); + var retainedBadTradeEliminations = new List(); + + foreach (var groupSet in groupPlans) + { + var group = TReg.All.First(x => x.UniqueId == groupSet.Key); + var candidates = new List(); + + foreach (var item in groupSet) + { + var snap = await LoadSnapshotAsync(item.Quant, ct); + if (snap == null) + { + if (MagicQuantDiagnostics.ShouldLogGroup(group)) + MagicQuantDiagnostics.Log("final-load", $"group={group.Name}(id={group.UniqueId}) candidate={BaselineQuants.FromId(item.TestedCandidateId!.Value).Names[0]} key={item.Key} loaded=no"); + continue; + } + + var candidateBaseline = BaselineQuants.FromId(item.TestedCandidateId!.Value); + RuntimeSearchSpace.ClearLearnedBaselinePruneForGroupCandidate(group, candidateBaseline); + + candidates.Add(new GroupCandidateEvaluation + { + Group = group, + CandidateBaseline = candidateBaseline, + SizeBytes = snap.SizeBytes, + SavingsRatio = ComputeReductionRatio(carrierBaseOnly.SizeBytes, snap.SizeBytes), + Kld = GetAggregateKld(snap), + PplDeltaPercent = GetAggregatePplDeltaPercent(snap, nativeBaseline) + }); + if (MagicQuantDiagnostics.ShouldLogGroup(group)) + { + var cb = candidates[^1]; + var allowedNow = RuntimeSearchSpace.GetAllowedRealExplicitCombinationCandidatesForGroup(group).Any(x => x.UniqueId == cb.CandidateBaseline.UniqueId); + MagicQuantDiagnostics.Log("final-load", $"group={group.Name}(id={group.UniqueId}) candidate={cb.CandidateBaseline.Names[0]}(id={cb.CandidateBaseline.UniqueId}) loaded=yes sizeGB={(cb.SizeBytes / 1024d / 1024d / 1024d):F3} savings={cb.SavingsRatio:P2} kld={cb.Kld:G6} pplDelta={cb.PplDeltaPercent:F4}% highPrecision={IsHighPrecisionCandidate(cb.CandidateBaseline)} runtimeBanned={RuntimeSearchSpace.IsCombinationCandidateRuntimeBannedForGroup(group, cb.CandidateBaseline)} allowedNow={allowedNow} key={item.Key}"); + } + } + + if (candidates.Count == 0) + continue; + + var decision = new IsolationGroupDecision + { + GroupName = group.Name, + BestReductionRatio = candidates.Max(x => x.SavingsRatio) + }; + + foreach (var candidate in candidates.ToList()) + { + if (IsHighPrecisionCandidate(candidate.CandidateBaseline)) + continue; + + bool hardFail = + candidate.PplDeltaPercent >= IsolationPruningConfig.MaximumIsolationPplDeltaPercent || + candidate.Kld >= IsolationPruningConfig.MaximumIsolationKld; + + if (!hardFail) + continue; + + RuntimeSearchSpace.BanCombinationCandidateForGroup(group, candidate.CandidateBaseline, phase: "HardDamage", reason: $"kld={candidate.Kld:G6}, pplDelta={candidate.PplDeltaPercent:F4}%"); + result.HardDamageEliminations++; + + result.Notes.Add( + $"Hard damage elimination: '{candidate.CandidateBaseline.Names[0]}' removed for '{group.Name}' " + + $"(savings={candidate.SavingsRatio:P2}, KLD={candidate.Kld:G6}, PPLΔ={candidate.PplDeltaPercent:F4}%)."); + } + + candidates = FilterSurvivors(group, candidates); + ApplyDominanceElimination(group, candidates, result); + candidates = FilterSurvivors(group, candidates); + ApplyBadTradeElimination(group, candidates, result, retainedBadTradeEliminations); + + groupWorkItems.Add(new GroupIsolationWorkItem + { + Group = group, + Candidates = candidates, + Decision = decision + }); + } + + ApplySynergySecondChanceReview(groupWorkItems, retainedBadTradeEliminations, result); + + foreach (var workItem in groupWorkItems) + { + var group = workItem.Group; + var decision = workItem.Decision; + var candidates = FilterSurvivors(group, workItem.Candidates); + + ApplyFinalKldCleanupElimination(group, candidates, result); + candidates = FilterSurvivors(group, candidates); + ApplyEquivalentTruthElimination(group, candidates, result); + + candidates = FilterSurvivors(group, candidates) + .OrderBy(x => x.Kld) + .ThenBy(x => Math.Abs(x.PplDeltaPercent)) + .ThenByDescending(x => x.SavingsRatio) + .ThenByDescending(x => EquivalentTruthSelectionHelper.GetBaselineSafetyRank( + x.CandidateBaseline, + isHybrid: false, + isExternalPureBaseline: x.CandidateBaseline.IsExternalRepositoryBaseline)) + .ThenBy(x => x.CandidateBaseline.Names[0], StringComparer.Ordinal) + .ToList(); + + if (candidates.Count == 0) + { + PopulateFinalGroupFlags(group, decision, result); + result.GroupDetails.Add(decision); + continue; + } + + var winner = candidates.First(); + decision.WinningCandidate = winner.CandidateBaseline.Names[0]; + decision.WinningSizeBytes = winner.SizeBytes; + decision.WinningKld = winner.Kld; + decision.WinningPplDelta = winner.PplDeltaPercent; + PopulateFinalGroupFlags(group, decision, result); + + foreach (var candidate in candidates.OrderBy(x => x.SizeBytes)) + { + decision.Candidates.Add( + $"{candidate.CandidateBaseline.Names[0]} | size={(candidate.SizeBytes / 1024.0 / 1024.0):F2}MB | savings={candidate.SavingsRatio:P2} | kld={candidate.Kld:G6} | pplΔ={candidate.PplDeltaPercent:F4}%"); + } + + var survivorIds = candidates.Select(x => x.CandidateBaseline.UniqueId).ToHashSet(); + foreach (var banInfo in RuntimeSearchSpace.GetLearnedBaselineMissingPrunedCandidatesForGroup(group)) + { + if (survivorIds.Contains(banInfo.Candidate.UniqueId)) + continue; + + string expected = banInfo.ExpectedTensorWeightSchemeIds.Count == 0 + ? "" + : string.Join(", ", banInfo.ExpectedTensorWeightSchemeIds); + string matched = banInfo.MatchedTensorWeightSchemeIds.Count == 0 + ? "" + : string.Join(", ", banInfo.MatchedTensorWeightSchemeIds); + string missing = banInfo.MissingTensorWeightSchemeIds.Count == 0 + ? "" + : string.Join(", ", banInfo.MissingTensorWeightSchemeIds); + + decision.Candidates.Add( + $"[pruned-early] {banInfo.Candidate.Names[0]} removed by learned candidate/group scheme matching " + + $"(expected schemes: {expected}; matched: {matched}; missing: {missing})."); + } + + result.GroupDetails.Add(decision); + } + + var baseOnlyPlans = fullPlan.Plans + .Where(x => x.Kind == RequiredSampleKind.BaseOnlyIsolation) + .Where(x => x.Key.StartsWith("baseonly:", StringComparison.Ordinal)) + .ToList(); + + var baseBaselineCandidates = new List(); + + foreach (var item in baseOnlyPlans) + { + var baseline = BaselineQuants.FromId(item.TestedBaselineId!.Value); + if (!baseline.IsCombinationCarrierCandidate) + continue; + + // IMPORTANT: + // Base-combination carrier isolation must primarily reason from the base-only blanket + // snapshot for the tested carrier, where known groups are forced back to native/BF16 + // and only uncovered tensors remain exposed to the base quant choice. + // + // If a model's group coverage effectively captures everything meaningful, these + // base-only snapshots should tie or nearly tie across carriers. Pure fully-quantized + // baseline artifacts are still useful as reference context, but they must not be used + // as the primary pruning metric here because that would conflate fully-quantized + // baseline quality/size with uncovered-tensor-only carrier isolation truth. + var baseOnlySnap = await LoadSnapshotAsync(item.Quant, ct); + var pureSnap = await LoadSnapshotAsync(HybridQuant.CreatePureBaseline(baseline), ct); + var snap = baseOnlySnap ?? pureSnap; + if (snap == null) + continue; + + var usedBaseOnly = baseOnlySnap != null; + var usedPureFallback = !usedBaseOnly && pureSnap != null; + + baseBaselineCandidates.Add(new BaseBaselineEvaluation + { + Baseline = baseline, + SizeBytes = snap.SizeBytes, + SavingsRatio = ComputeReductionRatio(nativeBaseline.SizeBytes, snap.SizeBytes), + Kld = GetAggregateKld(snap), + PplDeltaPercent = GetAggregatePplDeltaPercent(snap, nativeBaseline), + UsedBaseOnlySnapshot = usedBaseOnly, + UsedPureFallback = usedPureFallback, + PureBaselineSizeBytes = pureSnap?.SizeBytes, + PureBaselineKld = pureSnap == null ? null : GetAggregateKld(pureSnap), + PureBaselinePplDeltaPercent = pureSnap == null ? null : GetAggregatePplDeltaPercent(pureSnap, nativeBaseline) + }); + + if (usedPureFallback) + { + result.Notes.Add( + $"Carrier '{baseline.Names[0]}' fell back to pure-baseline metrics because a base-only isolation snapshot was not found."); + } + else if (pureSnap != null) + { + result.Notes.Add( + $"Carrier '{baseline.Names[0]}' is using base-only isolation metrics for pruning/display; pure-baseline metrics are retained only as reference context."); + } + } + + ApplyBaseBaselineReductionPruning(baseBaselineCandidates, options, result); + ApplyBaseBaselineDominanceElimination(baseBaselineCandidates, result); + ApplyBaseBaselineBadTradeElimination(baseBaselineCandidates, result); + AppendBaseCombinationCarrierDecision(baseBaselineCandidates, result); + + result.ExplicitQuantBannedGroups = RuntimeSearchSpace.GetGroupsWithExplicitQuantBanned().Count; + result.Bf16SuppressedGroups = result.GroupDetails.Count(x => x.Bf16Suppressed); + + return result; + } + + private static void AppendLearnedPrunedCandidates(TensorGroup group, IsolationGroupDecision decision) + { + var learnedPruned = RuntimeSearchSpace.GetLearnedBaselineMissingPrunedCandidatesForGroup(group); + if (learnedPruned.Count == 0) + return; + + foreach (var ban in learnedPruned.OrderBy(x => x.Candidate.ExplicitCandidateSortOrder).ThenBy(x => x.Candidate.UniqueId)) + { + decision.Candidates.Add( + $"[pruned-early] {ban.Candidate.Names[0]} removed by learned-baseline mapping for this group " + + $"(no matching tensor weights in baseline(s): {FormatSchemeNames(ban.ExpectedTensorWeightSchemeIds)})." ); + } + } + + private static void AppendBaseCombinationCarrierDecision( + List candidates, + IsolationOptimizationResult result) + { + if (candidates.Count == 0) + return; + + var decision = new IsolationGroupDecision + { + GroupName = "base_combination_carriers" + }; + + var ordered = candidates + .OrderByDescending(x => x.SavingsRatio) + .ThenBy(x => x.Kld) + .ThenBy(x => Math.Abs(x.PplDeltaPercent)) + .ThenByDescending(x => x.Baseline.BitRange) + .ThenBy(x => x.Baseline.Names[0], StringComparer.Ordinal) + .ToList(); + + var winner = ordered + .Where(x => !RuntimeSearchSpace.IsCombinationBaselineDisabled(x.Baseline)) + .OrderBy(x => x.Kld) + .ThenBy(x => Math.Abs(x.PplDeltaPercent)) + .ThenByDescending(x => x.Baseline.BitRange) + .ThenBy(x => x.Baseline.Names[0], StringComparer.Ordinal) + .FirstOrDefault(); + + if (winner != null) + { + decision.BestReductionRatio = winner.SavingsRatio; + decision.WinningCandidate = winner.Baseline.Names[0]; + decision.WinningSizeBytes = winner.SizeBytes; + decision.WinningKld = winner.Kld; + decision.WinningPplDelta = winner.PplDeltaPercent; + } + + foreach (var candidate in ordered) + { + var state = RuntimeSearchSpace.IsCombinationBaselineDisabled(candidate.Baseline) ? "DISABLED" : "ACTIVE"; + var source = candidate.UsedBaseOnlySnapshot ? "base-only" : (candidate.UsedPureFallback ? "pure-fallback" : "unknown"); + var line = + $"{candidate.Baseline.Names[0]} | size={(candidate.SizeBytes / 1024.0 / 1024.0):F2}MB | savings={candidate.SavingsRatio:P2} | kld={candidate.Kld:G6} | pplΔ={candidate.PplDeltaPercent:F4}% | source={source} | state={state}"; + + if (candidate.PureBaselineSizeBytes.HasValue && candidate.UsedBaseOnlySnapshot) + { + line += + $" | pure-ref={(candidate.PureBaselineSizeBytes.Value / 1024.0 / 1024.0):F2}MB / kld={candidate.PureBaselineKld:G6} / pplΔ={candidate.PureBaselinePplDeltaPercent:F4}%"; + } + + decision.Candidates.Add(line); + } + + foreach (var note in result.Notes.Where(x => x.Contains("combination baseline", StringComparison.OrdinalIgnoreCase) || x.Contains("carrier anchor", StringComparison.OrdinalIgnoreCase)).Distinct()) + { + decision.Candidates.Add($"[pruned-final] {note}"); + } + + result.GroupDetails.Add(decision); + } + + private static string FormatSchemeNames(IEnumerable schemeIds) + { + var names = schemeIds + .Distinct() + .Select(id => TensorWeightScheme.All.FirstOrDefault(x => x.UniqueId == id)?.Names[0] ?? id.ToString()) + .ToList(); + + return names.Count == 0 ? "" : string.Join(", ", names); + } + + private static bool IsHighPrecisionCandidate(BaselineQuants candidate) + => BaselineQuants.IsNativeExactAlias(candidate); + + private static void PopulateFinalGroupFlags(TensorGroup group, IsolationGroupDecision decision, IsolationOptimizationResult result) + { + var (explicitAllowed, bf16Allowed) = RuntimeSearchSpace.GetFinalAllowedQuantFamiliesForGroup(group); + + decision.ExplicitQuantBanned = !explicitAllowed; + decision.Bf16Suppressed = !bf16Allowed; + + if (!explicitAllowed && !bf16Allowed) + { + result.Notes.Add( + $"[invariant-warning] Invalid final quant-family state for '{group.Name}': neither explicit candidate nor BF16 is allowed."); + } + } + + private static List FilterSurvivors(TensorGroup group, List candidates) + { + return candidates + .Where(x => IsHighPrecisionCandidate(x.CandidateBaseline) || + !RuntimeSearchSpace.IsCombinationCandidateRuntimeBannedForGroup(group, x.CandidateBaseline)) + .ToList(); + } + + private static void ApplyDominanceElimination(TensorGroup group, List candidates, IsolationOptimizationResult result) + { + var explicitCandidates = GetActiveExplicitCandidates(group, candidates, phase: "Dominance"); + + for (int i = 0; i < explicitCandidates.Count; i++) + { + for (int j = 0; j < explicitCandidates.Count; j++) + { + if (i == j) + continue; + + var a = explicitCandidates[i]; + var b = explicitCandidates[j]; + + bool sameOrSmaller = a.SizeBytes <= b.SizeBytes; + bool kldNoWorse = a.Kld <= b.Kld + IsolationPruningConfig.FloatingPointEpsilon; + bool pplNoWorse = Math.Abs(a.PplDeltaPercent) <= Math.Abs(b.PplDeltaPercent) + IsolationPruningConfig.FloatingPointEpsilon; + + bool strictlyBetter = + a.Kld + IsolationPruningConfig.FloatingPointEpsilon < b.Kld || + Math.Abs(a.PplDeltaPercent) + IsolationPruningConfig.FloatingPointEpsilon < Math.Abs(b.PplDeltaPercent) || + a.SizeBytes < b.SizeBytes; + + if (sameOrSmaller && kldNoWorse && pplNoWorse && strictlyBetter) + { + if (!RuntimeSearchSpace.IsCombinationCandidateRuntimeBannedForGroup(group, b.CandidateBaseline)) + { + RuntimeSearchSpace.BanCombinationCandidateForGroup(group, b.CandidateBaseline, phase: "Dominance", reason: $"dominated by {a.CandidateBaseline.Names[0]}"); + result.DominatedGroupCandidatesBanned++; + + result.Notes.Add( + $"Dominance elimination: '{b.CandidateBaseline.Names[0]}' removed for '{group.Name}' because '{a.CandidateBaseline.Names[0]}' was same-size-or-smaller and no worse on KLD/PPL."); + } + } + } + } + } + + private static void ApplyBadTradeElimination( + TensorGroup group, + List candidates, + IsolationOptimizationResult result, + List? retainedBadTradeEliminations = null) + { + var activeCandidates = GetActiveExplicitCandidates(group, candidates, phase: "BadTrade"); + if (activeCandidates.Count <= 1) + return; + + var sizeBuckets = BuildSizeBuckets(activeCandidates); + if (sizeBuckets.Count == 0) + return; + + var acceptedAnchor = SelectBestBucketSurvivor(sizeBuckets[0]); + if (acceptedAnchor == null) + return; + + for (int i = 1; i < sizeBuckets.Count; i++) + { + var bucketSurvivors = new List(); + + foreach (var candidate in sizeBuckets[i]) + { + if (RuntimeSearchSpace.IsCombinationCandidateRuntimeBannedForGroup(group, candidate.CandidateBaseline)) + continue; + + if (ShouldEliminateAsBadTrade(acceptedAnchor, candidate, out var reason)) + { + RuntimeSearchSpace.BanCombinationCandidateForGroup(group, candidate.CandidateBaseline, phase: "BadTrade", reason: reason); + result.BadTradeEliminations++; + result.BadTradeDetails.Add(CreateBadTradeRecord( + scope: "group", + groupName: group.Name, + removedName: candidate.CandidateBaseline.Names[0], + anchorName: acceptedAnchor.CandidateBaseline.Names[0], + reason: reason, + removedSizeBytes: candidate.SizeBytes, + removedKld: candidate.Kld, + removedPplDeltaPercent: candidate.PplDeltaPercent, + anchorSizeBytes: acceptedAnchor.SizeBytes, + anchorKld: acceptedAnchor.Kld, + anchorPplDeltaPercent: acceptedAnchor.PplDeltaPercent)); + + retainedBadTradeEliminations?.Add(new SynergyBadTradeElimination + { + Group = group, + Removed = candidate, + Anchor = acceptedAnchor, + Reason = reason + }); + + result.Notes.Add( + $"Bad trade elimination: '{candidate.CandidateBaseline.Names[0]}' removed vs accepted anchor '{acceptedAnchor.CandidateBaseline.Names[0]}' for '{group.Name}'. {reason}"); + continue; + } + + bucketSurvivors.Add(candidate); + } + + var promotedAnchor = SelectBestBucketSurvivor(bucketSurvivors); + if (promotedAnchor != null) + acceptedAnchor = promotedAnchor; + } + } + + + private static void ApplySynergySecondChanceReview( + IReadOnlyList groupWorkItems, + IReadOnlyList badTradeEliminations, + IsolationOptimizationResult result) + { + if (groupWorkItems.Count == 0 || badTradeEliminations.Count == 0) + return; + + var workByGroupId = groupWorkItems.ToDictionary(x => x.Group.UniqueId); + var activeCandidateIdsByGroup = groupWorkItems.ToDictionary( + x => x.Group.UniqueId, + x => FilterSurvivors(x.Group, x.Candidates) + .Where(c => !IsHighPrecisionCandidate(c.CandidateBaseline)) + .Select(c => c.CandidateBaseline.UniqueId) + .ToHashSet()); + + var restoredKeys = new HashSet<(byte GroupId, byte CandidateId)>(); + + foreach (var synergy in TensorGroupSynergies.All) + { + var members = synergy.Groups + .Where(g => workByGroupId.ContainsKey(g.UniqueId)) + .ToList(); + + if (members.Count < 2) + continue; + + foreach (var eliminated in badTradeEliminations + .Where(x => synergy.Contains(x.Group)) + .OrderBy(x => x.Group.UniqueId) + .ThenBy(x => x.Removed.CandidateBaseline.UniqueId)) + { + var group = eliminated.Group; + var candidate = eliminated.Removed; + var candidateId = candidate.CandidateBaseline.UniqueId; + + if (!RuntimeSearchSpace.IsCombinationCandidateRuntimeBannedForGroup(group, candidate.CandidateBaseline)) + continue; + + if (!restoredKeys.Add((group.UniqueId, candidateId))) + continue; + + var peer = members + .Where(g => g.UniqueId != group.UniqueId) + .FirstOrDefault(g => activeCandidateIdsByGroup.TryGetValue(g.UniqueId, out var ids) && ids.Contains(candidateId)); + + if (peer == null) + continue; + + if (ShouldEliminateAsBadTradeIgnoringPpl(eliminated.Anchor, candidate, out var kldOnlyReason)) + { + result.Notes.Add( + $"Synergy second-chance rejected: '{candidate.CandidateBaseline.Names[0]}' remained removed for '{group.Name}' even though it survived in '{peer.Name}' under synergy '{synergy.Name}'. {kldOnlyReason}"); + continue; + } + + string restoreReason = + $"synergy '{synergy.Name}' second chance because '{candidate.CandidateBaseline.Names[0]}' survived in peer group '{peer.Name}' and the original bad-trade decision does not survive KLD-only review vs anchor '{eliminated.Anchor.CandidateBaseline.Names[0]}'"; + + if (!RuntimeSearchSpace.UnbanCombinationCandidateForGroup( + group, + candidate.CandidateBaseline, + phase: "SynergySecondChance", + reason: restoreReason)) + { + continue; + } + + if (activeCandidateIdsByGroup.TryGetValue(group.UniqueId, out var groupIds)) + groupIds.Add(candidateId); + + result.SynergySecondChanceReinstatements++; + result.SynergySecondChanceDetails.Add(new IsolationSynergySecondChanceRecord + { + SynergyName = synergy.Name, + GroupName = group.Name, + PeerGroupName = peer.Name, + RestoredCandidate = candidate.CandidateBaseline.Names[0], + AcceptedAnchor = eliminated.Anchor.CandidateBaseline.Names[0], + OriginalBadTradeReason = eliminated.Reason, + SecondChanceReason = restoreReason, + RestoredSizeBytes = candidate.SizeBytes, + RestoredKld = candidate.Kld, + RestoredPplDeltaPercent = candidate.PplDeltaPercent, + AnchorSizeBytes = eliminated.Anchor.SizeBytes, + AnchorKld = eliminated.Anchor.Kld, + AnchorPplDeltaPercent = eliminated.Anchor.PplDeltaPercent + }); + + result.Notes.Add( + $"Synergy second-chance restored: '{candidate.CandidateBaseline.Names[0]}' restored for '{group.Name}' because it survived in peer group '{peer.Name}' under synergy '{synergy.Name}', and KLD-only bad-trade review did not eliminate it vs anchor '{eliminated.Anchor.CandidateBaseline.Names[0]}'."); + } + } + } + + private static bool ShouldEliminateAsBadTradeIgnoringPpl( + GroupCandidateEvaluation anchor, + GroupCandidateEvaluation candidate, + out string reason) + { + reason = string.Empty; + + if (anchor.SizeBytes <= candidate.SizeBytes) + return false; + + double sizeDeltaPercent = ((double)anchor.SizeBytes - candidate.SizeBytes) / anchor.SizeBytes * 100.0; + if (sizeDeltaPercent > IsolationPruningConfig.BadTradeMaxSizeDeltaPercent) + return false; + + double kldRatio = anchor.Kld <= IsolationPruningConfig.FloatingPointEpsilon + ? double.PositiveInfinity + : candidate.Kld / anchor.Kld; + + bool kldBadTrade = candidate.Kld > anchor.Kld * IsolationPruningConfig.BadTradeKldMultiplier; + if (!kldBadTrade) + return false; + + reason = + $"Reason: small size gain ({sizeDeltaPercent:F2}%) but disproportionate KLD damage after PPL was ignored for synergy review (KLD x{kldRatio:F2})."; + + return true; + } + + + private static void ApplyFinalKldCleanupElimination( + TensorGroup group, + List candidates, + IsolationOptimizationResult result) + { + var activeCandidates = GetActiveExplicitCandidates(group, candidates, phase: "FinalKldCleanup"); + if (activeCandidates.Count <= 1) + return; + + foreach (var candidate in activeCandidates + .OrderByDescending(x => x.SizeBytes) + .ThenByDescending(x => x.Kld) + .ToList()) + { + if (RuntimeSearchSpace.IsCombinationCandidateRuntimeBannedForGroup(group, candidate.CandidateBaseline)) + continue; + + var better = activeCandidates + .Where(x => x.CandidateBaseline.UniqueId != candidate.CandidateBaseline.UniqueId) + .Where(x => !RuntimeSearchSpace.IsCombinationCandidateRuntimeBannedForGroup(group, x.CandidateBaseline)) + .Where(x => x.SizeBytes <= candidate.SizeBytes) + .Where(x => x.Kld + IsolationPruningConfig.FloatingPointEpsilon < candidate.Kld) + .OrderBy(x => x.Kld) + .ThenBy(x => x.SizeBytes) + .ThenBy(x => Math.Abs(x.PplDeltaPercent)) + .ThenByDescending(x => EquivalentTruthSelectionHelper.GetBaselineSafetyRank( + x.CandidateBaseline, + isHybrid: false, + isExternalPureBaseline: x.CandidateBaseline.IsExternalRepositoryBaseline)) + .ThenBy(x => x.CandidateBaseline.Names[0], StringComparer.Ordinal) + .FirstOrDefault(); + + if (better == null) + continue; + + RuntimeSearchSpace.BanCombinationCandidateForGroup( + group, + candidate.CandidateBaseline, + phase: "FinalKldCleanup", + reason: $"same-size-or-larger and higher KLD than {better.CandidateBaseline.Names[0]} after bad-trade anchoring"); + + result.FinalKldCleanupEliminations++; + result.Notes.Add( + $"Final KLD cleanup elimination: '{candidate.CandidateBaseline.Names[0]}' removed for '{group.Name}' because '{better.CandidateBaseline.Names[0]}' was same-size-or-smaller and lower KLD after bad-trade anchoring completed " + + $"(removed size={candidate.SizeBytes:N0}, kld={candidate.Kld:G6}; replacement size={better.SizeBytes:N0}, kld={better.Kld:G6})."); + } + } + + + + private static void ApplyEquivalentTruthElimination( + TensorGroup group, + List candidates, + IsolationOptimizationResult result) + { + var explicitCandidates = GetActiveExplicitCandidates(group, candidates, phase: "EquivalentTruth"); + if (explicitCandidates.Count <= 1) + return; + + var ordered = explicitCandidates + .OrderBy(x => x.SizeBytes) + .ThenBy(x => x.Kld) + .ThenBy(x => Math.Abs(x.PplDeltaPercent)) + .ThenByDescending(x => EquivalentTruthSelectionHelper.GetBaselineSafetyRank( + x.CandidateBaseline, + isHybrid: false, + isExternalPureBaseline: x.CandidateBaseline.IsExternalRepositoryBaseline)) + .ThenBy(x => x.CandidateBaseline.Names[0], StringComparer.Ordinal) + .ToList(); + + var used = new bool[ordered.Count]; + + for (int i = 0; i < ordered.Count; i++) + { + if (used[i]) + continue; + + var seed = ordered[i]; + var tied = new List { seed }; + used[i] = true; + + for (int j = i + 1; j < ordered.Count; j++) + { + if (used[j]) + continue; + + if (!EquivalentTruthSelectionHelper.AreEquivalentTruths( + seed.SizeBytes, + seed.Kld, + Math.Abs(seed.PplDeltaPercent), + ordered[j].SizeBytes, + ordered[j].Kld, + Math.Abs(ordered[j].PplDeltaPercent))) + continue; + + tied.Add(ordered[j]); + used[j] = true; + } + + if (tied.Count == 1) + continue; + + var representative = tied + .OrderByDescending(x => EquivalentTruthSelectionHelper.GetBaselineSafetyRank( + x.CandidateBaseline, + isHybrid: false, + isExternalPureBaseline: x.CandidateBaseline.IsExternalRepositoryBaseline)) + .ThenBy(x => x.CandidateBaseline.Names[0], StringComparer.Ordinal) + .First(); + + foreach (var loser in tied) + { + if (ReferenceEquals(loser, representative)) + continue; + + if (RuntimeSearchSpace.IsCombinationCandidateRuntimeBannedForGroup(group, loser.CandidateBaseline)) + continue; + + RuntimeSearchSpace.BanCombinationCandidateForGroup(group, loser.CandidateBaseline, phase: "EquivalentTruth", reason: $"equivalent to {representative.CandidateBaseline.Names[0]}"); + result.DominatedGroupCandidatesBanned++; + + result.Notes.Add( + $"Equivalent-truth elimination: '{loser.CandidateBaseline.Names[0]}' removed for '{group.Name}' because it had identical measured truth to safer representative '{representative.CandidateBaseline.Names[0]}'."); + } + } + } + + private static List GetActiveExplicitCandidates(TensorGroup group, List candidates, string phase = "Unknown") + { + var active = new List(); + foreach (var candidate in candidates) + { + bool hp = IsHighPrecisionCandidate(candidate.CandidateBaseline); + bool rb = RuntimeSearchSpace.IsCombinationCandidateRuntimeBannedForGroup(group, candidate.CandidateBaseline); + bool include = !hp && !rb; + if (include) + active.Add(candidate); + if (MagicQuantDiagnostics.ShouldLogGroup(group)) + MagicQuantDiagnostics.Log("active-filter", $"phase={phase} group={group.Name}(id={group.UniqueId}) candidate={candidate.CandidateBaseline.Names[0]}(id={candidate.CandidateBaseline.UniqueId}) included={include} highPrecision={hp} runtimeBanned={rb}"); + } + return active; + } + + private static List> BuildSizeBuckets(List candidates) + { + return candidates + .GroupBy(x => x.SizeBytes) + .OrderByDescending(x => x.Key) + .Select(x => x + .OrderBy(c => c.Kld) + .ThenBy(c => Math.Abs(c.PplDeltaPercent)) + .ThenByDescending(c => EquivalentTruthSelectionHelper.GetBaselineSafetyRank(c.CandidateBaseline, isHybrid: false, isExternalPureBaseline: c.CandidateBaseline.IsExternalRepositoryBaseline)) + .ThenBy(c => c.CandidateBaseline.Names[0], StringComparer.Ordinal) + .ToList()) + .ToList(); + } + + private static IsolationBadTradeRecord CreateBadTradeRecord( + string scope, + string? groupName, + string removedName, + string anchorName, + string reason, + ulong removedSizeBytes, + double removedKld, + double removedPplDeltaPercent, + ulong anchorSizeBytes, + double anchorKld, + double anchorPplDeltaPercent) + { + double sizeDeltaPercent = anchorSizeBytes > 0 && anchorSizeBytes > removedSizeBytes + ? ((double)anchorSizeBytes - removedSizeBytes) / anchorSizeBytes * 100.0 + : 0.0; + + double kldRatio = anchorKld <= IsolationPruningConfig.FloatingPointEpsilon + ? double.PositiveInfinity + : removedKld / anchorKld; + + double anchorPplAbs = Math.Abs(anchorPplDeltaPercent); + double removedPplAbs = Math.Abs(removedPplDeltaPercent); + double pplRatio = anchorPplAbs <= IsolationPruningConfig.FloatingPointEpsilon + ? double.PositiveInfinity + : removedPplAbs / anchorPplAbs; + + return new IsolationBadTradeRecord + { + Scope = scope, + GroupName = groupName, + RemovedCandidate = removedName, + AcceptedAnchor = anchorName, + Reason = reason, + RemovedSizeBytes = removedSizeBytes, + RemovedSizeGB = ToGBNumber(removedSizeBytes), + RemovedSizeGiB = ToGiBNumber(removedSizeBytes), + RemovedKld = removedKld, + RemovedPplDeltaPercent = removedPplDeltaPercent, + AnchorSizeBytes = anchorSizeBytes, + AnchorSizeGB = ToGBNumber(anchorSizeBytes), + AnchorSizeGiB = ToGiBNumber(anchorSizeBytes), + AnchorKld = anchorKld, + AnchorPplDeltaPercent = anchorPplDeltaPercent, + SizeDeltaPercent = sizeDeltaPercent, + KldRatio = kldRatio, + PplAbsRatio = pplRatio + }; + } + + private static bool ShouldEliminateAsBadTrade(GroupCandidateEvaluation anchor, GroupCandidateEvaluation candidate, out string reason) + { + reason = string.Empty; + + if (anchor.SizeBytes <= candidate.SizeBytes) + return false; + + double sizeDeltaPercent = ((double)anchor.SizeBytes - candidate.SizeBytes) / anchor.SizeBytes * 100.0; + if (sizeDeltaPercent > IsolationPruningConfig.BadTradeMaxSizeDeltaPercent) + return false; + + double anchorPplAbs = Math.Abs(anchor.PplDeltaPercent); + double candidatePplAbs = Math.Abs(candidate.PplDeltaPercent); + + double kldRatio = anchor.Kld <= IsolationPruningConfig.FloatingPointEpsilon ? double.PositiveInfinity : candidate.Kld / anchor.Kld; + double pplRatio = anchorPplAbs <= IsolationPruningConfig.FloatingPointEpsilon ? double.PositiveInfinity : candidatePplAbs / anchorPplAbs; + + bool kldBadTrade = candidate.Kld > anchor.Kld * IsolationPruningConfig.BadTradeKldMultiplier; + bool pplBadTrade = candidatePplAbs > anchorPplAbs * IsolationPruningConfig.BadTradePplMultiplier; + + bool candidateMeaningfullyBetterKld = + candidate.Kld + IsolationPruningConfig.FloatingPointEpsilon < anchor.Kld * 0.90; + + bool candidateMeaningfullyBetterPpl = + candidatePplAbs + IsolationPruningConfig.FloatingPointEpsilon < anchorPplAbs * 0.90; + + bool mixedTradeoff = + (kldBadTrade && candidateMeaningfullyBetterPpl) || + (pplBadTrade && candidateMeaningfullyBetterKld); + + if (mixedTradeoff || (!kldBadTrade && !pplBadTrade)) + return false; + + reason = + $"Reason: small size gain ({sizeDeltaPercent:F2}%) but disproportionate damage (KLD x{kldRatio:F2}, |PPL| x{pplRatio:F2})."; + + return true; + } + + private static GroupCandidateEvaluation? SelectBestBucketSurvivor(List survivors) + { + return survivors + .OrderBy(x => x.Kld) + .ThenBy(x => Math.Abs(x.PplDeltaPercent)) + .ThenByDescending(x => EquivalentTruthSelectionHelper.GetBaselineSafetyRank(x.CandidateBaseline, isHybrid: false, isExternalPureBaseline: x.CandidateBaseline.IsExternalRepositoryBaseline)) + .ThenBy(x => x.CandidateBaseline.Names[0], StringComparer.Ordinal) + .FirstOrDefault(); + } + + + private async Task LoadSnapshotAsync(HybridQuant quant, CancellationToken ct) + { + await using var db = new MagicQuantContext(); + + var exactAiModelHashId = await ArchitectureFamilyService.ResolveExactCurrentAiModelHashIdOrNullAsync(db, ct); + if (exactAiModelHashId == null) + return null; + + var imatrixDefinitionId = await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, exactAiModelHashId.Value, createIfMissing: false, ct); + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + var lookup = (TensorConfig)quant; + + var row = await db.AiBenchmarks + .Include(x => x.CategorBenchmarks) + .Join(db.TensorCombos, + b => b.TensorComboId, + c => c.Id, + (b, c) => new { b, c }) + .FirstOrDefaultAsync(x => + x.b.ArchitectureFamilyId == architectureFamilyId && + x.b.TensorGroupProfileId == tensorGroupProfileId && + x.b.AiModelHashId == exactAiModelHashId.Value && + x.b.ImatrixDefinitionId == imatrixDefinitionId && + x.c.BaseQuant == lookup.BaseQuant && + x.c.Embeddings == lookup.Embeddings && + x.c.LmHead == lookup.LmHead && + x.c.AttnQ == lookup.AttnQ && + x.c.AttnKV == lookup.AttnKV && + x.c.AttnOutput == lookup.AttnOutput && + x.c.FfnUpGate == lookup.FfnUpGate && + x.c.FfnDown == lookup.FfnDown && + x.c.MoeExperts == lookup.MoeExperts && + x.c.MoeRouter == lookup.MoeRouter, + ct); + + if (row == null) + return null; + + return new BenchmarkSnapshot + { + SizeBytes = row.b.SizeBytes, + Benchmarks = row.b.CategorBenchmarks + .Select(x => new CategorySnapshot + { + Category = x.Category, + Kld = x.Kld, + Ppl = x.Ppl, + PplError = x.PplError + }) + .ToList() + }; + } + + private static double ComputeReductionRatio(ulong baselineBytes, ulong candidateBytes) + { + if (baselineBytes == 0) + return 0d; + + double delta = (double)baselineBytes - (double)candidateBytes; + return delta / (double)baselineBytes; + } + + + private static void ApplyBaseBaselineReductionPruning( + List candidates, + IsolationOptimizationOptions options, + IsolationOptimizationResult result) + { + var activeCandidates = GetActiveBaseBaselineCandidates(candidates); + if (activeCandidates.Count <= 1) + return; + + var belowThreshold = activeCandidates + .Where(x => x.SavingsRatio < options.MinMeaningfulBaseOnlyReductionRatio) + .OrderBy(x => x.Baseline.BitRange) + .ThenBy(x => x.SizeBytes) + .ThenBy(x => x.Baseline.Names[0], StringComparer.Ordinal) + .ToList(); + + if (belowThreshold.Count == 0) + return; + + if (belowThreshold.Count == activeCandidates.Count) + { + var keeper = activeCandidates + .OrderBy(x => x.Kld) + .ThenBy(x => Math.Abs(x.PplDeltaPercent)) + .ThenByDescending(x => x.Baseline.BitRange) + .ThenBy(x => x.Baseline.Names[0], StringComparer.Ordinal) + .First(); + + result.Notes.Add( + $"All active combination baselines had uncovered-tensor reduction below the meaningful threshold of {options.MinMeaningfulBaseOnlyReductionRatio:P2}. " + + $"Base-carrier influence was therefore treated as negligible, and the search was collapsed to the deterministic safe carrier '{keeper.Baseline.Names[0]}'."); + + foreach (var candidate in activeCandidates) + { + if (candidate.Baseline.UniqueId == keeper.Baseline.UniqueId) + continue; + + if (!RuntimeSearchSpace.DisableCombinationBaseline(candidate.Baseline)) + continue; + + result.DisabledBaselines++; + result.Notes.Add( + $"Disabled combination baseline '{candidate.Baseline.Names[0]}' because all surviving carriers were below the meaningful uncovered-tensor threshold and '{keeper.Baseline.Names[0]}' was selected as the deterministic safe representative."); + } + + return; + } + + foreach (var candidate in belowThreshold) + { + if (!RuntimeSearchSpace.DisableCombinationBaseline(candidate.Baseline)) + continue; + + result.DisabledBaselines++; + result.Notes.Add( + $"Disabled combination baseline '{candidate.Baseline.Names[0]}' because uncovered-tensor reduction was only {candidate.SavingsRatio:P2}."); + } + } + + private static void ApplyBaseBaselineDominanceElimination( + List candidates, + IsolationOptimizationResult result) + { + var activeCandidates = GetActiveBaseBaselineCandidates(candidates); + if (activeCandidates.Count <= 1) + return; + + for (int i = 0; i < activeCandidates.Count; i++) + { + for (int j = 0; j < activeCandidates.Count; j++) + { + if (i == j) + continue; + + var a = activeCandidates[i]; + var b = activeCandidates[j]; + + bool sameBitRange = a.Baseline.BitRange == b.Baseline.BitRange; + bool sameSize = a.SizeBytes == b.SizeBytes; + bool sameOrSmaller = a.SizeBytes <= b.SizeBytes; + bool kldNoWorse = a.Kld <= b.Kld + IsolationPruningConfig.FloatingPointEpsilon; + bool pplNoWorse = Math.Abs(a.PplDeltaPercent) <= Math.Abs(b.PplDeltaPercent) + IsolationPruningConfig.FloatingPointEpsilon; + + bool effectivelyTied = + sameSize && + Math.Abs(a.Kld - b.Kld) <= IsolationPruningConfig.FloatingPointEpsilon && + Math.Abs(Math.Abs(a.PplDeltaPercent) - Math.Abs(b.PplDeltaPercent)) <= IsolationPruningConfig.FloatingPointEpsilon; + + // Cross-BitRange ties must be preserved. Those ties are exactly what allows + // downstream range-aware prediction/bucketing to explore multiple size neighborhoods. + if (effectivelyTied && !sameBitRange) + continue; + + bool deterministicSameBucketTieWinner = + effectivelyTied && + sameBitRange && + string.Compare(a.Baseline.CanonicalKey, b.Baseline.CanonicalKey, StringComparison.Ordinal) < 0; + + bool strictlyBetter = + a.Kld + IsolationPruningConfig.FloatingPointEpsilon < b.Kld || + Math.Abs(a.PplDeltaPercent) + IsolationPruningConfig.FloatingPointEpsilon < Math.Abs(b.PplDeltaPercent) || + a.SizeBytes < b.SizeBytes || + deterministicSameBucketTieWinner; + + if (!sameOrSmaller || !kldNoWorse || !pplNoWorse || !strictlyBetter) + continue; + + if (!RuntimeSearchSpace.DisableCombinationBaseline(b.Baseline)) + continue; + + result.DisabledBaselines++; + + if (deterministicSameBucketTieWinner) + { + result.Notes.Add( + $"Disabled same-BitRange tied combination baseline '{b.Baseline.Names[0]}' because '{a.Baseline.Names[0]}' was chosen as the deterministic representative for BitRange {a.Baseline.BitRange}."); + } + else + { + result.Notes.Add( + $"Disabled combination baseline '{b.Baseline.Names[0]}' because '{a.Baseline.Names[0]}' was same-size-or-smaller and no worse on KLD/PPL."); + } + } + } + } + + private static void ApplyBaseBaselineBadTradeElimination( + List candidates, + IsolationOptimizationResult result) + { + var activeCandidates = GetActiveBaseBaselineCandidates(candidates); + if (activeCandidates.Count <= 1) + return; + + var sizeBuckets = BuildBaseBaselineSizeBuckets(activeCandidates); + if (sizeBuckets.Count == 0) + return; + + var acceptedAnchor = SelectBestBaseBaselineBucketSurvivor(sizeBuckets[0]); + if (acceptedAnchor == null) + return; + + for (int i = 1; i < sizeBuckets.Count; i++) + { + var bucketSurvivors = new List(); + + foreach (var candidate in sizeBuckets[i]) + { + if (RuntimeSearchSpace.IsCombinationBaselineDisabled(candidate.Baseline)) + continue; + + if (ShouldEliminateBaseBaselineAsBadTrade(acceptedAnchor, candidate, out var reason)) + { + if (RuntimeSearchSpace.DisableCombinationBaseline(candidate.Baseline)) + { + result.DisabledBaselines++; + result.BadTradeDetails.Add(CreateBadTradeRecord( + scope: "base-baseline", + groupName: null, + removedName: candidate.Baseline.Names[0], + anchorName: acceptedAnchor.Baseline.Names[0], + reason: reason, + removedSizeBytes: candidate.SizeBytes, + removedKld: candidate.Kld, + removedPplDeltaPercent: candidate.PplDeltaPercent, + anchorSizeBytes: acceptedAnchor.SizeBytes, + anchorKld: acceptedAnchor.Kld, + anchorPplDeltaPercent: acceptedAnchor.PplDeltaPercent)); + result.Notes.Add( + $"Disabled combination baseline '{candidate.Baseline.Names[0]}' vs accepted carrier anchor '{acceptedAnchor.Baseline.Names[0]}'. {reason}"); + } + + continue; + } + + bucketSurvivors.Add(candidate); + } + + var promotedAnchor = SelectBestBaseBaselineBucketSurvivor(bucketSurvivors); + if (promotedAnchor != null) + acceptedAnchor = promotedAnchor; + } + } + + private static List GetActiveBaseBaselineCandidates(List candidates) + { + return candidates + .Where(x => !RuntimeSearchSpace.IsCombinationBaselineDisabled(x.Baseline)) + .OrderByDescending(x => x.Baseline.BitRange) + .ThenBy(x => x.SizeBytes) + .ThenBy(x => x.Kld) + .ThenBy(x => Math.Abs(x.PplDeltaPercent)) + .ToList(); + } + + private static List> BuildBaseBaselineSizeBuckets(List candidates) + { + return candidates + .GroupBy(x => x.SizeBytes) + .OrderByDescending(x => x.Key) + .Select(x => x + .OrderBy(c => c.Kld) + .ThenBy(c => Math.Abs(c.PplDeltaPercent)) + .ThenByDescending(c => c.Baseline.BitRange) + .ThenBy(c => c.Baseline.Names[0], StringComparer.Ordinal) + .ToList()) + .ToList(); + } + + private static BaseBaselineEvaluation? SelectBestBaseBaselineBucketSurvivor(List survivors) + { + return survivors + .OrderBy(x => x.Kld) + .ThenBy(x => Math.Abs(x.PplDeltaPercent)) + .ThenByDescending(x => x.Baseline.BitRange) + .ThenBy(x => x.Baseline.Names[0], StringComparer.Ordinal) + .FirstOrDefault(); + } + + private static bool ShouldEliminateBaseBaselineAsBadTrade( + BaseBaselineEvaluation anchor, + BaseBaselineEvaluation candidate, + out string reason) + { + reason = string.Empty; + + if (anchor.SizeBytes <= candidate.SizeBytes) + return false; + + double sizeDeltaPercent = ((double)anchor.SizeBytes - candidate.SizeBytes) / anchor.SizeBytes * 100.0; + if (sizeDeltaPercent > IsolationPruningConfig.BadTradeMaxSizeDeltaPercent) + return false; + + double anchorPplAbs = Math.Abs(anchor.PplDeltaPercent); + double candidatePplAbs = Math.Abs(candidate.PplDeltaPercent); + + double kldRatio = anchor.Kld <= IsolationPruningConfig.FloatingPointEpsilon + ? double.PositiveInfinity + : candidate.Kld / anchor.Kld; + + double pplRatio = anchorPplAbs <= IsolationPruningConfig.FloatingPointEpsilon + ? double.PositiveInfinity + : candidatePplAbs / anchorPplAbs; + + bool kldBadTrade = candidate.Kld > anchor.Kld * IsolationPruningConfig.BadTradeKldMultiplier; + bool pplBadTrade = candidatePplAbs > anchorPplAbs * IsolationPruningConfig.BadTradePplMultiplier; + + bool candidateMeaningfullyBetterKld = + candidate.Kld + IsolationPruningConfig.FloatingPointEpsilon < anchor.Kld * 0.90; + + bool candidateMeaningfullyBetterPpl = + candidatePplAbs + IsolationPruningConfig.FloatingPointEpsilon < anchorPplAbs * 0.90; + + bool mixedTradeoff = + (kldBadTrade && candidateMeaningfullyBetterPpl) || + (pplBadTrade && candidateMeaningfullyBetterKld); + + if (mixedTradeoff || (!kldBadTrade && !pplBadTrade)) + return false; + + reason = + $"Reason: small size gain ({sizeDeltaPercent:F2}%) but disproportionate damage (KLD x{kldRatio:F2}, |PPL| x{pplRatio:F2})."; + + return true; + } + + private static double ToGBNumber(ulong bytes) => bytes / 1000d / 1000d / 1000d; + private static double ToGiBNumber(ulong bytes) => bytes / 1024d / 1024d / 1024d; + + private static double GetAggregateKld(BenchmarkSnapshot snapshot) + { + return snapshot.Benchmarks + .Where(x => x.Kld.HasValue) + .Select(x => x.Kld!.Value) + .DefaultIfEmpty(double.PositiveInfinity) + .Average(); + } + + private static double GetAggregatePplDeltaPercent(BenchmarkSnapshot snapshot, BenchmarkSnapshot nativeBaseline) + { + var nativeMap = nativeBaseline.Benchmarks.ToDictionary(x => x.Category); + var deltas = new List(); + + foreach (var bench in snapshot.Benchmarks) + { + if (!nativeMap.TryGetValue(bench.Category, out var native)) + continue; + + if (native.Ppl <= IsolationPruningConfig.FloatingPointEpsilon) + continue; + + double deltaPercent = ((bench.Ppl - native.Ppl) / native.Ppl) * 100.0; + deltas.Add(deltaPercent); + } + + return deltas.Count == 0 ? double.PositiveInfinity : deltas.Average(); + } + + private sealed class GroupIsolationWorkItem + { + public TensorGroup Group { get; set; } = default!; + public List Candidates { get; set; } = new(); + public IsolationGroupDecision Decision { get; set; } = default!; + } + + private sealed class SynergyBadTradeElimination + { + public TensorGroup Group { get; set; } = default!; + public GroupCandidateEvaluation Removed { get; set; } = default!; + public GroupCandidateEvaluation Anchor { get; set; } = default!; + public string Reason { get; set; } = string.Empty; + } + + private sealed class GroupCandidateEvaluation + { + public TensorGroup Group { get; set; } = default!; + public BaselineQuants CandidateBaseline { get; set; } = default!; + public ulong SizeBytes { get; set; } + public double SavingsRatio { get; set; } + public double Kld { get; set; } + public double PplDeltaPercent { get; set; } + } + + private sealed class BaseBaselineEvaluation + { + public BaselineQuants Baseline { get; set; } = default!; + public ulong SizeBytes { get; set; } + public double SavingsRatio { get; set; } + public double Kld { get; set; } + public double PplDeltaPercent { get; set; } + public bool UsedBaseOnlySnapshot { get; set; } + public bool UsedPureFallback { get; set; } + public ulong? PureBaselineSizeBytes { get; set; } + public double? PureBaselineKld { get; set; } + public double? PureBaselinePplDeltaPercent { get; set; } + } + + private sealed class BenchmarkSnapshot + { + public ulong SizeBytes { get; set; } + public List Benchmarks { get; set; } = new(); + } + + private sealed class CategorySnapshot + { + public byte Category { get; set; } + public double? Kld { get; set; } + public double Ppl { get; set; } + public double PplError { get; set; } + } +} \ No newline at end of file diff --git a/src/MagicQuant/Services/IsolationPlanningService.cs b/src/MagicQuant/Services/IsolationPlanningService.cs new file mode 100644 index 0000000..ce4fe48 --- /dev/null +++ b/src/MagicQuant/Services/IsolationPlanningService.cs @@ -0,0 +1,26 @@ +using MagicQuant.Helpers; +using MQ.DB.Models; + +namespace MagicQuant.Services; + +/// +/// Centralized authority for isolation probe planning in baseline-family candidate space. +/// +public sealed class IsolationPlanningService +{ + public RequiredSampleGenerationResult BuildInitialPlan(List? missingTensorGroups = null) + => TensorConfigGenerator.GenerateInitialIsolationSamplePlan(missingTensorGroups); + + public RequiredSampleGenerationResult BuildContinuationPlan(IEnumerable groupIdsToContinue, List? missingTensorGroups = null) + => TensorConfigGenerator.GenerateContinuationIsolationSamplePlan(groupIdsToContinue, missingTensorGroups); + + + public RequiredSampleGenerationResult BuildArchivalCoveragePlan( + IEnumerable? groupIdsToArchive = null, + IEnumerable? existingPlanKeys = null, + List? missingTensorGroups = null) + => TensorConfigGenerator.GenerateArchivalIsolationCoverageSamplePlan(groupIdsToArchive, existingPlanKeys, missingTensorGroups); + + public List BuildRequiredStartupCombos(List? missingTensorGroups = null) + => TensorConfigGenerator.GenerateRequiredDataSampleCombos(missingTensorGroups); +} \ No newline at end of file diff --git a/src/MagicQuant/Services/LearnedBaselinePruningService.cs b/src/MagicQuant/Services/LearnedBaselinePruningService.cs new file mode 100644 index 0000000..e29779c --- /dev/null +++ b/src/MagicQuant/Services/LearnedBaselinePruningService.cs @@ -0,0 +1,45 @@ +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace MagicQuant.Services; + +public sealed class LearnedBaselinePruningResult +{ + public int GroupCandidateEliminations { get; set; } + public int BaselinesSkippedWithoutLearnedRows { get; set; } + public List Notes { get; } = new(); +} + +public sealed class LearnedBaselineCoverageStatus +{ + public bool HasAnyLearnedRows { get; set; } + public bool SafeToApplyBeforeStartup { get; set; } + public int ExpectedCandidateGroupPairs { get; set; } + public int PresentCandidateGroupPairs { get; set; } + public List MissingPairs { get; } = new(); +} + +public sealed class LearnedBaselinePruningService +{ + public Task GetCoverageStatusAsync(CancellationToken ct = default) + { + var status = new LearnedBaselineCoverageStatus + { + HasAnyLearnedRows = false, + SafeToApplyBeforeStartup = false, + ExpectedCandidateGroupPairs = 0, + PresentCandidateGroupPairs = 0 + }; + + status.MissingPairs.Add("Learned-baseline early pruning is disabled."); + return Task.FromResult(status); + } + + public Task AnalyzeAndApplyAsync(CancellationToken ct = default) + { + var result = new LearnedBaselinePruningResult(); + result.Notes.Add("Learned-baseline early pruning is disabled. No candidates were removed from the search space."); + return Task.FromResult(result); + } +} \ No newline at end of file diff --git a/src/MagicQuant/Services/Learning/TensorGroupingAuditService.cs b/src/MagicQuant/Services/Learning/TensorGroupingAuditService.cs new file mode 100644 index 0000000..ba59bb2 --- /dev/null +++ b/src/MagicQuant/Services/Learning/TensorGroupingAuditService.cs @@ -0,0 +1,116 @@ +using MagicQuant.Models.Learning; +using MQ.DB.Models; + +namespace MagicQuant.Services.Learning; + +public sealed class TensorGroupingAuditService +{ + public TensorGroupingAuditResult Audit( + IReadOnlyCollection tensorNames, + IReadOnlyDictionary truthByTensor) + { + var grouped = new Dictionary(StringComparer.Ordinal); + var ambiguous = new List(); + var illegalUnresolved = new List(); + var baseQuantExceptions = new List(); + + foreach (var tensorName in tensorNames.OrderBy(x => x, StringComparer.Ordinal)) + { + var matchedGroups = TReg.FindMatchingGroups(tensorName); + + if (matchedGroups.Length == 1) + { + grouped[tensorName] = new TensorGroupingResult + { + PrimaryGroup = matchedGroups[0], + MatchedGroups = [matchedGroups[0].Name] + }; + + continue; + } + + if (matchedGroups.Length > 1) + { + var names = matchedGroups.Select(x => x.Name).ToList(); + truthByTensor.TryGetValue(tensorName, out var truth); + grouped[tensorName] = new TensorGroupingResult + { + PrimaryGroup = null, + MatchedGroups = names + }; + + ambiguous.Add(new TensorGroupingAuditIssue + { + TensorName = tensorName, + IssueKind = "AmbiguousSemanticGroupCollision", + MatchedGroups = names, + FinalQuantType = truth?.FinalQuantType, + LearningSource = truth?.Source.ToString() + }); + + continue; + } + + var matchedPattern = FindMatchingBaseQuantExceptionPattern(tensorName); + if (matchedPattern != null) + { + truthByTensor.TryGetValue(tensorName, out var truth); + grouped[tensorName] = new TensorGroupingResult + { + PrimaryGroup = null, + MatchedGroups = [], + IsBaseQuantException = true, + MatchedExceptionPattern = matchedPattern + }; + + baseQuantExceptions.Add(new TensorGroupingAuditIssue + { + TensorName = tensorName, + IssueKind = "BaseQuantExceptionFallback", + MatchedExceptionPattern = matchedPattern, + FinalQuantType = truth?.FinalQuantType, + LearningSource = truth?.Source.ToString() + }); + + continue; + } + + grouped[tensorName] = new TensorGroupingResult + { + PrimaryGroup = null, + MatchedGroups = [] + }; + + truthByTensor.TryGetValue(tensorName, out var unresolvedTruth); + illegalUnresolved.Add(new TensorGroupingAuditIssue + { + TensorName = tensorName, + IssueKind = "IllegalUnresolvedTensor", + FinalQuantType = unresolvedTruth?.FinalQuantType, + LearningSource = unresolvedTruth?.Source.ToString() + }); + } + + return new TensorGroupingAuditResult + { + GroupedByTensor = grouped, + Ambiguous = ambiguous, + IllegalUnresolved = illegalUnresolved, + BaseQuantExceptions = baseQuantExceptions + }; + } + + private static string? FindMatchingBaseQuantExceptionPattern(string tensorName) + { + var patterns = TReg.GetBaseQuantExceptionPatterns(); + var regexes = TReg.GetBaseQuantExceptionRegexes(); + + for (int i = 0; i < regexes.Length; i++) + { + if (regexes[i].IsMatch(tensorName)) + return i < patterns.Length ? patterns[i] : regexes[i].ToString(); + } + + return null; + } +} diff --git a/src/MagicQuant/Services/Learning/TensorLearningDiagnosticWriter.cs b/src/MagicQuant/Services/Learning/TensorLearningDiagnosticWriter.cs new file mode 100644 index 0000000..f1ff4d9 --- /dev/null +++ b/src/MagicQuant/Services/Learning/TensorLearningDiagnosticWriter.cs @@ -0,0 +1,136 @@ +using System.Text; +using System.Text.Json; +using MagicQuant.Helpers; +using MagicQuant.Models.Learning; +using MQ.DB; +using MQ.DB.Models; + +namespace MagicQuant.Services.Learning; + +public sealed class TensorLearningDiagnosticWriter +{ + public async Task WriteFailureAsync( + string baselineName, + string schemeName, + string sourceKind, + string? sourceRepository, + string? sourceFileName, + IReadOnlyDictionary truthByTensor, + TensorGroupingAuditResult audit, + TensorTruthVerificationResult verification, + CancellationToken ct = default) + { + string dir = GetTensorConfigLogDirectory(); + Directory.CreateDirectory(Path.Combine(Cache.ModelMagicQuantDirectory!, "Logs")); + await HardDeleteHelper.DeleteDirectoryIfExistsAsync(dir, ct); + Directory.CreateDirectory(dir); + + string safeBaseline = SanitizeForFileName(baselineName); + string safeScheme = SanitizeForFileName(schemeName); + string baseName = $"tensor-group-learning-failure-{safeBaseline}-{safeScheme}"; + string jsonPath = Path.Combine(dir, baseName + ".json"); + string txtPath = Path.Combine(dir, baseName + ".txt"); + + var payload = new + { + GeneratedUtc = DateTime.UtcNow, + ModelDirectory = Cache.ModelDirectory, + ModelMagicQuantDirectory = Cache.ModelMagicQuantDirectory, + ActiveConfigPath = TReg.TensorGroupsYamlPathOverride, + Baseline = baselineName, + Scheme = schemeName, + SourceKind = sourceKind, + SourceRepository = sourceRepository, + SourceFileName = sourceFileName, + TotalTruthTensors = truthByTensor.Count, + SemanticMatchedCount = audit.GroupedByTensor.Count(x => x.Value.PrimaryGroup != null), + BaseQuantExceptionCount = audit.BaseQuantExceptions.Count, + IllegalUnresolvedCount = audit.IllegalUnresolved.Count, + AmbiguousCount = audit.Ambiguous.Count, + MismatchCount = verification.HardMismatches.Count + verification.SoftMismatches.Count, + HighSeverityMismatchCount = verification.HardMismatches.Count, + BaseQuantExceptions = audit.BaseQuantExceptions, + IllegalUnresolved = audit.IllegalUnresolved, + Ambiguous = audit.Ambiguous, + HardMismatches = verification.HardMismatches, + SoftMismatches = verification.SoftMismatches, + LogOnly = verification.LogOnly + }; + + await File.WriteAllTextAsync(jsonPath, JsonSerializer.Serialize(payload, new JsonSerializerOptions { WriteIndented = true }), ct); + + var sb = new StringBuilder(); + sb.AppendLine("MagicQuant Tensor Group Learning Failure"); + sb.AppendLine($"GeneratedUtc: {DateTime.UtcNow:O}"); + sb.AppendLine($"ModelDirectory: {Cache.ModelDirectory}"); + sb.AppendLine($"ModelMagicQuantDirectory: {Cache.ModelMagicQuantDirectory}"); + sb.AppendLine($"ActiveConfigPath: {TReg.TensorGroupsYamlPathOverride}"); + sb.AppendLine($"Baseline: {baselineName}"); + sb.AppendLine($"Scheme: {schemeName}"); + sb.AppendLine($"SourceKind: {sourceKind}"); + sb.AppendLine($"SourceRepository: {sourceRepository}"); + sb.AppendLine($"SourceFileName: {sourceFileName}"); + sb.AppendLine($"TotalTruthTensors: {truthByTensor.Count}"); + sb.AppendLine($"SemanticMatchedCount: {audit.GroupedByTensor.Count(x => x.Value.PrimaryGroup != null)}"); + sb.AppendLine($"BaseQuantExceptionCount: {audit.BaseQuantExceptions.Count}"); + sb.AppendLine($"IllegalUnresolvedCount: {audit.IllegalUnresolved.Count}"); + sb.AppendLine($"AmbiguousCount: {audit.Ambiguous.Count}"); + sb.AppendLine($"MismatchCount: {verification.HardMismatches.Count + verification.SoftMismatches.Count}"); + sb.AppendLine($"HighSeverityMismatchCount: {verification.HardMismatches.Count}"); + sb.AppendLine(); + + AppendIssues(sb, "Base Quant Exceptions", audit.BaseQuantExceptions); + AppendIssues(sb, "Illegal Unresolved", audit.IllegalUnresolved); + AppendIssues(sb, "Ambiguous", audit.Ambiguous); + + sb.AppendLine("Mismatches:"); + foreach (var mismatch in verification.HardMismatches.Concat(verification.SoftMismatches)) + sb.AppendLine($"- {mismatch.TensorName}: log={mismatch.LogQuantType}, gguf={mismatch.GgufQuantType}, severity={(mismatch.IsHighSeverity ? "high" : "soft")}, source decision=GGUF"); + + if (verification.LogOnly.Count > 0) + { + sb.AppendLine(); + sb.AppendLine("Log-only tensors ignored:"); + foreach (var entry in verification.LogOnly) + sb.AppendLine($"- {entry}"); + } + + await File.WriteAllTextAsync(txtPath, sb.ToString(), ct); + return txtPath; + } + + private static void AppendIssues(StringBuilder sb, string heading, IReadOnlyList issues) + { + sb.AppendLine(heading + ":"); + if (issues.Count == 0) + { + sb.AppendLine("- none"); + sb.AppendLine(); + return; + } + + foreach (var issue in issues) + { + var groups = issue.MatchedGroups.Count > 0 ? $", matchedGroups=[{string.Join(", ", issue.MatchedGroups)}]" : string.Empty; + var pattern = string.IsNullOrWhiteSpace(issue.MatchedExceptionPattern) ? string.Empty : $", exceptionPattern={issue.MatchedExceptionPattern}"; + sb.AppendLine($"- {issue.TensorName}: quant={issue.FinalQuantType}, source={issue.LearningSource}{groups}{pattern}"); + } + + sb.AppendLine(); + } + + private static string GetTensorConfigLogDirectory() + { + if (string.IsNullOrWhiteSpace(Cache.ModelMagicQuantDirectory)) + throw new InvalidOperationException("Cache.ModelMagicQuantDirectory is not set."); + + return Path.Combine(Cache.ModelMagicQuantDirectory, "Logs", "TensorConfigs"); + } + + private static string SanitizeForFileName(string value) + { + var invalidChars = Path.GetInvalidFileNameChars(); + var cleaned = new string(value.Select(ch => invalidChars.Contains(ch) ? '-' : ch).ToArray()); + return string.IsNullOrWhiteSpace(cleaned) ? "unknown" : cleaned; + } +} diff --git a/src/MagicQuant/Services/LlamaGpuArgumentBuilder.cs b/src/MagicQuant/Services/LlamaGpuArgumentBuilder.cs new file mode 100644 index 0000000..893e79b --- /dev/null +++ b/src/MagicQuant/Services/LlamaGpuArgumentBuilder.cs @@ -0,0 +1,43 @@ +using System.Globalization; + +namespace MagicQuant.Services; + +internal enum LlamaGpuTool +{ + CommonCli = 1, + LlamaBench = 2 +} + +internal static class LlamaGpuArgumentBuilder +{ + public static string BuildTensorSplitArgs( + IReadOnlyList deviceIndices, + IReadOnlyDictionary gpuMemoryLimitsGb, + LlamaGpuTool tool) + { + ArgumentNullException.ThrowIfNull(deviceIndices); + ArgumentNullException.ThrowIfNull(gpuMemoryLimitsGb); + + if (deviceIndices.Count <= 1 || gpuMemoryLimitsGb.Count == 0) + return string.Empty; + + var missing = deviceIndices + .Where(i => !gpuMemoryLimitsGb.ContainsKey(i)) + .ToArray(); + + if (missing.Length > 0) + { + throw new InvalidOperationException( + $"GPU memory limits were configured, but limits are missing for GPU(s): {string.Join(", ", missing)}."); + } + + // llama-bench uses commas to generate a Cartesian product of benchmark cases; + // members of one multi-GPU split are slash-separated. The common llama.cpp + // argument parser used by llama-perplexity expects comma-separated members. + string separator = tool == LlamaGpuTool.LlamaBench ? "/" : ","; + string split = string.Join(separator, deviceIndices.Select(i => + gpuMemoryLimitsGb[i].ToString("0.###", CultureInfo.InvariantCulture))); + + return $" --tensor-split {split}"; + } +} diff --git a/src/MagicQuant/Services/MagicQuantManifestPathService.cs b/src/MagicQuant/Services/MagicQuantManifestPathService.cs new file mode 100644 index 0000000..a46540b --- /dev/null +++ b/src/MagicQuant/Services/MagicQuantManifestPathService.cs @@ -0,0 +1,97 @@ +using Spectre.Console; + +namespace MagicQuant.Services; + +public static class MagicQuantManifestPathService +{ + public const string ManifestDirectoryName = "magicquant-manifest"; + + public const string CloneConfigsFileName = "magicquant.clone-configs.json"; + public const string FinalSurvivorsFileName = "magicquant.final-survivors.json"; + public const string ReplacementsFileName = "magicquant.replacements.json"; + public const string HybridMapFileName = "magicquant.hybrid-map.json"; + public const string CloneBenchmarksFileName = "magicquant.clone-benchmarks.json"; + public const string IsolationSamplesFileName = "magicquant.isolation-samples.json"; + public const string BadTradesFileName = "magicquant.bad-trades.json"; + + public static readonly IReadOnlyList KnownManifestFileNames = + [ + CloneConfigsFileName, + FinalSurvivorsFileName, + ReplacementsFileName, + HybridMapFileName, + CloneBenchmarksFileName, + IsolationSamplesFileName, + BadTradesFileName + ]; + + public static string EnsureManifestDirectory(string outputDirectory) + { + if (string.IsNullOrWhiteSpace(outputDirectory)) + throw new ArgumentException("Output directory is required.", nameof(outputDirectory)); + + string normalizedOutput = NormalizeDirectoryPath(outputDirectory); + + // Idempotency guard: callers sometimes already pass the manifest directory itself. + // Do not create magicquant-manifest/magicquant-manifest. + if (string.Equals(Path.GetFileName(normalizedOutput), ManifestDirectoryName, StringComparison.OrdinalIgnoreCase)) + { + Directory.CreateDirectory(normalizedOutput); + return normalizedOutput; + } + + string path = Path.Combine(normalizedOutput, ManifestDirectoryName); + Directory.CreateDirectory(path); + return path; + } + + public static string GetManifestFilePath(string outputDirectory, string fileName) + => Path.Combine(EnsureManifestDirectory(outputDirectory), NormalizeManifestFileName(fileName)); + + public static string RelativeManifestPath(string fileName) + { + string normalized = NormalizeManifestFileName(fileName).Replace('\\', '/').TrimStart('/'); + + if (normalized.StartsWith(ManifestDirectoryName + "/", StringComparison.OrdinalIgnoreCase)) + return normalized; + + return $"{ManifestDirectoryName}/{normalized}"; + } + + public static string HuggingFaceResolvePath(string fileName, bool download = false) + { + string path = $"./../../resolve/main/{RelativeManifestPath(fileName)}"; + return download ? path + "?download=true" : path; + } + + public static string HuggingFaceGgufResolvePath(string fileName) + => $"./../../resolve/main/{Uri.EscapeDataString(fileName)}?download=true"; + + public static void WriteManifestLog(string message, bool isWarning = false, bool isError = false) + { + string color = isError ? "red" : isWarning ? "yellow" : "grey"; + string line = $"[{DateTime.Now:HH:mm:ss}] manifest: {message}"; + AnsiConsole.MarkupLine($"[{color}]{Markup.Escape(line)}[/]"); + } + + private static string NormalizeDirectoryPath(string path) + { + string full = Path.GetFullPath(path); + string root = Path.GetPathRoot(full) ?? string.Empty; + string trimmed = full.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + return string.IsNullOrWhiteSpace(trimmed) ? root : trimmed; + } + + private static string NormalizeManifestFileName(string fileName) + { + if (string.IsNullOrWhiteSpace(fileName)) + throw new ArgumentException("Manifest file name is required.", nameof(fileName)); + + string normalized = fileName.Trim().Replace('\\', '/').TrimStart('/'); + + if (normalized.StartsWith(ManifestDirectoryName + "/", StringComparison.OrdinalIgnoreCase)) + normalized = normalized[(ManifestDirectoryName.Length + 1)..]; + + return normalized; + } +} diff --git a/src/MagicQuant/Services/ModelArtifactPathService.cs b/src/MagicQuant/Services/ModelArtifactPathService.cs new file mode 100644 index 0000000..cf3535e --- /dev/null +++ b/src/MagicQuant/Services/ModelArtifactPathService.cs @@ -0,0 +1,83 @@ +using System.Text; +using MQ.DB; +using MQ.DB.Models; + +namespace MagicQuant.Services; + +/// +/// Names durable model artifacts and quantization logs from the active model context. +/// Temporary heavy writes belong to ScratchStorageService leases instead. +/// +public sealed class ModelArtifactPathService +{ + public string ModelDirectory => Cache.ModelDirectory + ?? throw new InvalidOperationException("Cache.ModelDirectory is not set."); + + public string ModelMagicQuantDirectory => Cache.ModelMagicQuantDirectory + ?? throw new InvalidOperationException("Cache.ModelMagicQuantDirectory is not set."); + + public string GgufDir => Path.Combine(ModelMagicQuantDirectory, "GGUF"); + public string BenchDir => Path.Combine(ModelMagicQuantDirectory, "Benchmarks"); + public string LogsDir => Path.Combine(ModelMagicQuantDirectory, "Logs"); + public string QuantizationLogsDir => Path.Combine(LogsDir, "Quantization"); + public string ExternalBaselinesDir => Cache.ExternalBaselineCacheDirectory + ?? Path.Combine(ModelMagicQuantDirectory, "ExternalBaselines"); + + public string ScratchModelNamespace + { + get + { + var modelName = new DirectoryInfo(ModelDirectory).Name; + if (string.IsNullOrWhiteSpace(modelName)) + modelName = "model"; + + var modelId = string.IsNullOrWhiteSpace(Cache.CurrentModelId) ? "unknown" : Cache.CurrentModelId; + return MakeSafeFileComponent($"{modelName}_{modelId}"); + } + } + + public string GetBenchmarkDir(string modelName) => Path.Combine(BenchDir, modelName); + + public string GetBaseLogitsDirectory() + { + string typeStr = (Cache.TorchType ?? Cache.MainTorchType.BF16).ToString(); + return Path.Combine(BenchDir, typeStr, "logits"); + } + + public string GetNativeBaseGgufPath() + { + string modelName = new DirectoryInfo(ModelDirectory).Name; + var torchType = Cache.TorchType ?? Cache.MainTorchType.BF16; + string typeStr = torchType.ToString(); + return Path.Combine(GgufDir, $"{modelName}-{typeStr}.gguf"); + } + + public string GetExternalBaselineDurablePath(BaselineQuants baseline) + { + string safe = MakeSafeFileComponent(baseline.CanonicalKey); + string extension = Path.GetExtension(baseline.SourceFileName ?? string.Empty); + if (string.IsNullOrWhiteSpace(extension)) + extension = ".gguf"; + + return Path.Combine(ExternalBaselinesDir, safe + extension); + } + + public string GetQuantizationLogPath(string artifactName, Guid leaseId) + { + string safeName = MakeSafeFileComponent(artifactName); + return Path.Combine(QuantizationLogsDir, $"{safeName}-{leaseId:N}.quantize.log"); + } + + public static string MakeSafeFileComponent(string value) + { + if (string.IsNullOrWhiteSpace(value)) + return "artifact"; + + var invalid = Path.GetInvalidFileNameChars().ToHashSet(); + var sb = new StringBuilder(value.Length); + foreach (var ch in value) + sb.Append(invalid.Contains(ch) ? '_' : ch); + + return sb.ToString(); + } +} diff --git a/src/MagicQuant/Services/ModelCompatibilityService.cs b/src/MagicQuant/Services/ModelCompatibilityService.cs new file mode 100644 index 0000000..44fff3e --- /dev/null +++ b/src/MagicQuant/Services/ModelCompatibilityService.cs @@ -0,0 +1,368 @@ +using System.Text.Json; +using MagicQuant.Helpers; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models; +using Microsoft.EntityFrameworkCore; +using Spectre.Console; + +namespace MagicQuant.Services; + +public class ModelCompatibilityService +{ + private const string CompatVerboseEnv = "MAGICQUANT_DIAG_VERBOSE_TENSOR_COMPATIBILITY"; + private const string CompatFocusCandidatesEnv = "MAGICQUANT_DIAG_FOCUS_CANDIDATES"; + private readonly PythonManager _pyManager; + + public ModelCompatibilityService(PythonManager pyManager) + { + _pyManager = pyManager; + } + + public async Task RunCompatibilityCheckAsync(string ggufPath) + { + AnsiConsole.Write(new Rule("[yellow]Tensor Compatibility Check[/]") { Justification = Justify.Left }); + + if (!File.Exists(ggufPath)) + throw new FileNotFoundException($"Base model not found at {ggufPath}"); + + TensorWeightScheme.ValidateSmallestConfiguration(); + + RuntimeSearchSpace.ResetForCompatibilityPass(); + Cache.UnusedTensorGroups.Clear(); + + string directory = Path.GetDirectoryName(ggufPath)!; + string scriptPath = Path.Combine(directory, "check_compat.py"); + string resultPath = Path.Combine(directory, "compat_results.json"); + string debugPath = Path.Combine(directory, "compat_debug.txt"); + + try + { + var groupDefinitions = TReg.All.ToDictionary(g => g.Name, g => g.Tensors); + + var candidateBlockRequirements = BaselineQuants.GetGroupCombinationCandidates(RuntimeSearchSpace.HasUsableImatrix(), allowHighPrecisionHybrids: false) + .Where(c => c.DefaultTensorScheme?.BlockNeo.HasValue == true) + .ToDictionary(c => c.Names[0], c => c.DefaultTensorScheme!.BlockNeo!.Value); + + var payload = new + { + gguf_path = ggufPath, + output_path = resultPath, + groups = groupDefinitions, + schemes = candidateBlockRequirements + }; + + string pyCode = GeneratePythonScript(JsonSerializer.Serialize(payload)); + await File.WriteAllTextAsync(scriptPath, pyCode); + + AnsiConsole.MarkupLine("[grey]Inspecting GGUF structure...[/]"); + await _pyManager.RunPythonScriptAsync(scriptPath); + + if (!File.Exists(resultPath)) + throw new Exception("Compatibility script finished but produced no result file."); + + string jsonResult = await File.ReadAllTextAsync(resultPath); + + if (jsonResult.Contains("\"Error\"", StringComparison.Ordinal)) + { + var errorRes = JsonSerializer.Deserialize(jsonResult); + if (!string.IsNullOrEmpty(errorRes?.Error)) + throw new Exception($"Python Inspection Failed: {errorRes.Error}"); + } + + var result = JsonSerializer.Deserialize(jsonResult); + if (result == null) + return; + + bool compatVerbose = IsCompatVerbose(); + var focusCandidates = GetFocusCandidates(); + var runtimeCandidates = BaselineQuants.GetGroupCombinationCandidates(RuntimeSearchSpace.HasUsableImatrix(), allowHighPrecisionHybrids: false).ToList(); + int unusedCount = 0; + int usedCount = 0; + int observedShapeIncompatibilityCount = 0; + int explicitQuantBannedCount = 0; + + foreach (var group in TReg.All) + { + bool exists = result.FoundGroups.Contains(group.Name, StringComparer.OrdinalIgnoreCase); + + if (exists) + { + usedCount++; + continue; + } + + unusedCount++; + Cache.UnusedTensorGroups.Add(group); + + RuntimeSearchSpace.BanAllExplicitCombinationCandidatesForGroup(group, phase: "TensorCompatibilityCheck", reason: "group missing in GGUF"); + } + + var failuresByGroupAndScheme = result.Failures + .GroupBy(x => $"{x.Group}::{x.Scheme}", StringComparer.OrdinalIgnoreCase) + .ToDictionary(x => x.Key, x => x.ToList(), StringComparer.OrdinalIgnoreCase); + + foreach (var failure in result.Incompatible) + { + var group = TReg.GetByName(failure.Group); + var candidate = runtimeCandidates.FirstOrDefault(c => c.Names.Any(n => n.Equals(failure.Scheme, StringComparison.OrdinalIgnoreCase))); + + if (group == null || candidate == null) + continue; + if (RuntimeSearchSpace.IsCombinationCandidateRuntimeBannedForGroup(group, candidate)) + continue; + + var beforeRuntimeBan = RuntimeSearchSpace.IsCombinationCandidateRuntimeBannedForGroup(group, candidate); + + // BlockNeo compatibility validation is currently observed-only and under review for removal. + // It is not reliable for some architectures (especially MoE), so runtime bans remain disabled. + //RuntimeSearchSpace.BanCombinationCandidateForGroup(group, candidate, phase: "TensorCompatibilityCheck", reason: "Block Alignment"); + observedShapeIncompatibilityCount++; + + if (ShouldLogCompatDetail(compatVerbose, group, candidate, focusCandidates)) + { + MagicQuantDiagnostics.Log("compat:decision", + $"group={group.Name}(id={group.UniqueId}) candidate={candidate.Names[0]}(id={candidate.UniqueId}) scheme={candidate.DefaultTensorScheme?.Names[0] ?? ""} block={candidate.DefaultTensorScheme?.BlockNeo?.ToString() ?? ""} staticBanned={candidate.BannedGroupIds.Contains(group.UniqueId)} runtimeBannedBefore={beforeRuntimeBan} result=observed-only reason=Block Alignment restriction=disabled"); + } + + if (failuresByGroupAndScheme.TryGetValue($"{failure.Group}::{failure.Scheme}", out var details) && details.Count > 0) + { + LogFailureSummary(group, candidate, details); + } + } + + if (compatVerbose) + await LogGroupCompatibilityOutcomeAndTruthCrossCheckAsync(result, runtimeCandidates, focusCandidates, ct: CancellationToken.None); + + foreach (var group in TReg.All.Except(Cache.UnusedTensorGroups)) + { + if (RuntimeSearchSpace.IsGroupExplicitCandidateBanned(group)) + explicitQuantBannedCount++; + } + + AnsiConsole.MarkupLine("[green]✔[/] Analysis Complete."); + AnsiConsole.MarkupLine($" Active Groups: [bold cyan]{usedCount}[/]"); + + if (unusedCount > 0) + { + string unusedNames = string.Join(", ", Cache.UnusedTensorGroups.Select(g => g.Name)); + AnsiConsole.MarkupLine($" Unused Groups: [grey]{unusedNames}[/] (Forced to NULL)"); + } + + if (explicitQuantBannedCount > 0) + { + string groups = string.Join(", ", + RuntimeSearchSpace.GetGroupsWithExplicitQuantBanned().Select(x => x.Name)); + + AnsiConsole.MarkupLine($" Explicit-Quant-Banned Groups: [yellow]{groups}[/]"); + } + else + { + AnsiConsole.MarkupLine("[green]No groups were reduced to explicit-banned/NULL-only by compatibility checks.[/]"); + } + + if (observedShapeIncompatibilityCount > 0) + { + AnsiConsole.MarkupLine($"[yellow]Observed {observedShapeIncompatibilityCount:N0} BlockNeo/shape incompatibilities; runtime restrictions are currently disabled while compatibility validation is under review.[/]"); + } + else + { + AnsiConsole.MarkupLine("[green]No BlockNeo/shape incompatibilities observed.[/]"); + } + + AnsiConsole.WriteLine(); + } + finally + { + if (File.Exists(scriptPath)) File.Delete(scriptPath); + if (File.Exists(resultPath)) File.Delete(resultPath); + _ = debugPath; + } + } + + private static bool IsCompatVerbose() + => string.Equals(Environment.GetEnvironmentVariable(CompatVerboseEnv), "1", StringComparison.OrdinalIgnoreCase) + || string.Equals(Environment.GetEnvironmentVariable(CompatVerboseEnv), "true", StringComparison.OrdinalIgnoreCase); + + private static HashSet GetFocusCandidates() + => (Environment.GetEnvironmentVariable(CompatFocusCandidatesEnv) ?? string.Empty) + .Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + private static bool ShouldLogCompatDetail(bool compatVerbose, TensorGroup group, BaselineQuants candidate, HashSet focusCandidates) + => compatVerbose && (MagicQuantDiagnostics.ShouldLogGroup(group) || focusCandidates.Count == 0 || candidate.Names.Any(x => focusCandidates.Contains(x))); + + private static void LogFailureSummary(TensorGroup group, BaselineQuants candidate, List details) + { + var failCount = details.Count; + var firstFive = details.Take(5).ToList(); + var shapes = details.GroupBy(x => $"[{string.Join(",", x.Shape)}]").Select(x => $"{x.Key} x{x.Count()}").ToList(); + var remainders = details.Select(x => x.Remainder).Distinct().OrderBy(x => x).ToList(); + MagicQuantDiagnostics.Log("compat:summary", $"group={group.Name} candidate={candidate.Names[0]} checkedTensors={details.Max(x => x.CheckedTensorCount)} failingTensors={failCount} shapePatterns={string.Join("; ", shapes)} remainders={string.Join(",", remainders)}"); + foreach (var f in firstFive) + { + MagicQuantDiagnostics.Log("compat:block-alignment-fail", + $"group={group.Name}(id={group.UniqueId}) candidate={candidate.Names[0]}(id={candidate.UniqueId}) tensor={f.Tensor} tensorClass={f.TensorClass} dims=[{string.Join(",", f.Shape)}] nDims={f.Shape.Count} checkedDimension={f.CheckedDimension} checkedValue={f.CheckedValue} requiredMultiple={f.RequiredMultiple} remainder={f.Remainder} pass=false reason=Block Alignment"); + } + } + + private static async Task LogGroupCompatibilityOutcomeAndTruthCrossCheckAsync(CompatResult result, List candidates, HashSet focusCandidates, CancellationToken ct) + { + await using var db = new MagicQuantContext(); + var modelHashId = await ArchitectureFamilyService.ResolveExactCurrentAiModelHashIdOrNullAsync(db, ct); + var imatrixId = modelHashId == null ? (long?)null : await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, modelHashId.Value, createIfMissing: false, ct); + foreach (var group in TReg.All.Except(Cache.UnusedTensorGroups)) + { + if (!MagicQuantDiagnostics.ShouldLogGroup(group)) + continue; + var raw = RuntimeSearchSpace.GetRealExplicitCombinationCandidatesForGroup(group); + var allowed = RuntimeSearchSpace.GetAllowedRealExplicitCombinationCandidatesForGroup(group); + var restricted = raw.Where(x => RuntimeSearchSpace.IsCombinationCandidateRuntimeBannedForGroup(group, x)).ToList(); + MagicQuantDiagnostics.Log("compat:group-result", $"group={group.Name}(id={group.UniqueId}) before={raw.Count} after={allowed.Count} allowed={string.Join(", ", allowed.Select(MagicQuantDiagnostics.CandidateLabel))} restricted={string.Join(", ", restricted.Select(MagicQuantDiagnostics.CandidateLabel))}"); + if (raw.Count > 5 && allowed.Count == 1 && allowed[0].UniqueId == BaselineQuants.Q8_0.UniqueId) + { + var focusRestricted = restricted.Where(x => x.Names.Any(n => focusCandidates.Contains(n)) || x.Names[0] is "Q6_K" or "Q5_K" or "Q4_K_M").Select(x => x.Names[0]); + MagicQuantDiagnostics.Log("compat:collapse-warning", $"group={group.Name} collapsed to Q8_0 only restrictions={restricted.Count} topReason=Block Alignment focusCandidatesRestricted={string.Join(",", focusRestricted)}"); + } + } + } + + private string GeneratePythonScript(string jsonPayload) + { + return $@" +import sys +import json +import re + +payload_str = r'''{jsonPayload}''' +config = json.loads(payload_str) +output_path = config['output_path'] +debug_path = output_path.replace('compat_results.json', 'compat_debug.txt') + +def write_error(msg): + with open(output_path, 'w') as f: + json.dump({{""FoundGroups"": [], ""Incompatible"": [], ""Error"": msg}}, f) + sys.exit(0) + +try: + import gguf +except ImportError: + write_error('gguf module not installed') + +try: + reader = gguf.GGUFReader(config['gguf_path']) +except Exception as e: + write_error(str(e)) + +tensors_map = {{t.name: t for t in reader.tensors}} +tensor_names = list(tensors_map.keys()) + +found_groups = [] +failures = [] +failure_details = [] +debug_lines = [] + +debug_lines.append('Inspecting ' + str(len(tensor_names)) + ' tensors against ' + str(len(config[""schemes""])) + ' block requirements.') + +for g_name, patterns in config['groups'].items(): + matched = [] + first_reason = None + + for pat in patterns: + try: + regex = re.compile(pat) + for t in tensor_names: + if regex.fullmatch(t): + matched.append(t) + if not first_reason: + first_reason = ""Match: '"" + pat + ""' -> '"" + t + ""'"" + except: + continue + + if matched: + found_groups.append(g_name) + debug_lines.append(""[FOUND] "" + g_name + "" ("" + str(len(matched)) + "" tensors). "" + str(first_reason)) + weights = [t for t in matched if t.endswith('.weight')] + + if weights: + for scheme, block_size in config['schemes'].items(): + is_valid = True + + for w_name in weights: + t_obj = tensors_map[w_name] + ne0 = t_obj.shape[0] + n_dims = len(t_obj.shape) + + if n_dims != 2: + is_valid = False + debug_lines.append("" [FAIL] "" + g_name + "" vs "" + scheme + "": "" + w_name + "" is "" + str(n_dims) + ""D (Required 2D)"") + break + + if ne0 % block_size != 0: + is_valid = False + tclass = 'unknown' + lower_name = w_name.lower() + if 'exps' in lower_name: + tclass = 'routed_expert' + elif 'router' in lower_name: + tclass = 'router' + elif 'ffn_' in lower_name: + tclass = 'dense_ffn' + failure_details.append({{ + ""Group"": g_name, + ""Scheme"": scheme, + ""Tensor"": w_name, + ""Shape"": list(t_obj.shape), + ""CheckedDimension"": 0, + ""CheckedValue"": int(ne0), + ""RequiredMultiple"": int(block_size), + ""Remainder"": int(ne0 % block_size), + ""TensorClass"": tclass, + ""CheckedTensorCount"": len(weights) + }}) + debug_lines.append("" [FAIL] "" + g_name + "" vs "" + scheme + "" (Block "" + str(block_size) + ""): "" + w_name + "" ne0="" + str(ne0) + "". Remainder="" + str(ne0 % block_size)) + break + + if not is_valid: + failures.append({{""Group"": g_name, ""Scheme"": scheme}}) + else: + debug_lines.append(""[MISSING] "" + g_name) + +try: + with open(debug_path, 'w') as f: + f.write('\\n'.join(debug_lines)) +except: + pass + +with open(output_path, 'w') as f: + json.dump({{ + ""FoundGroups"": found_groups, + ""Incompatible"": failures, + ""Failures"": failure_details, + ""Error"": None + }}, f, indent=2) +"; + } + + private class CompatResult + { + public List FoundGroups { get; set; } = new(); + public List Incompatible { get; set; } = new(); + public List Failures { get; set; } = new(); + public string? Error { get; set; } + } + + private class CompatFailure + { + public string Group { get; set; } = string.Empty; + public string Scheme { get; set; } = string.Empty; + public string Tensor { get; set; } = string.Empty; + public List Shape { get; set; } = new(); + public int CheckedDimension { get; set; } + public long CheckedValue { get; set; } + public int RequiredMultiple { get; set; } + public long Remainder { get; set; } + public string TensorClass { get; set; } = "unknown"; + public int CheckedTensorCount { get; set; } + } +} diff --git a/src/MagicQuant/Services/ModelRuntimePathService.cs b/src/MagicQuant/Services/ModelRuntimePathService.cs new file mode 100644 index 0000000..239d359 --- /dev/null +++ b/src/MagicQuant/Services/ModelRuntimePathService.cs @@ -0,0 +1,21 @@ +using MQ.DB; + +namespace MagicQuant.Services; + +/// +/// Initializes model-scoped cache locations after the command has selected its source model. +/// +public static class ModelRuntimePathService +{ + public static void InitializeForCurrentModel() + { + if (string.IsNullOrWhiteSpace(Cache.ModelMagicQuantDirectory)) + throw new InvalidOperationException("Cache.ModelMagicQuantDirectory must be set before initializing runtime model paths."); + + string externalName = Config.Current.Paths.ExternalBaselineCacheDirName; + if (string.IsNullOrWhiteSpace(externalName)) + externalName = "ExternalBaselines"; + + Cache.ExternalBaselineCacheDirectory = Path.Combine(Cache.ModelMagicQuantDirectory, externalName); + } +} diff --git a/src/MagicQuant/Services/ModelSidecarArtifactService.cs b/src/MagicQuant/Services/ModelSidecarArtifactService.cs new file mode 100644 index 0000000..6028820 --- /dev/null +++ b/src/MagicQuant/Services/ModelSidecarArtifactService.cs @@ -0,0 +1,273 @@ +using System.Diagnostics; +using System.Text.Json; +using MagicQuant.Helpers; +using MQ.DB; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed record VisionCapabilityDetection( + bool IsLikelyVisionCapable, + bool IsStrongSignal, + List Reasons, + List Warnings); + +public sealed record MmprojArtifactResult +{ + public bool IsVisionCapable { get; init; } + public bool ExistingFound { get; init; } + public bool Built { get; init; } + public bool Copied { get; init; } + public string? SourcePath { get; init; } + public string? OutputPath { get; init; } + public List Warnings { get; init; } = new(); +} + +public sealed class ModelSidecarArtifactService +{ + public const string CanonicalMmprojFileName = "mmproj-BF16.gguf"; + + private static readonly string[] MultimodalTokens = + [ + "llava", "qwen2_vl", "qwen2_5_vl", "qwen3_vl", "gemma3", "internvl", "minicpm", "phi4mm", "glmv", "mllama", "idefics", "florence", "paligemma" + ]; + + private readonly PythonManager _pythonManager; + + public ModelSidecarArtifactService(PythonManager pythonManager) + { + _pythonManager = pythonManager; + } + + public async Task EnsureMmprojArtifactAvailableAsync(CancellationToken ct = default) + { + var detection = DetectVisionCapability(); + var warnings = new List(detection.Warnings); + string? existing = FindExistingMmprojArtifact(); + if (existing != null) + { + AnsiConsole.MarkupLine($"[green]mmproj artifact ready:[/] {Markup.Escape(existing)}"); + return new MmprojArtifactResult { IsVisionCapable = detection.IsLikelyVisionCapable, ExistingFound = true, SourcePath = existing, Warnings = warnings }; + } + + if (!detection.IsLikelyVisionCapable) + { + AnsiConsole.MarkupLine("[grey]No mmproj artifact present and no strong vision capability hints detected. Continuing.[/]"); + return new MmprojArtifactResult { IsVisionCapable = false, Warnings = warnings }; + } + + if (!Config.AttemptMmprojBuild || !detection.IsStrongSignal) + { + string warning = "Model has vision-capability hints, but no mmproj GGUF was found or generated. Continuing without mmproj sidecar. Vision inference may require a separate --mmproj file."; + warnings.Add(warning); + AnsiConsole.MarkupLine($"[yellow]{Markup.Escape(warning)}[/]"); + return HandleStrictRequirement(new MmprojArtifactResult { IsVisionCapable = true, Warnings = warnings }, warning); + } + + var built = await BuildMmprojArtifactAsync(warnings, ct); + if (!built.Built) + return HandleStrictRequirement(built, warnings.LastOrDefault() ?? "mmproj build failed."); + + return built; + } + + public async Task CopyMmprojArtifactsAsync(string outputDirectory, CancellationToken ct = default) + { + var detection = DetectVisionCapability(); + var warnings = new List(detection.Warnings); + Directory.CreateDirectory(outputDirectory); + string target = Path.Combine(outputDirectory, CanonicalMmprojFileName); + + if (Config.ReuseExistingFinalArtifacts && File.Exists(target) && new FileInfo(target).Length > 0) + { + AnsiConsole.MarkupLine($"[green]Reused existing mmproj artifact:[/] {Markup.Escape(target)}"); + return new MmprojArtifactResult { IsVisionCapable = detection.IsLikelyVisionCapable, ExistingFound = true, Copied = false, SourcePath = target, OutputPath = target, Warnings = warnings }; + } + + string? source = FindExistingMmprojArtifact(); + if (source == null) + { + if (detection.IsLikelyVisionCapable) + { + string warning = "Model has vision-capability hints, but no mmproj GGUF was found or generated. Continuing without mmproj sidecar. Vision inference may require a separate --mmproj file."; + warnings.Add(warning); + AnsiConsole.MarkupLine($"[yellow]{Markup.Escape(warning)}[/]"); + } + else + { + AnsiConsole.MarkupLine("[grey]No mmproj artifact present and no strong vision capability hints detected. Continuing.[/]"); + } + + return new MmprojArtifactResult { IsVisionCapable = detection.IsLikelyVisionCapable, Warnings = warnings }; + } + + if (!string.Equals(Path.GetFullPath(source), Path.GetFullPath(target), StringComparison.OrdinalIgnoreCase)) + File.Copy(source, target, overwrite: true); + + await Task.Yield(); + AnsiConsole.MarkupLine($"[green]Copied mmproj artifact:[/] {Markup.Escape(target)}"); + + return new MmprojArtifactResult { IsVisionCapable = detection.IsLikelyVisionCapable, ExistingFound = true, Copied = true, SourcePath = source, OutputPath = target, Warnings = warnings }; + } + + public string? FindExistingMmprojArtifact() + { + if (string.IsNullOrWhiteSpace(Cache.ModelMagicQuantDirectory) || string.IsNullOrWhiteSpace(Cache.ModelDirectory)) + return null; + + var roots = new[] + { + Cache.ModelDirectory!, + Cache.ModelMagicQuantDirectory!, + Path.Combine(Cache.ModelMagicQuantDirectory!, "Sidecars"), + Path.Combine(Cache.ModelMagicQuantDirectory!, "GGUF") + }; + + var distinctRoots = roots.Distinct(StringComparer.OrdinalIgnoreCase).ToList(); + + foreach (var root in distinctRoots) + { + if (!Directory.Exists(root)) + continue; + + string canonical = Path.Combine(root, CanonicalMmprojFileName); + if (File.Exists(canonical) && new FileInfo(canonical).Length > 0) + return canonical; + } + + foreach (var root in distinctRoots) + { + if (!Directory.Exists(root)) + continue; + + var found = Directory.EnumerateFiles(root, "*mmproj*.gguf", SearchOption.TopDirectoryOnly) + .FirstOrDefault(path => new FileInfo(path).Length > 0); + if (found != null) + return found; + } + + return null; + } + + public VisionCapabilityDetection DetectVisionCapability() + { + var reasons = new List(); + var warnings = new List(); + if (string.IsNullOrWhiteSpace(Cache.ModelDirectory)) + return new VisionCapabilityDetection(false, false, reasons, warnings); + + string configPath = Path.Combine(Cache.ModelDirectory!, "config.json"); + if (!File.Exists(configPath)) + { + warnings.Add("config.json not found; unable to evaluate vision capability hints."); + return new VisionCapabilityDetection(false, false, reasons, warnings); + } + + try + { + using var doc = JsonDocument.Parse(File.ReadAllText(configPath)); + var root = doc.RootElement; + string[] strongKeys = ["vision_config", "vision_tower", "mm_vision_tower", "visual", "image_token_id", "video_token_id", "vision_start_token_id", "vision_end_token_id"]; + foreach (var key in strongKeys) + { + if (root.TryGetProperty(key, out _)) + reasons.Add($"config has '{key}'"); + } + + if (root.TryGetProperty("model_type", out var modelType) && modelType.ValueKind == JsonValueKind.String) + { + string value = modelType.GetString() ?? string.Empty; + if (MultimodalTokens.Any(t => value.Contains(t, StringComparison.OrdinalIgnoreCase))) + reasons.Add($"model_type suggests multimodal: {value}"); + } + + if (root.TryGetProperty("architectures", out var archs) && archs.ValueKind == JsonValueKind.Array) + { + foreach (var arch in archs.EnumerateArray()) + { + var value = arch.GetString() ?? string.Empty; + if (MultimodalTokens.Any(t => value.Contains(t, StringComparison.OrdinalIgnoreCase))) + reasons.Add($"architecture suggests multimodal: {value}"); + } + } + } + catch (Exception ex) + { + warnings.Add($"Failed to parse config.json for vision capability hints: {ex.Message}"); + return new VisionCapabilityDetection(false, false, reasons, warnings); + } + + bool likely = reasons.Count > 0; + bool strong = reasons.Any(r => r.Contains("config has", StringComparison.OrdinalIgnoreCase)); + return new VisionCapabilityDetection(likely, strong, reasons, warnings); + } + + private async Task BuildMmprojArtifactAsync(List warnings, CancellationToken ct) + { + string sidecarDir = Path.Combine(Cache.ModelMagicQuantDirectory!, "Sidecars"); + Directory.CreateDirectory(sidecarDir); + string targetPath = Path.Combine(sidecarDir, CanonicalMmprojFileName); + string successPath = targetPath + ".success.json"; + string logPath = targetPath + ".convert.log"; + + if (File.Exists(targetPath) && new FileInfo(targetPath).Length > 0 && File.Exists(successPath)) + { + AnsiConsole.MarkupLine($"[green]mmproj artifact ready:[/] {Markup.Escape(targetPath)}"); + return new MmprojArtifactResult { IsVisionCapable = true, ExistingFound = true, SourcePath = targetPath, Warnings = warnings }; + } + + var psi = new ProcessStartInfo + { + FileName = _pythonManager.GetPythonExecutable(), + WorkingDirectory = Cache.LlamaRoot, + RedirectStandardError = true, + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true + }; + psi.ArgumentList.Add("convert_hf_to_gguf.py"); + psi.ArgumentList.Add(Cache.ModelDirectory!); + psi.ArgumentList.Add("--mmproj"); + psi.ArgumentList.Add("--outtype"); + psi.ArgumentList.Add("f16"); + psi.ArgumentList.Add("--outfile"); + psi.ArgumentList.Add(targetPath); + + using var proc = Process.Start(psi) + ?? throw new InvalidOperationException("Failed to start mmproj conversion process."); + + Task stdoutTask = proc.StandardOutput.ReadToEndAsync(ct); + Task stderrTask = proc.StandardError.ReadToEndAsync(ct); + Task waitTask = proc.WaitForExitAsync(ct); + + await Task.WhenAll(stdoutTask, stderrTask, waitTask); + + string stdout = await stdoutTask; + string stderr = await stderrTask; + + await File.WriteAllTextAsync(logPath, stdout + Environment.NewLine + stderr, ct); + + if (proc.ExitCode != 0 || !File.Exists(targetPath) || new FileInfo(targetPath).Length == 0) + { + if (File.Exists(targetPath)) + await HardDeleteHelper.DeleteFileIfExistsAsync(targetPath); + + string warning = $"Model has vision-capability hints, but no mmproj GGUF was found or generated. Continuing without mmproj sidecar. Vision inference may require a separate --mmproj file. Log: {logPath}"; + warnings.Add(warning); + AnsiConsole.MarkupLine($"[yellow]{Markup.Escape(warning)}[/]"); + return new MmprojArtifactResult { IsVisionCapable = true, Warnings = warnings }; + } + + await File.WriteAllTextAsync(successPath, "{\"status\":\"success\"}", ct); + AnsiConsole.MarkupLine($"[green]mmproj artifact ready:[/] {Markup.Escape(targetPath)}"); + return new MmprojArtifactResult { IsVisionCapable = true, Built = true, SourcePath = targetPath, Warnings = warnings }; + } + + private static MmprojArtifactResult HandleStrictRequirement(MmprojArtifactResult result, string message) + { + if (Config.RequireMmprojForVisionModels && result.IsVisionCapable && !result.ExistingFound && !result.Built) + throw new InvalidOperationException(message); + + return result; + } +} \ No newline at end of file diff --git a/src/MagicQuant/Services/NativeModelConversionService.cs b/src/MagicQuant/Services/NativeModelConversionService.cs new file mode 100644 index 0000000..a064c34 --- /dev/null +++ b/src/MagicQuant/Services/NativeModelConversionService.cs @@ -0,0 +1,127 @@ +using MagicQuant.Helpers; +using MagicQuant.Runtime; +using MQ.DB; +using Spectre.Console; + +namespace MagicQuant.Services; + +/// +/// Owns native GGUF conversion and its success-marker lifecycle. It has no benchmark +/// or learned-truth dependency, so conversion can be exercised independently. +/// +public sealed class NativeModelConversionService(ModelArtifactPathService paths, PythonManager python, IProcessRunner? runner = null) +{ + private readonly ModelArtifactPathService _paths = paths; + private readonly PythonManager _python = python; + private readonly IProcessRunner _runner = runner ?? new ProcessRunner(); + private static readonly SemaphoreSlim BaseModelLock = new(1, 1); + + public async Task EnsureAsync(bool deleteProcess = false) + { + await BaseModelLock.WaitAsync(MagicQuant.Runtime.RunCancellation.Token); + try + { + string modelName = new DirectoryInfo(Cache.ModelDirectory!).Name; + var torchType = Cache.TorchType ?? Cache.MainTorchType.BF16; + string typeStr = torchType.ToString(); + + string fileName = $"{modelName}-{typeStr}.gguf"; + string outputPath = Path.Combine(_paths.GgufDir, fileName); + string successFile = Path.Combine(_paths.GgufDir, $"{fileName}.success.json"); + string convertLogPath = outputPath + ".convert.log"; + + if (deleteProcess) + { + if (!Directory.Exists(_paths.GgufDir)) + Directory.CreateDirectory(_paths.GgufDir); + + var normalizedFileName = Path.GetFileName(fileName); + var successFileName = normalizedFileName + ".success.json"; + var successFilePath = Path.Combine(_paths.GgufDir, successFileName); + bool isImmune = File.Exists(successFilePath); + + foreach (var filePath in Directory.EnumerateFiles(_paths.GgufDir, "*.gguf", SearchOption.TopDirectoryOnly)) + { + var currentFileName = Path.GetFileName(filePath); + var currentModelName = Path.GetFileNameWithoutExtension(currentFileName); + + if (isImmune && + string.Equals(currentFileName, normalizedFileName, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + if (!string.IsNullOrWhiteSpace(currentModelName) && IsProtectedModel(currentModelName)) + { + continue; + } + + await HardDeleteHelper.DeleteFileIfExistsAsync(filePath); + } + } + + if (!File.Exists(outputPath) || new FileInfo(outputPath).Length == 0 || !File.Exists(successFile)) + { + AnsiConsole.MarkupLine($"[bold cyan]Converting to {Markup.Escape(typeStr)}...[/]"); + + await HardDeleteHelper.DeleteFileIfExistsAsync(outputPath); + + string convertScript = Cache.ConvertScript + ?? throw new Exception("ConvertScript path missing in Cache"); + + string outTypeArg = typeStr.ToLowerInvariant(); + + var psi = new MagicQuant.Runtime.NativeCommand(_python.GetPythonExecutable(), + [convertScript, Cache.ModelDirectory!, "--outtype", outTypeArg, "--outfile", outputPath]).CreateStartInfo(); + psi.WorkingDirectory = Cache.LlamaRoot; + + MagicQuant.Runtime.ProcessResult result; + try + { + result = await _runner.RunAsync(psi, convertLogPath, (line, _) => { if (Cache.VerboseProcessOutput) AnsiConsole.WriteLine(line); }, RunCancellation.Token); + } + catch + { + // No success marker may survive an interrupted/failed conversion. + await HardDeleteHelper.DeleteFileIfExistsAsync(outputPath); + await HardDeleteHelper.DeleteFileIfExistsAsync(successFile); + throw; + } + + if (result.ExitCode != 0) + { + await HardDeleteHelper.DeleteFileIfExistsAsync(outputPath); + + throw new Exception( + $"{typeStr} conversion failed. ExitCode={result.ExitCode}. See '{convertLogPath}'."); + } + + if (!File.Exists(outputPath) || new FileInfo(outputPath).Length == 0) + { + await HardDeleteHelper.DeleteFileIfExistsAsync(outputPath); + + throw new InvalidOperationException( + $"Conversion exited successfully but produced no valid GGUF output: {outputPath}"); + } + + await File.WriteAllTextAsync(successFile, "{\"status\":\"success\"}"); + } + + return outputPath; + } + finally + { + BaseModelLock.Release(); + } + } + + + private static bool IsProtectedModel(string name) + { + return name.EndsWith("BF16", StringComparison.OrdinalIgnoreCase) || + name.EndsWith("F16", StringComparison.OrdinalIgnoreCase) || + name.EndsWith("F32", StringComparison.OrdinalIgnoreCase) || + name.EndsWith("Q8_0", StringComparison.OrdinalIgnoreCase); + } + +} diff --git a/src/MagicQuant/Services/OutputPathService.cs b/src/MagicQuant/Services/OutputPathService.cs new file mode 100644 index 0000000..5f503ef --- /dev/null +++ b/src/MagicQuant/Services/OutputPathService.cs @@ -0,0 +1,27 @@ +namespace MagicQuant.Services; + +/// +/// Resolves output locations without creating directories. Relative-path differences +/// are historical command contracts; keep them explicit to avoid relocating artifacts. +/// +public static class OutputPathService +{ + public static string Pipeline(string modelWorkDirectory, string? configuredOutput) => + string.IsNullOrWhiteSpace(configuredOutput) + ? Path.Combine(modelWorkDirectory, "Final_Outputs") + : Path.GetFullPath(Path.Combine(modelWorkDirectory, configuredOutput)); + + public static string Clone(string modelWorkDirectory, string? explicitOutput, string? configuredOutput) => + Path.GetFullPath(!string.IsNullOrWhiteSpace(explicitOutput) + ? explicitOutput + : !string.IsNullOrWhiteSpace(configuredOutput) + ? configuredOutput + : Path.Combine(modelWorkDirectory, "FinalOutput")); + + public static string PredictionValidation(string modelWorkDirectory, string? explicitOutput, string? configuredOutput) => + !string.IsNullOrWhiteSpace(explicitOutput) + ? Path.GetFullPath(explicitOutput) + : !string.IsNullOrWhiteSpace(configuredOutput) + ? Path.Combine(Path.GetFullPath(configuredOutput), "PredictionValidation") + : Path.Combine(modelWorkDirectory, "PredictionValidation"); +} diff --git a/src/MagicQuant/Services/PathSafety.cs b/src/MagicQuant/Services/PathSafety.cs new file mode 100644 index 0000000..5bbafa3 --- /dev/null +++ b/src/MagicQuant/Services/PathSafety.cs @@ -0,0 +1,49 @@ +namespace MagicQuant.Services; + +/// Filesystem-aware containment checks for directories that the program may clean. +public static class PathSafety +{ + private static StringComparison Comparison => OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + + public static string ResolvePhysicalPath(string path) + { + string full = Path.GetFullPath(path); + string current = Path.GetPathRoot(full)!; + foreach (string part in full[current.Length..].Split(Path.DirectorySeparatorChar, StringSplitOptions.RemoveEmptyEntries)) + { + current = Path.Combine(current, part); + var info = new DirectoryInfo(current); + if (info.LinkTarget != null) + current = info.ResolveLinkTarget(returnFinalTarget: true)?.FullName + ?? throw new IOException($"Cannot resolve directory link '{current}'."); + } + return Path.TrimEndingDirectorySeparator(current); + } + + public static bool Contains(string parent, string child) + { + string root = ResolvePhysicalPath(parent); + string candidate = ResolvePhysicalPath(child); + return string.Equals(root, candidate, Comparison) || + candidate.StartsWith(Path.EndsInDirectorySeparator(root) ? root : root + Path.DirectorySeparatorChar, Comparison); + } + + public static void ValidateExportDirectory(string output, string model, string runtimeRoot, params string[] managedDirectories) + { + // An export cannot contain source/runtime data, nor live inside managed working data. + foreach (string protectedPath in new[] { model, runtimeRoot, Path.Combine(model, "MagicQuant") }.Concat(managedDirectories)) + if (Contains(output, protectedPath)) + throw new InvalidOperationException($"Output '{output}' contains protected data '{protectedPath}'. Choose a dedicated export directory."); + foreach (string managed in managedDirectories) + if (Contains(managed, output)) + throw new InvalidOperationException($"Output '{output}' overlaps managed artifacts '{managed}'. Choose a dedicated export directory."); + if (File.Exists(output)) + throw new InvalidOperationException($"Output '{output}' is a file, not a directory."); + } + + public static void ValidateFolderName(string name, string setting) + { + if (string.IsNullOrWhiteSpace(name) || name is "." or ".." || name.IndexOfAny(['/', '\\', ':']) >= 0 || name.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) + throw new InvalidOperationException($"{setting} must be one folder/file name, not a path."); + } +} diff --git a/src/MagicQuant/Services/PredictionGuidedHybridSelectionService.cs b/src/MagicQuant/Services/PredictionGuidedHybridSelectionService.cs new file mode 100644 index 0000000..0dbf32c --- /dev/null +++ b/src/MagicQuant/Services/PredictionGuidedHybridSelectionService.cs @@ -0,0 +1,3108 @@ +using System.Text.Json; +using MagicQuant.Models; +using MagicQuant.Services.Progress; +using Microsoft.EntityFrameworkCore; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models.DbModels; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Services; + +/// +/// Final hybrid chooser driven by the rank-safe isolation prediction engine. +/// +/// This service intentionally does not brute-force the whole remaining DuckDB space. +/// It only validates candidates whose predicted outcome proves one of the user-defined +/// survival claims: +/// 1. strict dominance over a pure/current anchor: lower KLD at same-or-smaller size +/// 2. near-baseline replacement: <= configured small size premium and better-than-linear KLD +/// 3. interior subspace discovery: better-than-linear KLD inside configurable size windows +/// +public sealed class PredictionGuidedHybridSelectionService +{ + private const int DiagnosticPreviewLimit = 25; + private const int DiagnosticPreviewDisplayCount = 8; + + private static readonly JsonSerializerOptions JsonOptions = new() + { + WriteIndented = true + }; + + private readonly QuantizationService _quantizationService; + private readonly HybridBenchmarkRepository _repository; + private readonly FinalRealBenchmarkEliminationService _finalEliminator; + private readonly RemainingCombinationStore _predictedStore; + private readonly SmartBaselineTuningFallbackService _smartFallbackService; + + public PredictionGuidedHybridSelectionService( + QuantizationService quantizationService, + HybridBenchmarkRepository repository, + FinalRealBenchmarkEliminationService finalEliminator, + RemainingCombinationStore predictedStore) + { + _quantizationService = quantizationService; + _repository = repository; + _finalEliminator = finalEliminator; + _predictedStore = predictedStore; + _smartFallbackService = new SmartBaselineTuningFallbackService(repository); + } + + public async Task RunAsync( + IReadOnlyList pureBaselineSnapshots, + CancellationToken ct = default) + { + var eliminationRecords = new List(); + var validationFailures = new List(); + var validationAttempts = new List(); + var phaseDiagnostics = new List(); + + var current = _finalEliminator.Eliminate(pureBaselineSnapshots).Survivors.ToList(); + AnsiConsole.MarkupLine($"[green]Pure/current anchor survivors after dominance:[/] [cyan]{current.Count:N0}[/]"); + PrintAnchorFrontier(current, "Initial anchor frontier after dominance"); + + var predictedAnchors = await _predictedStore.GetPredictedAnchorRowsAsync(ct); + PrintPredictionAnchorFrontier(predictedAnchors, current, "Prediction Anchor Frontier"); + + var strict = await RunStrictDominanceReplacementAsync(current, predictedAnchors, eliminationRecords, validationFailures, validationAttempts, phaseDiagnostics, ct); + current = MergeAndDominanceFilter(current, strict.AcceptedSnapshots, eliminationRecords, "strict predicted hybrid dominance validated by real benchmark"); + + predictedAnchors = AugmentPredictedAnchorsWithAcceptedValidationRows(predictedAnchors, validationAttempts); + + var near = await RunNearBaselineReplacementAsync(current, predictedAnchors, eliminationRecords, validationFailures, validationAttempts, phaseDiagnostics, ct); + current = MergeAndDominanceFilter(current, near.AcceptedSnapshots, eliminationRecords, "near-baseline size-premium replacement validated by real benchmark"); + + predictedAnchors = AugmentPredictedAnchorsWithAcceptedValidationRows(predictedAnchors, validationAttempts); + + var interior = await RunInteriorSubspaceDiscoveryAsync(current, predictedAnchors, validationFailures, validationAttempts, phaseDiagnostics, ct); + current = MergeAndDominanceFilter(current, interior.AcceptedSnapshots, eliminationRecords, "interior subspace discovery dominated by real benchmark truth"); + + var bestConfirmedAnomaly = await LoadBestConfirmedBeneficialAnomalySnapshotAsync(ct); + if (bestConfirmedAnomaly != null && current.All(x => TensorConfigIdentity.ToKey(x.Config) != TensorConfigIdentity.ToKey(bestConfirmedAnomaly.Config))) + { + AnsiConsole.MarkupLine($"[yellow]Best confirmed anomaly reconciliation:[/] adding probe-confirmed anomaly to final frontier consideration: [cyan]{Markup.Escape(bestConfirmedAnomaly.DisplayName)}[/]"); + current = MergeAndDominanceFilter(current, new[] { bestConfirmedAnomaly }, eliminationRecords, "best confirmed beneficial anomaly included for final reconciliation"); + } + + current = ApplyMeaningfulSpacing(current, eliminationRecords); + + var finalDominance = _finalEliminator.Eliminate(current); + foreach (var eliminated in finalDominance.Eliminated) + { + var eliminator = finalDominance.Survivors + .FirstOrDefault(x => Dominates(x, eliminated)); + + if (eliminator != null) + { + eliminationRecords.Add(new BaselineEliminationRecord + { + Eliminated = eliminated, + Eliminator = eliminator, + Reason = "final dominance pass" + }); + } + } + + await WriteAnomalySelectionReconciliationAsync(finalDominance.Survivors, bestConfirmedAnomaly, ct); + await WriteSelectionPhaseDiagnosticsAsync(phaseDiagnostics, validationFailures, validationAttempts, ct); + + return new PredictionGuidedSelectionResult + { + Survivors = finalDominance.Survivors.ToList(), + Eliminations = eliminationRecords + .DistinctBy(x => $"{TensorConfigIdentity.ToKey(x.Eliminated.Config)}::{TensorConfigIdentity.ToKey(x.Eliminator.Config)}::{NormalizePublicEliminationReason(x.Reason)}") + .ToList(), + ValidationFailures = validationFailures + }; + } + + private async Task RunStrictDominanceReplacementAsync( + IReadOnlyList currentAnchors, + IReadOnlyList predictedAnchors, + List eliminations, + List validationFailures, + List validationAttempts, + List phaseDiagnostics, + CancellationToken ct) + { + AnsiConsole.Write(new Rule("[yellow]Prediction Phase 1: Strict Hybrid Dominance[/]") { Justification = Justify.Left }); + AnsiConsole.MarkupLine($"[grey]Strict dominance retry policy:[/] max attempts per anchor=[cyan]{Config.SelectionMaxFallbackAttemptsPerAnchor:N0}[/], epsilon=[cyan]{Config.SelectionMinimumKldImprovementEpsilon:0.########}[/], validate all anomaly/Q8 top-N after first success=[cyan]{Config.SelectionValidateAllAnomalyStrictCandidatesAfterSuccess}[/]"); + + var accepted = new List(); + + foreach (var anchor in currentAnchors.OrderBy(x => x.Kld).ThenBy(x => x.SizeBytes)) + { + if (ShouldSkipAnchorReplacement(anchor)) + { + AnsiConsole.MarkupLine($"[grey]Skipping 8-bit anchor replacement attempts:[/] {Markup.Escape(anchor.DisplayName)}"); + phaseDiagnostics.Add(new SelectionPhaseDiagnostic + { + Phase = "StrictDominanceReplacement", + WindowLabel = $"strict <= {anchor.DisplayName}", + HigherDamageSmaller = ToAnchorLog(anchor), + LowerDamageLarger = ToAnchorLog(anchor), + WindowMinSizeBytes = 0, + WindowMaxSizeBytes = anchor.SizeBytes, + CandidateAttemptLimit = Config.SelectionMaxFallbackAttemptsPerAnchor, + Notes = ["Skipped because SelectionAllowEightBitAnchorReplacements=false and anchor is an 8-bit/non-exact anchor."] + }); + continue; + } + + var predictedAnchor = await _predictedStore.FindPredictedAnchorForRealAnchorAsync(anchor, predictedAnchors, ct); + if (predictedAnchor == null) + { + AnsiConsole.MarkupLine($"[yellow]Skipping strict prediction-space discovery:[/] no predicted virtual anchor matched real anchor [cyan]{Markup.Escape(anchor.DisplayName)}[/]."); + phaseDiagnostics.Add(new SelectionPhaseDiagnostic + { + Phase = "StrictDominanceReplacement", + WindowLabel = $"strict <= {anchor.DisplayName}", + HigherDamageSmaller = ToAnchorLog(anchor), + LowerDamageLarger = ToAnchorLog(anchor), + WindowMinSizeBytes = 0, + WindowMaxSizeBytes = anchor.SizeBytes, + CandidateAttemptLimit = Config.SelectionMaxFallbackAttemptsPerAnchor + Config.SelectionSmartFallbackAttemptsPerFailure, + Notes = ["Skipped because no predicted virtual anchor row was available. DuckDB preselection intentionally does not fall back to real anchor KLD/size. Smart baseline fallback may still inspect SQLite isolation truth."] + }); + + await TryRunSmartStrictFallbackAsync(anchor, accepted, eliminations, validationFailures, validationAttempts, ct); + continue; + } + + int attemptLimit = Config.SelectionMaxFallbackAttemptsPerAnchor; + ulong predictedAnchorSizeBytes = predictedAnchor.PredictedSizeBytes; + ulong realAnchorSizeBytes = anchor.SizeBytes; + ulong effectiveStrictMaxSizeBytes = Math.Min(predictedAnchorSizeBytes, realAnchorSizeBytes); + + long predictedPoolCount = await _predictedStore.CountStrictDominanceCandidatesAsync(predictedAnchor, predictedAnchorSizeBytes, ct); + long poolCount = await _predictedStore.CountStrictDominanceCandidatesAsync(predictedAnchor, effectiveStrictMaxSizeBytes, ct); + long deterministicEligibleCount = poolCount; + long rejectedByRealStrictSizeCeiling = Math.Max(0, predictedPoolCount - deterministicEligibleCount); + long rejectedByOtherPhaseDeterministicRules = 0; + + bool diversityEligible = ShouldUseDiversityForWindow(anchor, anchor, predictedAnchor, predictedAnchor); + int strictScanLimit = ResolveValidationScanLimit(attemptLimit, poolCount, diversityEligible); + var strictRows = await _predictedStore.QueryStrictDominanceCandidatesAsync(predictedAnchor, effectiveStrictMaxSizeBytes, strictScanLimit, ct); + var rankedStrictCandidates = strictRows.Select((x, i) => new HybridSelectionCandidate + { + Prediction = x, + Reason = HybridSelectionReason.StrictDominanceReplacement, + LowerDamageAnchor = anchor, + HigherDamageAnchor = anchor, + LowerDamagePredictionAnchor = predictedAnchor, + HigherDamagePredictionAnchor = predictedAnchor, + PredictionWindowMinSizeBytes = 0, + PredictionWindowMaxSizeBytes = effectiveStrictMaxSizeBytes, + WindowMinSizeBytes = 0, + WindowMaxSizeBytes = effectiveStrictMaxSizeBytes, + LinearExpectedKld = predictedAnchor.PredictedKld, + PredictedGainOverLine = predictedAnchor.PredictedKld - x.PredictedKld, + AttemptOrder = i + 1, + WindowLabel = $"strict <= {anchor.DisplayName}", + CandidatePoolSize = poolCount, + WindowCandidateCount = poolCount, + LineBeatingCandidateCount = poolCount, + FetchedCandidateCount = strictRows.Count, + CandidatesAfterBrutalityCount = strictRows.Count, + CandidateAttemptLimit = attemptLimit, + PhaseWindowIndex = 1, + PhaseWindowCount = 1, + RawSelectionRank = i + 1, + CandidateSelectionNotes = + [ + "Strict DuckDB query uses prediction-space KLD, but predicted-size eligibility is capped by the real anchor size because MagicQuant size prediction is trusted/exact.", + $"predictedAnchorSizeBytes={predictedAnchorSizeBytes:N0}; realAnchorSizeBytes={realAnchorSizeBytes:N0}; effectiveStrictMaxSizeBytes={effectiveStrictMaxSizeBytes:N0}; deterministicEligibleCount={deterministicEligibleCount:N0}; rejectedByRealStrictSizeCeiling={rejectedByRealStrictSizeCeiling:N0}; rejectedByOtherPhaseDeterministicRules={rejectedByOtherPhaseDeterministicRules:N0}." + ] + }).ToList(); + + var selection = SelectValidationCandidates( + rankedStrictCandidates, + attemptLimit, + anchor, + anchor, + predictedAnchor, + predictedAnchor, + "StrictDominanceReplacement", + diversityEligible); + var candidates = selection.Candidates.ToList(); + + var strictNotes = new List + { + "Strict KLD eligibility remains prediction-space, but strict predicted-size eligibility is capped by the real anchor size because MagicQuant size prediction is trusted/exact.", + $"Prediction anchor={predictedAnchor.DisplayName}; predictedKld={predictedAnchor.PredictedKld:0.000000}; predictedAnchorSizeBytes={predictedAnchorSizeBytes:N0}; realKld={anchor.Kld:0.000000}; realAnchorSizeBytes={realAnchorSizeBytes:N0}; effectiveStrictMaxSizeBytes={effectiveStrictMaxSizeBytes:N0}; deterministicEligibleCount={deterministicEligibleCount:N0}; rejectedByRealStrictSizeCeiling={rejectedByRealStrictSizeCeiling:N0}; rejectedByOtherPhaseDeterministicRules={rejectedByOtherPhaseDeterministicRules:N0}." + }; + bool anomalyStrictMode = IsQ8Anchor(anchor) || candidates.Any(x => Math.Abs(x.Prediction.AnomalyAdjustmentKld) > 1e-12); + bool validateAllAfterSuccess = anomalyStrictMode && Config.SelectionValidateAllAnomalyStrictCandidatesAfterSuccess; + if (anomalyStrictMode) + { + strictNotes.Add(validateAllAfterSuccess + ? "Q8/anomaly strict mode: legacy validate-all-after-success is enabled, so all fetched candidates up to the configured attempt limit may be built before choosing by actual KLD/size truth." + : "Q8/anomaly strict mode: stop after the first candidate validates for this anchor. Set candidate_selection.validate_all_anomaly_strict_candidates_after_success=true to restore legacy top-N validation."); + } + + var diag = new SelectionPhaseDiagnostic + { + Phase = "StrictDominanceReplacement", + WindowLabel = $"strict <= {anchor.DisplayName}", + HigherDamageSmaller = ToAnchorLog(anchor), + LowerDamageLarger = ToAnchorLog(anchor), + PredictionHigherDamageSmaller = ToPredictionAnchorLog(predictedAnchor), + PredictionLowerDamageLarger = ToPredictionAnchorLog(predictedAnchor), + PredictionWindowMinSizeBytes = 0, + PredictionWindowMaxSizeBytes = effectiveStrictMaxSizeBytes, + WindowMinSizeBytes = 0, + WindowMaxSizeBytes = effectiveStrictMaxSizeBytes, + CandidatePoolSize = poolCount, + WindowCandidateCount = poolCount, + LineBeatingCandidateCount = poolCount, + FetchedCandidateCount = strictRows.Count, + CandidatesAfterBrutalityCount = strictRows.Count, + SelectedForValidationCount = candidates.Count, + CandidateAttemptLimit = attemptLimit, + QueryFetchLimit = strictScanLimit, + DiversityEnabled = selection.DiversityEnabled, + DiversityMode = selection.Mode, + DiversityScanLimit = strictScanLimit, + DiversityScanFetched = strictRows.Count, + CandidateFamilyCount = selection.CandidateFamilyCount, + SelectedFamilyCount = selection.SelectedFamilyCount, + SelectedFamilyKeys = selection.SelectedFamilyKeys, + DiversitySelectionStrategy = selection.SelectionStrategy, + DiversityOverflowCount = selection.OverflowCount, + DiversitySizeFloorStartBytes = selection.SizeFloorStartBytes, + DiversitySizeFloorEndBytes = selection.SizeFloorEndBytes, + TopCandidates = candidates.Take(DiagnosticPreviewDisplayCount).Select(ToCandidatePreviewLog).ToList(), + Notes = strictNotes.Concat(selection.Notes).ToList() + }; + phaseDiagnostics.Add(diag); + + AnsiConsole.MarkupLine($"[grey]Strict candidates for {Markup.Escape(anchor.DisplayName)}:[/] pool={poolCount:N0}, predictedPoolCount={predictedPoolCount:N0}, deterministicEligibleCount={deterministicEligibleCount:N0}, scanLimit={strictScanLimit:N0}, scanFetched={strictRows.Count:N0}, afterBrutality={strictRows.Count:N0}, diversity={Markup.Escape(selection.Mode)}, selectionStrategy={Markup.Escape(selection.SelectionStrategy)}, candidateFamilies={selection.CandidateFamilyCount:N0}, selectedFamilies={selection.SelectedFamilyCount:N0}, selected={candidates.Count:N0}/{attemptLimit:N0}, overflowCount={selection.OverflowCount:N0}, sizeFloorStart={selection.SizeFloorStartBytes?.ToString("N0") ?? "n/a"}, predictedAnchorSizeBytes={predictedAnchorSizeBytes:N0}, realAnchorSizeBytes={realAnchorSizeBytes:N0}, effectiveStrictMaxSizeBytes={effectiveStrictMaxSizeBytes:N0}, rejectedByRealStrictSizeCeiling={rejectedByRealStrictSizeCeiling:N0}, rejectedByOtherPhaseDeterministicRules={rejectedByOtherPhaseDeterministicRules:N0}, q8/anomaly-mode={anomalyStrictMode}, validate-all-after-success={validateAllAfterSuccess}"); + PrintSelectedCandidateFamilySummary(candidates); + PrintSelectionLadderNotes(selection.Notes); + + if (deterministicEligibleCount == 0) + AnsiConsole.MarkupLine($"[yellow]Strict dominance skipped builds for {Markup.Escape(anchor.DisplayName)}:[/] no physically eligible predicted candidates remained after deterministic size/KLD filters."); + + if (candidates.Count == 0) + { + await TryRunSmartStrictFallbackAsync(anchor, accepted, eliminations, validationFailures, validationAttempts, ct); + continue; + } + + var acceptedForAnchor = new List(); + foreach (var candidate in candidates) + { + var validation = await BuildAndValidateSingleAsync( + candidate, + snapshot => snapshot.SizeBytes <= anchor.SizeBytes && + snapshot.Kld + Config.SelectionMinimumKldImprovementEpsilon < anchor.Kld, + $"must be <= {anchor.SizeBytes:N0} bytes and lower KLD than {anchor.DisplayName}", + ct); + + validationAttempts.Add(validation); + + if (validation.Accepted && validation.Snapshot != null) + { + acceptedForAnchor.Add(validation); + if (!validateAllAfterSuccess) + break; + + continue; + } + + validationFailures.Add(validation); + } + + if (acceptedForAnchor.Count == 0) + { + AnsiConsole.MarkupLine($"[grey]No strict predicted replacement validated for anchor:[/] {Markup.Escape(anchor.DisplayName)}"); + await TryRunSmartStrictFallbackAsync(anchor, accepted, eliminations, validationFailures, validationAttempts, ct); + continue; + } + + var chosen = ChooseBestStrictDominanceCandidate(anchor, acceptedForAnchor); + accepted.Add(chosen.Snapshot!); + eliminations.Add(new BaselineEliminationRecord + { + Eliminated = anchor, + Eliminator = chosen.Snapshot!, + Reason = "strict hybrid dominance: best accepted actual KLD at same-or-smaller real size" + }); + + var nonChosen = acceptedForAnchor + .Where(x => !ReferenceEquals(x, chosen)) + .Select(x => new + { + candidate = x.Snapshot!.DisplayName, + actualKld = x.Snapshot.Kld, + actualSizeBytes = x.Snapshot.SizeBytes, + reasonLost = ExplainStrictAcceptedLoss(anchor, chosen.Snapshot!, x.Snapshot) + }) + .ToList(); + + strictNotes.Add($"validated candidates={validationAttempts.Count(v => v.Candidate.WindowLabel == $"strict <= {anchor.DisplayName}")}; accepted candidates={acceptedForAnchor.Count}; chosen={chosen.Snapshot!.DisplayName}"); + foreach (var loss in nonChosen) + strictNotes.Add($"accepted-but-not-chosen: {loss.candidate} lost because {loss.reasonLost}"); + + AnsiConsole.MarkupLine("[green]Best strict dominance candidate selected:[/]"); + AnsiConsole.MarkupLine($"[grey] anchor=[/] [cyan]{Markup.Escape(anchor.DisplayName)}[/]"); + AnsiConsole.MarkupLine($"[grey] chosen=[/] [cyan]{Markup.Escape(chosen.Snapshot!.DisplayName)}[/]"); + AnsiConsole.MarkupLine($"[grey] actualKld=[/] [cyan]{chosen.Snapshot.Kld:0.000000}[/]"); + AnsiConsole.MarkupLine($"[grey] actualSizeBytes=[/] [cyan]{chosen.Snapshot.SizeBytes:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] gainVsAnchor=[/] [cyan]{anchor.Kld - chosen.Snapshot.Kld:0.000000}[/]"); + AnsiConsole.MarkupLine($"[grey] acceptedCandidateCount=[/] [cyan]{acceptedForAnchor.Count:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] reason=[/] [cyan]{Markup.Escape(ResolveStrictChosenReason(anchor, chosen.Snapshot!))}[/]"); + } + + return new PhaseValidationResult { AcceptedSnapshots = accepted }; + } + + + private async Task TryRunSmartStrictFallbackAsync( + BenchmarkSnapshotRecord anchor, + List accepted, + List eliminations, + List validationFailures, + List validationAttempts, + CancellationToken ct) + { + if (!Config.SelectionSmartFallbackEnabled) + return false; + + var smartCandidates = (await _smartFallbackService.BuildStrictDominanceCandidatesAsync(anchor, 1, 1, ct)).ToList(); + if (smartCandidates.Count == 0) + return false; + + var acceptedForAnchor = new List(); + + foreach (var candidate in smartCandidates) + { + var validation = await BuildAndValidateSingleAsync( + candidate, + snapshot => snapshot.SizeBytes <= anchor.SizeBytes && + snapshot.Kld + Config.SelectionMinimumKldImprovementEpsilon < anchor.Kld, + $"smart fallback must be <= {anchor.SizeBytes:N0} bytes and lower KLD than {anchor.DisplayName}", + ct); + + validationAttempts.Add(validation); + + if (validation.Accepted && validation.Snapshot != null) + { + acceptedForAnchor.Add(validation); + break; + } + + validationFailures.Add(validation); + } + + if (acceptedForAnchor.Count == 0) + { + AnsiConsole.MarkupLine($"[grey]Smart strict fallback found no validated replacement for anchor:[/] {Markup.Escape(anchor.DisplayName)}"); + return false; + } + + var chosen = ChooseBestStrictDominanceCandidate(anchor, acceptedForAnchor); + accepted.Add(chosen.Snapshot!); + eliminations.Add(new BaselineEliminationRecord + { + Eliminated = anchor, + Eliminator = chosen.Snapshot!, + Reason = "smart baseline-tuning strict dominance fallback: real benchmark validated lower KLD at same-or-smaller size" + }); + + AnsiConsole.MarkupLine("[green]Smart strict fallback candidate selected:[/]"); + AnsiConsole.MarkupLine($"[grey] anchor=[/] [cyan]{Markup.Escape(anchor.DisplayName)}[/]"); + AnsiConsole.MarkupLine($"[grey] chosen=[/] [cyan]{Markup.Escape(chosen.Snapshot!.DisplayName)}[/]"); + AnsiConsole.MarkupLine($"[grey] actualKld=[/] [cyan]{chosen.Snapshot.Kld:0.000000}[/]"); + AnsiConsole.MarkupLine($"[grey] actualSizeBytes=[/] [cyan]{chosen.Snapshot.SizeBytes:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] gainVsAnchor=[/] [cyan]{anchor.Kld - chosen.Snapshot.Kld:0.000000}[/]"); + return true; + } + + + + + private async Task LoadBestConfirmedBeneficialAnomalySnapshotAsync(CancellationToken ct) + { + if (!Config.AnomalyDetection.Enabled) + return null; + + await using var db = new MagicQuantContext(); + var modelHashId = await ArchitectureFamilyService.ResolveScopedAiModelHashIdOrNullAsync(db, ct); + if (modelHashId == null) + return null; + + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + int? imatrixId = await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, modelHashId.Value, createIfMissing: false, ct); + + // SQLite cannot translate ulong ordering expressions. Keep the database query + // to filtering/include only, then rank the tiny scoped anomaly observation set + // in LINQ-to-Objects. This preserves the intended ordering without tripping + // Microsoft.Data.Sqlite on SizeSavingsBytes. + var observations = await db.AnomalyProbeObservations + .AsNoTracking() + .Include(x => x.ProbeTensorCombo) + .Where(x => x.ArchitectureFamilyId == architectureFamilyId) + .Where(x => x.TensorGroupProfileId == tensorGroupProfileId) + .Where(x => x.AiModelHashId == modelHashId.Value) + .Where(x => x.ImatrixDefinitionId == imatrixId) + .Where(x => x.BenchmarkCategory == (byte)BenchmarkCategory.General) + .Where(x => x.RuleDirection == AnomalyRuleDirection.Beneficial.ToString()) + .Where(x => x.Accepted) + .Where(x => x.IsContextualAnomalyProbe && !x.OldBf16Isolation && x.AllActiveGroupsExplicit) + .Where(x => x.ProbeTensorCombo != null) + .ToListAsync(ct); + + var observation = observations + .OrderByDescending(x => x.ActualGainVsTwin) + .ThenBy(x => x.ActualKld) + .ThenByDescending(x => x.SizeSavingsBytes) + .FirstOrDefault(); + + if (observation?.ProbeTensorCombo == null) + return null; + + var combo = observation.ProbeTensorCombo; + var config = new TensorConfig(combo.BaseQuant, combo.Embeddings, combo.LmHead, combo.AttnQ, combo.AttnKV, combo.AttnOutput, combo.FfnUpGate, combo.FfnDown, combo.MoeExperts, combo.MoeRouter); + return await _repository.LoadBenchmarkSnapshotAsync(config, ct); + } + + private async Task WriteAnomalySelectionReconciliationAsync( + IReadOnlyList survivors, + BenchmarkSnapshotRecord? bestAnomaly, + CancellationToken ct) + { + if (!Config.AnomalyDetection.Enabled) + return; + + object payload; + if (bestAnomaly == null) + { + payload = new + { + generatedAtUtc = DateTime.UtcNow, + anomalyModeEnabled = true, + bestConfirmedAnomaly = (object?)null, + selectedAnomalyDerivedSurvivor = (object?)null, + bestAnomalyWasSelected = false, + reasonNotSelected = "no confirmed beneficial anomaly observation was available" + }; + } + else + { + string bestKey = TensorConfigIdentity.ToKey(bestAnomaly.Config); + bool selected = survivors.Any(x => TensorConfigIdentity.ToKey(x.Config) == bestKey); + string reason = selected ? string.Empty : ExplainBestAnomalyNotSelected(bestAnomaly, survivors); + payload = new + { + generatedAtUtc = DateTime.UtcNow, + anomalyModeEnabled = true, + bestConfirmedAnomaly = ToAnchorLog(bestAnomaly), + selectedAnomalyDerivedSurvivor = selected ? ToAnchorLog(bestAnomaly) : null, + bestAnomalyWasSelected = selected, + reasonNotSelected = reason, + survivorKeys = survivors.Select(x => new { key = TensorConfigIdentity.ToKey(x.Config), x.DisplayName, x.Kld, x.SizeBytes }).ToList() + }; + + AnsiConsole.MarkupLine("[yellow]Best confirmed beneficial anomaly:[/]"); + AnsiConsole.MarkupLine($"[grey] candidate=[/] [cyan]{Markup.Escape(bestAnomaly.DisplayName)}[/]"); + AnsiConsole.MarkupLine($"[grey] actualCandidateKld=[/] [cyan]{bestAnomaly.Kld:0.000000}[/]"); + AnsiConsole.MarkupLine($"[grey] actualCandidateSizeBytes=[/] [cyan]{bestAnomaly.SizeBytes:N0}[/]"); + AnsiConsole.MarkupLine($"[grey] selectedAsSurvivor=[/] [cyan]{selected}[/]"); + if (!selected) + AnsiConsole.MarkupLine($"[grey] reasonNotSelected=[/] [yellow]{Markup.Escape(reason)}[/]"); + } + + if (!string.IsNullOrWhiteSpace(Cache.OutputDirectory)) + { + string manifestDir = Path.Combine(Cache.OutputDirectory!, "magicquant-manifest"); + Directory.CreateDirectory(manifestDir); + await File.WriteAllTextAsync(Path.Combine(manifestDir, "magicquant.anomaly-selection-reconciliation.json"), JsonSerializer.Serialize(payload, JsonOptions), ct); + } + } + + private static string ExplainBestAnomalyNotSelected(BenchmarkSnapshotRecord bestAnomaly, IReadOnlyList survivors) + { + var dominator = survivors.FirstOrDefault(x => x.SizeBytes <= bestAnomaly.SizeBytes && x.Kld <= bestAnomaly.Kld && (x.SizeBytes < bestAnomaly.SizeBytes || x.Kld < bestAnomaly.Kld)); + if (dominator != null) + return $"dominated by survivor {dominator.DisplayName}"; + + var lower = survivors.OrderBy(x => x.Kld).ThenBy(x => x.SizeBytes).FirstOrDefault(); + if (lower != null && lower.Kld < bestAnomaly.Kld) + return $"survivor {lower.DisplayName} has lower actual KLD; spacing/final frontier kept that candidate"; + + return "not selected after spacing/final dominance; no direct dominator found"; + } + + private static bool IsQ8Anchor(BenchmarkSnapshotRecord anchor) + { + try + { + return BaselineQuants.FromId(anchor.Config.BaseQuant).Names.Any(x => x.Contains("Q8", StringComparison.OrdinalIgnoreCase)); + } + catch + { + return anchor.DisplayName.Contains("Q8", StringComparison.OrdinalIgnoreCase); + } + } + + private static IReadOnlyList AugmentPredictedAnchorsWithAcceptedValidationRows( + IReadOnlyList predictedAnchors, + IReadOnlyList validationAttempts) + { + if (validationAttempts.Count == 0) + return predictedAnchors; + + var result = new List(predictedAnchors); + var knownKeys = result + .Select(x => x.ConfigKey) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .ToHashSet(StringComparer.Ordinal); + + int added = 0; + + foreach (var validation in validationAttempts) + { + if (!validation.Accepted || validation.Snapshot == null) + continue; + + var prediction = validation.Candidate.Prediction; + if (!prediction.IsPredictable || !prediction.IsSizePredictable) + continue; + + var predictionSpaceConfig = CanonicalizeSelectionConfigForPredictionSpace(prediction.Config); + string key = TensorConfigIdentity.ToKey(predictionSpaceConfig); + if (!knownKeys.Add(key)) + continue; + + var sourceBaseline = HybridBenchmarkRepository.ResolveSourceBaselineForProvider(validation.Snapshot.Quant); + + result.Add(new PredictedAnchorRow + { + Config = predictionSpaceConfig, + ConfigKey = key, + DisplayName = validation.Snapshot.DisplayName, + BaselineCanonicalKey = sourceBaseline.CanonicalKey, + RuntimeBaselineId = sourceBaseline.UniqueId, + PredictedKld = prediction.PredictedKld, + PredictedSizeBytes = prediction.PredictedSizeBytes, + PredictionConfidence = prediction.PredictionConfidence, + PredictionRank = prediction.PredictedRank ?? ulong.MaxValue, + IsVirtualPredictionAnchor = false + }); + + added++; + } + + if (added > 0) + { + AnsiConsole.MarkupLine( + $"[grey]Prediction anchor frontier augmented from accepted validation rows:[/] [cyan]{added:N0}[/] phase-local anchor(s) added for smart-fallback / accepted hybrid coordinates outside the pruned DuckDB row set."); + } + + return added == 0 ? predictedAnchors : result; + } + + private static TensorConfig CanonicalizeSelectionConfigForPredictionSpace(TensorConfig config) + { + if (config.BaseQuant == BaselineQuants.Q8_0.UniqueId) + return config; + + var baseBaseline = BaselineQuants.FromId(config.BaseQuant); + byte inheritedBaseSlot = BaselineQuants.EncodeTensorConfigGroupSlot(baseBaseline); + + return new TensorConfig( + baseQuant: BaselineQuants.Q8_0.UniqueId, + embeddings: CanonicalizeSelectionPredictionSlot(TReg.Embeddings, config.Embeddings, inheritedBaseSlot), + lmHead: CanonicalizeSelectionPredictionSlot(TReg.LmHead, config.LmHead, inheritedBaseSlot), + attnQ: CanonicalizeSelectionPredictionSlot(TReg.AttnQ, config.AttnQ, inheritedBaseSlot), + attnKV: CanonicalizeSelectionPredictionSlot(TReg.AttnKV, config.AttnKV, inheritedBaseSlot), + attnOutput: CanonicalizeSelectionPredictionSlot(TReg.AttnOutput, config.AttnOutput, inheritedBaseSlot), + ffnUpGate: CanonicalizeSelectionPredictionSlot(TReg.FfnUpGate, config.FfnUpGate, inheritedBaseSlot), + ffnDown: CanonicalizeSelectionPredictionSlot(TReg.FfnDown, config.FfnDown, inheritedBaseSlot), + moeExperts: CanonicalizeSelectionPredictionSlot(TReg.MoeExperts, config.MoeExperts, inheritedBaseSlot), + moeRouter: CanonicalizeSelectionPredictionSlot(TReg.MoeRouter, config.MoeRouter, inheritedBaseSlot)); + } + + private static byte CanonicalizeSelectionPredictionSlot(TensorGroup group, byte storedValue, byte inheritedBaseSlot) + { + if (Cache.UnusedTensorGroups.Any(x => x.UniqueId == group.UniqueId)) + return BaselineQuants.TensorConfigNullSlotValue; + + return BaselineQuants.IsNullTensorConfigGroupSlot(storedValue) + ? inheritedBaseSlot + : storedValue; + } + + private static CandidateValidationResult ChooseBestStrictDominanceCandidate( + BenchmarkSnapshotRecord anchor, + IReadOnlyList accepted) + { + return accepted + .Where(x => x.Snapshot != null) + .OrderBy(x => x.Snapshot!.Kld) + .ThenBy(x => x.Snapshot!.SizeBytes) + .ThenByDescending(x => anchor.Kld - x.Snapshot!.Kld) + .ThenBy(x => x.Candidate.Prediction.PredictedRank ?? ulong.MaxValue) + .ThenByDescending(x => x.Candidate.Prediction.PredictionConfidence) + .First(); + } + + private static string ResolveStrictChosenReason(BenchmarkSnapshotRecord anchor, BenchmarkSnapshotRecord chosen) + => $"lowest actual KLD among accepted strict dominance candidates, then smaller actual size, gainVsAnchor={anchor.Kld - chosen.Kld:0.000000}"; + + private static string ExplainStrictAcceptedLoss( + BenchmarkSnapshotRecord anchor, + BenchmarkSnapshotRecord chosen, + BenchmarkSnapshotRecord loser) + { + if (loser.Kld > chosen.Kld) + return $"higher actual KLD ({loser.Kld:0.000000} > {chosen.Kld:0.000000})"; + + if (Math.Abs(loser.Kld - chosen.Kld) < 1e-12 && loser.SizeBytes > chosen.SizeBytes) + return $"same actual KLD but larger actual size ({loser.SizeBytes:N0} > {chosen.SizeBytes:N0})"; + + double chosenGain = anchor.Kld - chosen.Kld; + double loserGain = anchor.Kld - loser.Kld; + if (Math.Abs(loser.Kld - chosen.Kld) < 1e-12 && loser.SizeBytes == chosen.SizeBytes && loserGain < chosenGain) + return $"weaker gain over anchor ({loserGain:0.000000} < {chosenGain:0.000000})"; + + return "lost by prediction rank/confidence tie-breaker after actual KLD and size were equivalent"; + } + + private async Task RunNearBaselineReplacementAsync( + IReadOnlyList currentAnchors, + IReadOnlyList predictedAnchors, + List eliminations, + List validationFailures, + List validationAttempts, + List phaseDiagnostics, + CancellationToken ct) + { + AnsiConsole.Write(new Rule("[yellow]Prediction Phase 2: Near-Baseline Replacement[/]") { Justification = Justify.Left }); + + var accepted = new List(); + var pairs = BuildAdjacentPairs(currentAnchors); + int attemptLimit = Math.Max(1, Config.SelectionMaxFallbackAttemptsPerAnchor); + int fetchLimit = ResolveValidationScanLimit(attemptLimit, long.MaxValue, diversityEligible: Config.SelectionDiversifyValidationCandidates); + + AnsiConsole.MarkupLine($"[grey]Near-baseline neighbor pairs:[/] [cyan]{pairs.Count:N0}[/] | size premium=[cyan]{Config.SelectionNearBaselineMaxSizeGrowthPercent:0.###}%[/] | max scan/window=[cyan]{fetchLimit:N0}[/] | validation attempts/window=[cyan]{attemptLimit:N0}[/]"); + + for (int pairIndex = 0; pairIndex < pairs.Count; pairIndex++) + { + var pair = pairs[pairIndex]; + var lowerSizeHigherDamage = pair.HigherDamageSmaller; + var upperSizeLowerDamage = pair.LowerDamageLarger; + string windowLabel = $"near-baseline +{Config.SelectionNearBaselineMaxSizeGrowthPercent:0.###}% {lowerSizeHigherDamage.DisplayName}"; + + if (ShouldSkipAnchorReplacement(lowerSizeHigherDamage)) + { + AnsiConsole.MarkupLine($"[grey]Skipping near-baseline lower anchor replacement:[/] {Markup.Escape(lowerSizeHigherDamage.DisplayName)}"); + phaseDiagnostics.Add(new SelectionPhaseDiagnostic + { + Phase = "NearBaselineReplacement", + WindowLabel = windowLabel, + PhaseWindowIndex = pairIndex + 1, + PhaseWindowCount = pairs.Count, + HigherDamageSmaller = ToAnchorLog(lowerSizeHigherDamage), + LowerDamageLarger = ToAnchorLog(upperSizeLowerDamage), + Notes = ["Skipped because the smaller/higher-damage anchor is an 8-bit/non-exact anchor and SelectionAllowEightBitAnchorReplacements=false."] + }); + continue; + } + + ulong realMin = lowerSizeHigherDamage.SizeBytes; + ulong realMax = AddPercent(realMin, Config.SelectionNearBaselineMaxSizeGrowthPercent); + + if (realMax > upperSizeLowerDamage.SizeBytes) + realMax = upperSizeLowerDamage.SizeBytes; + + if (realMax <= realMin) + { + AnsiConsole.MarkupLine($"[grey]Skipping near-baseline pair with empty real window:[/] {Markup.Escape(lowerSizeHigherDamage.DisplayName)} -> {Markup.Escape(upperSizeLowerDamage.DisplayName)}"); + continue; + } + + var predictedLowerSizeHigherDamage = await _predictedStore.FindPredictedAnchorForRealAnchorAsync(lowerSizeHigherDamage, predictedAnchors, ct); + var predictedUpperSizeLowerDamage = await _predictedStore.FindPredictedAnchorForRealAnchorAsync(upperSizeLowerDamage, predictedAnchors, ct); + if (predictedLowerSizeHigherDamage == null || predictedUpperSizeLowerDamage == null) + { + AnsiConsole.MarkupLine($"[yellow]Skipping near-baseline prediction-space discovery:[/] missing predicted anchor for pair [cyan]{Markup.Escape(lowerSizeHigherDamage.DisplayName)}[/] -> [cyan]{Markup.Escape(upperSizeLowerDamage.DisplayName)}[/]."); + phaseDiagnostics.Add(new SelectionPhaseDiagnostic + { + Phase = "NearBaselineReplacement", + WindowLabel = windowLabel, + PhaseWindowIndex = pairIndex + 1, + PhaseWindowCount = pairs.Count, + HigherDamageSmaller = ToAnchorLog(lowerSizeHigherDamage), + LowerDamageLarger = ToAnchorLog(upperSizeLowerDamage), + PredictionHigherDamageSmaller = ToPredictionAnchorLog(predictedLowerSizeHigherDamage), + PredictionLowerDamageLarger = ToPredictionAnchorLog(predictedUpperSizeLowerDamage), + WindowMinSizeBytes = realMin, + WindowMaxSizeBytes = realMax, + CandidateAttemptLimit = Config.SelectionMaxFallbackAttemptsPerAnchor + Config.SelectionSmartFallbackAttemptsPerFailure, + Notes = ["Skipped because one or both predicted virtual anchor rows were unavailable. DuckDB preselection intentionally does not fall back to real anchor KLD/size. Smart baseline fallback may still inspect SQLite isolation truth."] + }); + + await TryRunSmartNearFallbackAsync(lowerSizeHigherDamage, upperSizeLowerDamage, realMin, realMax, pairIndex + 1, pairs.Count, accepted, eliminations, validationFailures, validationAttempts, ct); + continue; + } + + ulong predictionMin = predictedLowerSizeHigherDamage.PredictedSizeBytes; + ulong predictionMax = AddPercent(predictionMin, Config.SelectionNearBaselineMaxSizeGrowthPercent); + + if (predictionMax > predictedUpperSizeLowerDamage.PredictedSizeBytes) + predictionMax = predictedUpperSizeLowerDamage.PredictedSizeBytes; + + if (predictionMax <= predictionMin || realMax <= realMin) + { + AnsiConsole.MarkupLine($"[grey]Skipping near-baseline pair with empty prediction/real window:[/] {Markup.Escape(lowerSizeHigherDamage.DisplayName)} -> {Markup.Escape(upperSizeLowerDamage.DisplayName)}"); + await TryRunSmartNearFallbackAsync(lowerSizeHigherDamage, upperSizeLowerDamage, realMin, realMax, pairIndex + 1, pairs.Count, accepted, eliminations, validationFailures, validationAttempts, ct); + continue; + } + + LogPredictionAndRealPairLines(lowerSizeHigherDamage, upperSizeLowerDamage, predictedLowerSizeHigherDamage, predictedUpperSizeLowerDamage, "Near-baseline pair"); + + long predictedWindowRows = await _predictedStore.CountPredictedHybridCandidatesInSizeWindowAsync(predictionMin, predictionMax, ct); + long predictedPoolCount = await _predictedStore.CountBetterThanLinearCandidatesAsync(predictedLowerSizeHigherDamage, predictedUpperSizeLowerDamage, predictionMin, predictionMax, ct); + long deterministicWindowRows = await CountPredictedRowsInIntersectedSizeWindowAsync(predictionMin, predictionMax, realMin, realMax, ct); + long phaseSizeEligiblePool = await _predictedStore.CountBetterThanLinearCandidatesAsync( + predictedLowerSizeHigherDamage, + predictedUpperSizeLowerDamage, + predictionMin, + predictionMax, + realMin, + realMax, + ct); + long rejectedByRealSizeWindow = Math.Max(0, predictedPoolCount - phaseSizeEligiblePool); + bool diversityEligible = ShouldUseDiversityForWindow(lowerSizeHigherDamage, upperSizeLowerDamage, predictedLowerSizeHigherDamage, predictedUpperSizeLowerDamage); + fetchLimit = ResolveValidationScanLimit(attemptLimit, phaseSizeEligiblePool, diversityEligible); + var rawCandidates = (await _predictedStore.QueryBetterThanLinearCandidatesAsync( + lowerSizeHigherDamage, + upperSizeLowerDamage, + predictedLowerSizeHigherDamage, + predictedUpperSizeLowerDamage, + predictionMin, + predictionMax, + realMin, + realMax, + HybridSelectionReason.NearBaselineOnePercentReplacement, + windowLabel, + fetchLimit, + ct)).ToList(); + + var brutalityAnalyses = rawCandidates + .Select((x, rawIndex) => new { Candidate = x, Brutality = AnalyzeNearLowerAnchorBrutality(x), RawRank = rawIndex + 1 }) + .ToList(); + + int afterBrutalityCount = brutalityAnalyses.Count(y => y.Brutality.Passed); + long deterministicEligibleCount = afterBrutalityCount; + long rejectedByOtherPhaseDeterministicRules = Math.Max(0, rawCandidates.Count - afterBrutalityCount); + var rankedCandidates = brutalityAnalyses + .Where(x => x.Brutality.Passed) + .Select(x => AttachSelectionDiagnostics( + x.Candidate, + poolSize: phaseSizeEligiblePool, + windowCandidateCount: deterministicWindowRows, + lineBeatingCandidateCount: phaseSizeEligiblePool, + fetchedCandidateCount: rawCandidates.Count, + candidatesAfterBrutalityCount: afterBrutalityCount, + candidateAttemptLimit: attemptLimit, + phaseWindowIndex: pairIndex + 1, + phaseWindowCount: pairs.Count, + notes: [x.Brutality.Explanation], + rawSelectionRank: x.RawRank)) + .ToList(); + + var selection = SelectValidationCandidates( + rankedCandidates, + attemptLimit, + lowerSizeHigherDamage, + upperSizeLowerDamage, + predictedLowerSizeHigherDamage, + predictedUpperSizeLowerDamage, + "NearBaselineReplacement", + diversityEligible); + var candidates = selection.Candidates.ToList(); + + var rejectedByBrutality = brutalityAnalyses + .Where(x => !x.Brutality.Passed) + .Take(DiagnosticPreviewDisplayCount) + .Select(x => ToCandidatePreviewLog(x.Candidate, x.Brutality)) + .ToList(); + + var diag = new SelectionPhaseDiagnostic + { + Phase = "NearBaselineReplacement", + WindowLabel = windowLabel, + PhaseWindowIndex = pairIndex + 1, + PhaseWindowCount = pairs.Count, + HigherDamageSmaller = ToAnchorLog(lowerSizeHigherDamage), + LowerDamageLarger = ToAnchorLog(upperSizeLowerDamage), + PredictionHigherDamageSmaller = ToPredictionAnchorLog(predictedLowerSizeHigherDamage), + PredictionLowerDamageLarger = ToPredictionAnchorLog(predictedUpperSizeLowerDamage), + PredictionWindowMinSizeBytes = predictionMin, + PredictionWindowMaxSizeBytes = predictionMax, + WindowMinSizeBytes = realMin, + WindowMaxSizeBytes = realMax, + WindowSizeGiB = ToGiB(realMax > realMin ? realMax - realMin : 0), + CandidatePoolSize = phaseSizeEligiblePool, + PredictedPoolCount = predictedPoolCount, + DeterministicEligibleCount = deterministicEligibleCount, + RejectedByRealSizeWindow = rejectedByRealSizeWindow, + RejectedByOtherPhaseDeterministicRules = rejectedByOtherPhaseDeterministicRules, + WindowCandidateCount = deterministicWindowRows, + LineBeatingCandidateCount = phaseSizeEligiblePool, + FetchedCandidateCount = rawCandidates.Count, + CandidatesAfterBrutalityCount = afterBrutalityCount, + SelectedForValidationCount = candidates.Count, + CandidateAttemptLimit = attemptLimit, + QueryFetchLimit = fetchLimit, + DiversityEnabled = selection.DiversityEnabled, + DiversityMode = selection.Mode, + DiversityScanLimit = fetchLimit, + DiversityScanFetched = rawCandidates.Count, + CandidateFamilyCount = selection.CandidateFamilyCount, + SelectedFamilyCount = selection.SelectedFamilyCount, + SelectedFamilyKeys = selection.SelectedFamilyKeys, + DiversitySelectionStrategy = selection.SelectionStrategy, + DiversityOverflowCount = selection.OverflowCount, + DiversitySizeFloorStartBytes = selection.SizeFloorStartBytes, + DiversitySizeFloorEndBytes = selection.SizeFloorEndBytes, + TopCandidates = candidates.Take(DiagnosticPreviewDisplayCount).Select(ToCandidatePreviewLog).ToList(), + RejectedByBrutalityPreview = rejectedByBrutality, + Notes = new[] + { + "Near-baseline DuckDB discovery uses the predicted virtual KLD line, then deterministically caps candidate size to the real validation window before diversity/ladder selection. Real KLD is still used only after benchmark validation.", + $"predictedPoolCount={predictedPoolCount:N0}; phaseSizeEligiblePool={phaseSizeEligiblePool:N0}; deterministicEligibleCount={deterministicEligibleCount:N0}; rejectedByRealSizeWindow={rejectedByRealSizeWindow:N0}; rejectedByOtherPhaseDeterministicRules={rejectedByOtherPhaseDeterministicRules:N0}.", + $"Brutal zone fraction={Config.SelectionNearLowerAnchorBrutalZoneFractionOfPairSpan:0.###}; required gain fraction of pair KLD gap={Config.SelectionNearAnchorRequiredKldGainFractionOfPairGap:0.###}." + }.Concat(selection.Notes).ToList() + }; + phaseDiagnostics.Add(diag); + + AnsiConsole.MarkupLine( + $"[grey]Near-baseline window {pairIndex + 1:N0}/{pairs.Count:N0}:[/] {Markup.Escape(lowerSizeHigherDamage.DisplayName)} -> {Markup.Escape(upperSizeLowerDamage.DisplayName)} " + + $"| pred-window={predictionMin:N0}..{predictionMax:N0}, real-window={realMin:N0}..{realMax:N0}, predictedPoolCount={predictedPoolCount:N0}, phaseSizeEligiblePool={phaseSizeEligiblePool:N0}, deterministicEligibleCount={deterministicEligibleCount:N0}, rejectedByRealSizeWindow={rejectedByRealSizeWindow:N0}, rejectedByOtherPhaseDeterministicRules={rejectedByOtherPhaseDeterministicRules:N0}, rows-in-window={deterministicWindowRows:N0}, beat-line={phaseSizeEligiblePool:N0}, scanLimit={fetchLimit:N0}, scanFetched={rawCandidates.Count:N0}, after-brutality={afterBrutalityCount:N0}, diversity={Markup.Escape(selection.Mode)}, selectionStrategy={Markup.Escape(selection.SelectionStrategy)}, candidateFamilies={selection.CandidateFamilyCount:N0}, selectedFamilies={selection.SelectedFamilyCount:N0}, selected={candidates.Count:N0}/{attemptLimit:N0}, overflowCount={selection.OverflowCount:N0}, sizeFloorStart={selection.SizeFloorStartBytes?.ToString("N0") ?? "n/a"}"); + PrintSelectedCandidateFamilySummary(candidates); + PrintSelectionLadderNotes(selection.Notes); + + if (deterministicEligibleCount == 0) + AnsiConsole.MarkupLine($"[yellow]Near-baseline window skipped builds:[/] no physically eligible candidates remained after predicted line, real size window, and deterministic brutality filters."); + + if (rejectedByBrutality.Count > 0) + AnsiConsole.MarkupLine($"[grey] rejected by near-lower-anchor brutality preview:[/] [cyan]{rejectedByBrutality.Count:N0}[/] (see magicquant-selection-phase-diagnostics.json)"); + + if (candidates.Count == 0) + { + await TryRunSmartNearFallbackAsync(lowerSizeHigherDamage, upperSizeLowerDamage, realMin, realMax, pairIndex + 1, pairs.Count, accepted, eliminations, validationFailures, validationAttempts, ct); + continue; + } + + bool acceptedThisPair = false; + foreach (var candidate in candidates) + { + var validation = await BuildAndValidateSingleAsync( + candidate, + snapshot => snapshot.SizeBytes >= realMin && + snapshot.SizeBytes <= realMax && + BeatsLinearKldLine(snapshot.SizeBytes, snapshot.Kld, lowerSizeHigherDamage, upperSizeLowerDamage), + $"must land inside {realMin:N0}..{realMax:N0} bytes and beat the real linear KLD line", + ct); + + validationAttempts.Add(validation); + + if (validation.Accepted && validation.Snapshot != null) + { + accepted.Add(validation.Snapshot); + eliminations.Add(new BaselineEliminationRecord + { + Eliminated = lowerSizeHigherDamage, + Eliminator = validation.Snapshot, + Reason = $"near-baseline replacement within +{Config.SelectionNearBaselineMaxSizeGrowthPercent:0.###}% size premium" + }); + acceptedThisPair = true; + break; + } + + validationFailures.Add(validation); + } + + if (!acceptedThisPair) + { + await TryRunSmartNearFallbackAsync(lowerSizeHigherDamage, upperSizeLowerDamage, realMin, realMax, pairIndex + 1, pairs.Count, accepted, eliminations, validationFailures, validationAttempts, ct); + } + } + + return new PhaseValidationResult { AcceptedSnapshots = accepted }; + } + + + private async Task TryRunSmartNearFallbackAsync( + BenchmarkSnapshotRecord lowerSizeHigherDamage, + BenchmarkSnapshotRecord upperSizeLowerDamage, + ulong realMin, + ulong realMax, + int phaseWindowIndex, + int phaseWindowCount, + List accepted, + List eliminations, + List validationFailures, + List validationAttempts, + CancellationToken ct) + { + if (!Config.SelectionSmartFallbackEnabled) + return false; + + var smartCandidates = (await _smartFallbackService.BuildNearBaselineCandidatesAsync( + lowerSizeHigherDamage, + upperSizeLowerDamage, + realMin, + realMax, + phaseWindowIndex, + phaseWindowCount, + ct)).ToList(); + + if (smartCandidates.Count == 0) + return false; + + foreach (var candidate in smartCandidates) + { + var validation = await BuildAndValidateSingleAsync( + candidate, + snapshot => snapshot.SizeBytes >= realMin && + snapshot.SizeBytes <= realMax && + BeatsLinearKldLine(snapshot.SizeBytes, snapshot.Kld, lowerSizeHigherDamage, upperSizeLowerDamage), + $"smart fallback must land inside {realMin:N0}..{realMax:N0} bytes and beat the real linear KLD line", + ct); + + validationAttempts.Add(validation); + + if (validation.Accepted && validation.Snapshot != null) + { + accepted.Add(validation.Snapshot); + eliminations.Add(new BaselineEliminationRecord + { + Eliminated = lowerSizeHigherDamage, + Eliminator = validation.Snapshot, + Reason = $"smart baseline-tuning near-baseline fallback within +{Config.SelectionNearBaselineMaxSizeGrowthPercent:0.###}% size premium" + }); + + AnsiConsole.MarkupLine("[green]Smart near-baseline fallback candidate selected:[/]"); + AnsiConsole.MarkupLine($"[grey] lower anchor=[/] [cyan]{Markup.Escape(lowerSizeHigherDamage.DisplayName)}[/]"); + AnsiConsole.MarkupLine($"[grey] upper anchor=[/] [cyan]{Markup.Escape(upperSizeLowerDamage.DisplayName)}[/]"); + AnsiConsole.MarkupLine($"[grey] chosen=[/] [cyan]{Markup.Escape(validation.Snapshot.DisplayName)}[/]"); + AnsiConsole.MarkupLine($"[grey] actualKld=[/] [cyan]{validation.Snapshot.Kld:0.000000}[/]"); + AnsiConsole.MarkupLine($"[grey] actualSizeBytes=[/] [cyan]{validation.Snapshot.SizeBytes:N0}[/]"); + return true; + } + + validationFailures.Add(validation); + } + + AnsiConsole.MarkupLine($"[grey]Smart near-baseline fallback found no validated candidate for:[/] {Markup.Escape(lowerSizeHigherDamage.DisplayName)} -> {Markup.Escape(upperSizeLowerDamage.DisplayName)}"); + return false; + } + + + private async Task RunInteriorSubspaceDiscoveryAsync( + IReadOnlyList currentAnchors, + IReadOnlyList predictedAnchors, + List validationFailures, + List validationAttempts, + List phaseDiagnostics, + CancellationToken ct) + { + AnsiConsole.Write(new Rule("[yellow]Prediction Phase 3: Interior Subspace Discovery[/]") { Justification = Justify.Left }); + + var accepted = new List(); + var pairs = BuildAdjacentPairs(currentAnchors); + var fractions = Config.SelectionInteriorWindowFractions.ToList(); + + int interiorAttemptLimit = Math.Max(1, Math.Max(Config.SelectionMaxCandidatesPerInteriorWindow, Config.SelectionMaxFallbackAttemptsPerAnchor)); + int interiorFetchLimit = ResolveValidationScanLimit(interiorAttemptLimit, long.MaxValue, diversityEligible: Config.SelectionDiversifyValidationCandidates); + + AnsiConsole.MarkupLine($"[grey]Interior neighbor pairs:[/] [cyan]{pairs.Count:N0}[/] | window fractions=[cyan]{Markup.Escape(string.Join(", ", fractions.Select(x => x.ToString("0.###"))))}[/] | candidates/window=[cyan]{Config.SelectionMaxCandidatesPerInteriorWindow:N0}[/] | fallback attempts/window=[cyan]{Config.SelectionMaxFallbackAttemptsPerAnchor:N0}[/] | validation attempts/window=[cyan]{interiorAttemptLimit:N0}[/] | max scan/window=[cyan]{interiorFetchLimit:N0}[/]"); + + var allCandidates = new List(); + int globalWindowIndex = 0; + int estimatedWindowCount = pairs.Sum(pair => EstimateInteriorWindowCount(pair, fractions)); + + for (int pairIndex = 0; pairIndex < pairs.Count; pairIndex++) + { + var pair = pairs[pairIndex]; + var predictedHigherDamageSmaller = await _predictedStore.FindPredictedAnchorForRealAnchorAsync(pair.HigherDamageSmaller, predictedAnchors, ct); + var predictedLowerDamageLarger = await _predictedStore.FindPredictedAnchorForRealAnchorAsync(pair.LowerDamageLarger, predictedAnchors, ct); + if (predictedHigherDamageSmaller == null || predictedLowerDamageLarger == null) + { + AnsiConsole.MarkupLine($"[yellow]Skipping interior prediction-space discovery:[/] missing predicted anchor for pair [cyan]{Markup.Escape(pair.HigherDamageSmaller.DisplayName)}[/] -> [cyan]{Markup.Escape(pair.LowerDamageLarger.DisplayName)}[/]."); + continue; + } + + ulong realLowSize = pair.HigherDamageSmaller.SizeBytes; + ulong realHighSize = pair.LowerDamageLarger.SizeBytes; + ulong predictionLowSize = predictedHigherDamageSmaller.PredictedSizeBytes; + ulong predictionHighSize = predictedLowerDamageLarger.PredictedSizeBytes; + + if (realHighSize <= realLowSize || predictionHighSize <= predictionLowSize) + continue; + + LogPredictionAndRealPairLines(pair.HigherDamageSmaller, pair.LowerDamageLarger, predictedHigherDamageSmaller, predictedLowerDamageLarger, "Interior pair"); + + ulong realSpan = realHighSize - realLowSize; + ulong predictionSpan = predictionHighSize - predictionLowSize; + ulong realCursor = realLowSize; + ulong predictionCursor = predictionLowSize; + + for (int i = 0; i < fractions.Count; i++) + { + double fraction = fractions[i]; + if (fraction <= 0d) + continue; + + ulong realWidth = (ulong)Math.Round(realSpan * fraction, MidpointRounding.AwayFromZero); + ulong predictionWidth = (ulong)Math.Round(predictionSpan * fraction, MidpointRounding.AwayFromZero); + if (realWidth == 0 || predictionWidth == 0) + continue; + + ulong realMin = realCursor; + ulong realMax = Math.Min(realHighSize, realCursor + realWidth); + ulong predictionMin = predictionCursor; + ulong predictionMax = Math.Min(predictionHighSize, predictionCursor + predictionWidth); + + if (realMax <= realMin || predictionMax <= predictionMin) + continue; + + globalWindowIndex++; + string windowLabel = $"interior {i + 1}: {pair.HigherDamageSmaller.DisplayName} -> {pair.LowerDamageLarger.DisplayName}"; + long predictedWindowRows = await _predictedStore.CountPredictedHybridCandidatesInSizeWindowAsync(predictionMin, predictionMax, ct); + long predictedPoolCount = await _predictedStore.CountBetterThanLinearCandidatesAsync(predictedHigherDamageSmaller, predictedLowerDamageLarger, predictionMin, predictionMax, ct); + long deterministicWindowRows = await CountPredictedRowsInIntersectedSizeWindowAsync(predictionMin, predictionMax, realMin, realMax, ct); + long phaseSizeEligiblePool = await _predictedStore.CountBetterThanLinearCandidatesAsync( + predictedHigherDamageSmaller, + predictedLowerDamageLarger, + predictionMin, + predictionMax, + realMin, + realMax, + ct); + long rejectedByRealSizeWindow = Math.Max(0, predictedPoolCount - phaseSizeEligiblePool); + bool diversityEligible = ShouldUseDiversityForWindow(pair.HigherDamageSmaller, pair.LowerDamageLarger, predictedHigherDamageSmaller, predictedLowerDamageLarger); + interiorFetchLimit = ResolveValidationScanLimit(interiorAttemptLimit, phaseSizeEligiblePool, diversityEligible); + + var rawCandidates = (await _predictedStore.QueryBetterThanLinearCandidatesAsync( + pair.HigherDamageSmaller, + pair.LowerDamageLarger, + predictedHigherDamageSmaller, + predictedLowerDamageLarger, + predictionMin, + predictionMax, + realMin, + realMax, + HybridSelectionReason.InteriorSubspaceDiscovery, + windowLabel, + interiorFetchLimit, + ct)).ToList(); + + var brutalityAnalyses = rawCandidates + .Select((x, rawIndex) => new { Candidate = x, Brutality = AnalyzeNearLowerAnchorBrutality(x), RawRank = rawIndex + 1 }) + .ToList(); + + int afterBrutalityCount = brutalityAnalyses.Count(y => y.Brutality.Passed); + long deterministicEligibleCount = afterBrutalityCount; + long rejectedByOtherPhaseDeterministicRules = Math.Max(0, rawCandidates.Count - afterBrutalityCount); + + var rankedCandidates = brutalityAnalyses + .Where(x => x.Brutality.Passed) + .Select(x => AttachSelectionDiagnostics( + x.Candidate, + poolSize: phaseSizeEligiblePool, + windowCandidateCount: deterministicWindowRows, + lineBeatingCandidateCount: phaseSizeEligiblePool, + fetchedCandidateCount: rawCandidates.Count, + candidatesAfterBrutalityCount: afterBrutalityCount, + candidateAttemptLimit: interiorAttemptLimit, + phaseWindowIndex: globalWindowIndex, + phaseWindowCount: estimatedWindowCount, + notes: [x.Brutality.Explanation], + rawSelectionRank: x.RawRank)) + .ToList(); + + var selection = SelectValidationCandidates( + rankedCandidates, + interiorAttemptLimit, + pair.HigherDamageSmaller, + pair.LowerDamageLarger, + predictedHigherDamageSmaller, + predictedLowerDamageLarger, + "InteriorSubspaceDiscovery", + diversityEligible); + var kept = selection.Candidates.ToList(); + + allCandidates.AddRange(kept); + + var rejectedByBrutality = brutalityAnalyses + .Where(x => !x.Brutality.Passed) + .Take(DiagnosticPreviewDisplayCount) + .Select(x => ToCandidatePreviewLog(x.Candidate, x.Brutality)) + .ToList(); + + phaseDiagnostics.Add(new SelectionPhaseDiagnostic + { + Phase = "InteriorSubspaceDiscovery", + WindowLabel = windowLabel, + PhaseWindowIndex = globalWindowIndex, + PhaseWindowCount = estimatedWindowCount, + HigherDamageSmaller = ToAnchorLog(pair.HigherDamageSmaller), + LowerDamageLarger = ToAnchorLog(pair.LowerDamageLarger), + PredictionHigherDamageSmaller = ToPredictionAnchorLog(predictedHigherDamageSmaller), + PredictionLowerDamageLarger = ToPredictionAnchorLog(predictedLowerDamageLarger), + PredictionWindowMinSizeBytes = predictionMin, + PredictionWindowMaxSizeBytes = predictionMax, + WindowMinSizeBytes = realMin, + WindowMaxSizeBytes = realMax, + WindowSizeGiB = ToGiB(realMax > realMin ? realMax - realMin : 0), + CandidatePoolSize = phaseSizeEligiblePool, + PredictedPoolCount = predictedPoolCount, + DeterministicEligibleCount = deterministicEligibleCount, + RejectedByRealSizeWindow = rejectedByRealSizeWindow, + RejectedByOtherPhaseDeterministicRules = rejectedByOtherPhaseDeterministicRules, + WindowCandidateCount = deterministicWindowRows, + LineBeatingCandidateCount = phaseSizeEligiblePool, + FetchedCandidateCount = rawCandidates.Count, + CandidatesAfterBrutalityCount = afterBrutalityCount, + SelectedForValidationCount = kept.Count, + CandidateAttemptLimit = interiorAttemptLimit, + QueryFetchLimit = interiorFetchLimit, + DiversityEnabled = selection.DiversityEnabled, + DiversityMode = selection.Mode, + DiversityScanLimit = interiorFetchLimit, + DiversityScanFetched = rawCandidates.Count, + CandidateFamilyCount = selection.CandidateFamilyCount, + SelectedFamilyCount = selection.SelectedFamilyCount, + SelectedFamilyKeys = selection.SelectedFamilyKeys, + DiversitySelectionStrategy = selection.SelectionStrategy, + DiversityOverflowCount = selection.OverflowCount, + DiversitySizeFloorStartBytes = selection.SizeFloorStartBytes, + DiversitySizeFloorEndBytes = selection.SizeFloorEndBytes, + TopCandidates = kept.Take(DiagnosticPreviewDisplayCount).Select(ToCandidatePreviewLog).ToList(), + RejectedByBrutalityPreview = rejectedByBrutality, + Notes = new[] + { + "Interior DuckDB discovery uses the predicted virtual nonlinear KLD line/window, then deterministically caps candidate size to the real interior slice before diversity/ladder selection. Real KLD is still used only after benchmark validation.", + $"predictedPoolCount={predictedPoolCount:N0}; phaseSizeEligiblePool={phaseSizeEligiblePool:N0}; deterministicEligibleCount={deterministicEligibleCount:N0}; rejectedByRealSizeWindow={rejectedByRealSizeWindow:N0}; rejectedByOtherPhaseDeterministicRules={rejectedByOtherPhaseDeterministicRules:N0}." + }.Concat(selection.Notes).ToList() + }); + + AnsiConsole.MarkupLine( + $"[grey]Interior window {globalWindowIndex:N0}/{Math.Max(estimatedWindowCount, globalWindowIndex):N0}:[/] {Markup.Escape(pair.HigherDamageSmaller.DisplayName)} -> {Markup.Escape(pair.LowerDamageLarger.DisplayName)} " + + $"| pred-window={predictionMin:N0}..{predictionMax:N0}, real-window={realMin:N0}..{realMax:N0}, predictedPoolCount={predictedPoolCount:N0}, phaseSizeEligiblePool={phaseSizeEligiblePool:N0}, deterministicEligibleCount={deterministicEligibleCount:N0}, rejectedByRealSizeWindow={rejectedByRealSizeWindow:N0}, rejectedByOtherPhaseDeterministicRules={rejectedByOtherPhaseDeterministicRules:N0}, rows-in-window={deterministicWindowRows:N0}, beat-line={phaseSizeEligiblePool:N0}, scanLimit={interiorFetchLimit:N0}, scanFetched={rawCandidates.Count:N0}, after-brutality={afterBrutalityCount:N0}, diversity={Markup.Escape(selection.Mode)}, selectionStrategy={Markup.Escape(selection.SelectionStrategy)}, candidateFamilies={selection.CandidateFamilyCount:N0}, selectedFamilies={selection.SelectedFamilyCount:N0}, selected={kept.Count:N0}/{interiorAttemptLimit:N0}, overflowCount={selection.OverflowCount:N0}, sizeFloorStart={selection.SizeFloorStartBytes?.ToString("N0") ?? "n/a"}"); + PrintSelectedCandidateFamilySummary(kept); + PrintSelectionLadderNotes(selection.Notes); + + if (deterministicEligibleCount == 0) + AnsiConsole.MarkupLine($"[yellow]Interior window skipped builds:[/] no physically eligible candidates remained after predicted nonlinear line, real size slice, and deterministic brutality filters."); + + realCursor = realMax; + predictionCursor = predictionMax; + + if (realCursor >= realHighSize || predictionCursor >= predictionHighSize) + break; + } + } + + var deduped = allCandidates + .GroupBy(x => TensorConfigIdentity.ToKey(x.Prediction.Config), StringComparer.Ordinal) + .Select(g => g.OrderByDescending(x => x.PredictedGainOverLine).ThenBy(x => x.Prediction.PredictedSizeBytes).First()) + .OrderByDescending(x => x.PredictedGainOverLine) + .ThenBy(x => x.Prediction.PredictedSizeBytes) + .ToList(); + + int duplicateCount = Math.Max(0, allCandidates.Count - deduped.Count); + AnsiConsole.MarkupLine($"[grey]Interior candidate rollup:[/] raw-after-brutality={allCandidates.Count:N0}, duplicate-configs-removed={duplicateCount:N0}, selected-for-batch={deduped.Count:N0}"); + + if (deduped.Count == 0) + { + AnsiConsole.MarkupLine("[grey]No predicted interior candidates beat their local linear KLD lines after window/brutality filtering.[/]"); + var smartOnly = await RunSmartInteriorFallbackAsync(pairs, fractions, validationFailures, validationAttempts, ct); + return new PhaseValidationResult { AcceptedSnapshots = smartOnly }; + } + + AnsiConsole.MarkupLine($"[grey]Interior candidates selected for batch validation:[/] [cyan]{deduped.Count:N0}[/]"); + PrintCandidatePreviewTable(deduped, "Interior selected candidates preview"); + + var quantBatch = deduped.Select(x => x.Prediction.Quant).DistinctBy(x => TensorConfigIdentity.ToKey((TensorConfig)x)).ToList(); + var summary = await _quantizationService.ProcessHybridBatchAsync( + quantBatch, + new StageProgressOptions + { + StageName = "Interior candidate validation batch", + Total = quantBatch.Count, + MinimumNonSkippedSamplesBeforeEta = 2, + ShowEta = true, + CountSkippedForEta = false + }, + ct); + AnsiConsole.MarkupLine($"[grey]Interior validation batch:[/] requested={summary.Requested:N0} completed={summary.Completed:N0} skipped={summary.Skipped:N0} failed={summary.Failed:N0}"); + + foreach (var candidate in deduped) + { + var snapshot = await _repository.LoadBenchmarkSnapshotAsync(candidate.Prediction.Config, ct); + bool acceptedCandidate = snapshot != null && + snapshot.SizeBytes >= candidate.WindowMinSizeBytes && + snapshot.SizeBytes <= candidate.WindowMaxSizeBytes && + BeatsLinearKldLine(snapshot.SizeBytes, snapshot.Kld, candidate.HigherDamageAnchor, candidate.LowerDamageAnchor); + + var validation = new CandidateValidationResult + { + Candidate = candidate, + Snapshot = snapshot, + Accepted = acceptedCandidate, + FailureCode = acceptedCandidate ? string.Empty : snapshot == null ? "SNAPSHOT_MISSING_AFTER_BATCH" : "REAL_BENCHMARK_DID_NOT_BEAT_LINE", + Message = acceptedCandidate + ? "validated interior candidate" + : snapshot == null + ? $"benchmark snapshot was not found after batch build (requested={summary.Requested}, completed={summary.Completed}, skipped={summary.Skipped}, failed={summary.Failed})" + : BuildDetailedFailureMessage(candidate, snapshot, "real benchmark did not beat the local linear KLD line inside the requested size window") + }; + validationAttempts.Add(validation); + + if (acceptedCandidate && snapshot != null) + { + accepted.Add(snapshot); + PrintCandidateValidationOutcome(candidate, snapshot, accepted: true, "validated interior candidate"); + continue; + } + + if (snapshot != null) + PrintCandidateValidationOutcome(candidate, snapshot, accepted: false, validation.Message); + else + AnsiConsole.MarkupLine($"[yellow]Rejected predicted candidate:[/] {Markup.Escape(validation.Message)}"); + + validationFailures.Add(validation); + } + + if (accepted.Count == 0) + { + var smartAccepted = await RunSmartInteriorFallbackAsync(pairs, fractions, validationFailures, validationAttempts, ct); + accepted.AddRange(smartAccepted); + } + + return new PhaseValidationResult { AcceptedSnapshots = accepted }; + } + + private async Task> RunSmartInteriorFallbackAsync( + IReadOnlyList pairs, + IReadOnlyList fractions, + List validationFailures, + List validationAttempts, + CancellationToken ct) + { + if (!Config.SelectionSmartFallbackEnabled) + return Array.Empty(); + + var accepted = new List(); + int estimatedWindowCount = pairs.Sum(pair => EstimateInteriorWindowCount(pair, fractions)); + int globalWindowIndex = 0; + + foreach (var pair in pairs) + { + ulong realLowSize = pair.HigherDamageSmaller.SizeBytes; + ulong realHighSize = pair.LowerDamageLarger.SizeBytes; + if (realHighSize <= realLowSize) + continue; + + ulong realSpan = realHighSize - realLowSize; + ulong realCursor = realLowSize; + + for (int i = 0; i < fractions.Count; i++) + { + double fraction = fractions[i]; + if (fraction <= 0d) + continue; + + ulong realWidth = (ulong)Math.Max(1d, Math.Round(realSpan * Math.Clamp(fraction, 0d, 1d))); + ulong realMin = realCursor; + ulong realMax = i == fractions.Count - 1 + ? realHighSize + : Math.Min(realHighSize, realCursor + realWidth); + + if (realMax <= realMin) + continue; + + globalWindowIndex++; + string windowLabel = $"smart interior {globalWindowIndex:N0}: {pair.HigherDamageSmaller.DisplayName} -> {pair.LowerDamageLarger.DisplayName}"; + + var smartCandidates = (await _smartFallbackService.BuildInteriorCandidatesAsync( + pair.HigherDamageSmaller, + pair.LowerDamageLarger, + realMin, + realMax, + windowLabel, + globalWindowIndex, + Math.Max(estimatedWindowCount, globalWindowIndex), + ct)).ToList(); + + foreach (var candidate in smartCandidates) + { + var validation = await BuildAndValidateSingleAsync( + candidate, + snapshot => snapshot.SizeBytes >= realMin && + snapshot.SizeBytes <= realMax && + BeatsLinearKldLine(snapshot.SizeBytes, snapshot.Kld, pair.HigherDamageSmaller, pair.LowerDamageLarger), + $"smart fallback must land inside {realMin:N0}..{realMax:N0} bytes and beat the real interior linear KLD line", + ct); + + validationAttempts.Add(validation); + + if (validation.Accepted && validation.Snapshot != null) + { + accepted.Add(validation.Snapshot); + AnsiConsole.MarkupLine("[green]Smart interior fallback candidate selected:[/]"); + AnsiConsole.MarkupLine($"[grey] lower anchor=[/] [cyan]{Markup.Escape(pair.HigherDamageSmaller.DisplayName)}[/]"); + AnsiConsole.MarkupLine($"[grey] upper anchor=[/] [cyan]{Markup.Escape(pair.LowerDamageLarger.DisplayName)}[/]"); + AnsiConsole.MarkupLine($"[grey] chosen=[/] [cyan]{Markup.Escape(validation.Snapshot.DisplayName)}[/]"); + AnsiConsole.MarkupLine($"[grey] actualKld=[/] [cyan]{validation.Snapshot.Kld:0.000000}[/]"); + AnsiConsole.MarkupLine($"[grey] actualSizeBytes=[/] [cyan]{validation.Snapshot.SizeBytes:N0}[/]"); + return accepted; + } + + validationFailures.Add(validation); + } + + realCursor = realMax; + if (realCursor >= realHighSize) + break; + } + } + + AnsiConsole.MarkupLine("[grey]Smart interior fallback found no validated candidates.[/]"); + return accepted; + } + + private async Task BuildAndValidateSingleAsync( + HybridSelectionCandidate candidate, + Func accept, + string expectation, + CancellationToken ct) + { + AnsiConsole.MarkupLine( + $"[grey]Validating candidate:[/] {Markup.Escape(HybridBenchmarkRepository.BuildDisplayName(candidate.Prediction.Quant))} " + + $"[grey]| reason=[/] {candidate.Reason} [grey]| attempt=[/] {candidate.AttemptOrder:N0}/{Math.Max(candidate.CandidateAttemptLimit, candidate.AttemptOrder):N0} " + + $"[grey]| window=[/] {Markup.Escape(candidate.WindowLabel)}"); + PrintCandidatePredictionLine(candidate); + + var summary = await _quantizationService.ProcessHybridBatchAsync(new[] { candidate.Prediction.Quant }, ct); + var snapshot = await _repository.LoadBenchmarkSnapshotAsync(candidate.Prediction.Config, ct); + + bool accepted = snapshot != null && accept(snapshot); + string message = accepted + ? "validated" + : snapshot == null + ? $"no benchmark snapshot was available after build attempt (completed={summary.Completed}, skipped={summary.Skipped}, failed={summary.Failed})" + : BuildDetailedFailureMessage(candidate, snapshot, $"failed expectation: {expectation}"); + + if (accepted && snapshot != null) + { + PrintCandidateValidationOutcome(candidate, snapshot, accepted: true, "validated"); + } + else if (snapshot != null) + { + PrintCandidateValidationOutcome(candidate, snapshot, accepted: false, message); + } + else + { + AnsiConsole.MarkupLine($"[yellow]Rejected predicted candidate:[/] {Markup.Escape(message)}"); + } + + return new CandidateValidationResult + { + Candidate = candidate, + Snapshot = snapshot, + Accepted = accepted, + FailureCode = accepted ? string.Empty : snapshot == null ? "SNAPSHOT_MISSING_AFTER_BUILD" : "REAL_BENCHMARK_FAILED_EXPECTATION", + Message = message + }; + } + + private static HybridSelectionCandidate AttachSelectionDiagnostics( + HybridSelectionCandidate candidate, + long poolSize, + long windowCandidateCount, + long lineBeatingCandidateCount, + int fetchedCandidateCount, + int candidatesAfterBrutalityCount, + int candidateAttemptLimit, + int phaseWindowIndex, + int phaseWindowCount, + IReadOnlyList notes, + int? rawSelectionRank = null) + { + return CloneCandidateWithSelectionMetadata( + candidate, + attemptOrder: candidate.AttemptOrder, + rawSelectionRank: rawSelectionRank ?? candidate.RawSelectionRank, + familyKey: candidate.CandidateTheoryFamilyKey, + familyDisplay: candidate.CandidateTheoryFamilyDisplay, + familyRank: candidate.CandidateTheoryFamilyRank, + familyMemberRank: candidate.CandidateTheoryFamilyMemberRank, + diversityMode: candidate.DiversityMode, + notes: notes, + poolSize: poolSize, + windowCandidateCount: windowCandidateCount, + lineBeatingCandidateCount: lineBeatingCandidateCount, + fetchedCandidateCount: fetchedCandidateCount, + candidatesAfterBrutalityCount: candidatesAfterBrutalityCount, + candidateAttemptLimit: candidateAttemptLimit, + phaseWindowIndex: phaseWindowIndex, + phaseWindowCount: phaseWindowCount); + } + + private static int ResolveValidationScanLimit(int attemptLimit, long candidatePoolSize, bool diversityEligible) + { + attemptLimit = Math.Max(1, attemptLimit); + + if (!Config.SelectionDiversifyValidationCandidates || !diversityEligible) + return attemptLimit; + + if (candidatePoolSize > 0 && candidatePoolSize <= attemptLimit) + return attemptLimit; + + long requested = (long)attemptLimit * Config.SelectionDiversityScanMultiplier; + int min = Math.Max(attemptLimit, Config.SelectionDiversityScanMinCandidates); + int max = Math.Max(min, Config.SelectionDiversityScanMaxCandidates); + long clamped = Math.Clamp(requested, min, max); + + if (candidatePoolSize > 0 && candidatePoolSize < clamped) + clamped = candidatePoolSize; + + return checked((int)Math.Max(attemptLimit, clamped)); + } + + private static bool ShouldUseDiversityForWindow( + BenchmarkSnapshotRecord higherDamageSmaller, + BenchmarkSnapshotRecord lowerDamageLarger, + PredictedAnchorRow? higherDamagePredictionAnchor, + PredictedAnchorRow? lowerDamagePredictionAnchor) + { + if (!Config.SelectionDiversifyValidationCandidates) + return false; + + if (!Config.SelectionDiversityLowBitOnly) + return true; + + return IsQ4ishOrBelow(higherDamageSmaller.Quant.BaseQuant) || + IsQ4ishOrBelow(lowerDamageLarger.Quant.BaseQuant) || + IsQ4ishOrBelow(higherDamagePredictionAnchor?.RuntimeBaselineId) || + IsQ4ishOrBelow(lowerDamagePredictionAnchor?.RuntimeBaselineId); + } + + private static bool IsQ4ishOrBelow(byte? baselineId) + { + if (!baselineId.HasValue) + return false; + + try + { + return BaselineQuants.FromId(baselineId.Value).BitRange <= 4; + } + catch + { + return false; + } + } + + private static bool IsQ4ishOrBelow(BaselineQuants baseline) => baseline.BitRange <= 4; + + private static ValidationCandidateSelectionResult SelectValidationCandidates( + IReadOnlyList rankedCandidates, + int attemptLimit, + BenchmarkSnapshotRecord higherDamageSmaller, + BenchmarkSnapshotRecord lowerDamageLarger, + PredictedAnchorRow? higherDamagePredictionAnchor, + PredictedAnchorRow? lowerDamagePredictionAnchor, + string phaseName, + bool diversityEligible) + { + attemptLimit = Math.Max(1, attemptLimit); + if (rankedCandidates.Count == 0) + { + return new ValidationCandidateSelectionResult + { + Candidates = Array.Empty(), + Mode = Config.SelectionDiversifyValidationCandidates ? diversityEligible ? "enabled-empty" : "disabled-low-bit-only" : "disabled", + DiversityEnabled = Config.SelectionDiversifyValidationCandidates && diversityEligible, + SelectionStrategy = "none", + Notes = ["No deterministic-eligible candidates survived the prediction/brutality filters for this window; no phase-original primary candidate exists."] + }; + } + + var activeGroups = GetActiveTensorGroups(); + var entries = rankedCandidates + .Select((candidate, rawIndex) => + { + var signature = BuildCandidateTheorySignature(candidate, higherDamageSmaller, lowerDamageLarger, higherDamagePredictionAnchor, lowerDamagePredictionAnchor, activeGroups); + return new CandidateFamilyEntry + { + Candidate = candidate, + Signature = signature, + RawRank = candidate.RawSelectionRank > 0 ? candidate.RawSelectionRank : rawIndex + 1 + }; + }) + .ToList(); + + var families = BuildCandidateTheoryFamilies(entries); + var originalPrimary = entries[0]; + string originalPrimaryConfigKey = TensorConfigIdentity.ToKey(originalPrimary.Candidate.Prediction.Config); + + var originalOrderNotes = BuildOriginalPhaseOrderPreviewNotes(phaseName, rankedCandidates, previewLimit: 10); + PrintOriginalPhaseOrderPreview(phaseName, higherDamageSmaller, lowerDamageLarger, rankedCandidates, previewLimit: 10); + + bool canDiversify = Config.SelectionDiversifyValidationCandidates && diversityEligible && rankedCandidates.Count > attemptLimit; + if (!canDiversify || attemptLimit == 1) + { + string mode = Config.SelectionDiversifyValidationCandidates + ? diversityEligible + ? rankedCandidates.Count <= attemptLimit ? "not-needed" : "not-needed" + : "disabled-low-bit-only" + : "disabled"; + + var selectedWithoutDiversity = entries + .Take(attemptLimit) + .Select((entry, index) => DecorateSelectedCandidate( + entry, + families, + index + 1, + index == 0 ? "primary-original-phase-order" : mode)) + .ToList(); + + AssertPrimaryPreserved(phaseName, originalPrimaryConfigKey, selectedWithoutDiversity); + + var nonDiversityNotes = new List + { + BuildDiversityNote(mode, phaseName, rankedCandidates.Count, attemptLimit, families.Count, selectedWithoutDiversity.Count, overflowCount: 0), + $"primarySelectionSource=original-phase-order; primaryRawRankBeforeDiversity={originalPrimary.RawRank:N0}; primaryConfigKey={originalPrimaryConfigKey}; primaryPredictedSizeBytes={originalPrimary.Candidate.Prediction.PredictedSizeBytes:N0}; primaryEffectivePredictedKld={GetEffectivePredictedKld(originalPrimary.Candidate):0.000000}; primaryPredictionRank={FormatNullableRank(originalPrimary.Candidate.Prediction.PredictedRank)}; primaryWasExcluded=false.", + "fallbackSelectionStrategy=not-applied; diversity ladder did not run because the candidate count did not exceed the attempt limit, diversity was disabled, or only one attempt was allowed." + }; + nonDiversityNotes.AddRange(originalOrderNotes); + + return new ValidationCandidateSelectionResult + { + Candidates = selectedWithoutDiversity, + Mode = mode, + DiversityEnabled = false, + SelectionStrategy = "original-phase-order", + CandidateFamilyCount = families.Count, + SelectedFamilyCount = selectedWithoutDiversity.Select(x => x.CandidateTheoryFamilyKey).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct(StringComparer.Ordinal).Count(), + SelectedFamilyKeys = selectedWithoutDiversity.Select(x => x.CandidateTheoryFamilyKey).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct(StringComparer.Ordinal).ToList(), + Notes = nonDiversityNotes + }; + } + + var selectedEntries = new List(); + var selectedConfigKeys = new HashSet(StringComparer.Ordinal); + var selectedFamilyKeys = new HashSet(StringComparer.Ordinal); + var notesForFallback = new List(); + int overflowCount = 0; + int familyExhaustionCount = 0; + ulong? sizeFloorStart = null; + ulong sizeFloor = 0; + + bool AddEntry(CandidateFamilyEntry entry, string selectionMode, ulong? previousSizeFloor) + { + string configKey = TensorConfigIdentity.ToKey(entry.Candidate.Prediction.Config); + if (!selectedConfigKeys.Add(configKey)) + return false; + + entry.SelectionMode = selectionMode; + entry.PreviousSizeFloorBytes = previousSizeFloor; + entry.SizeDeltaVsFloorBytes = previousSizeFloor.HasValue + ? unchecked((long)entry.Candidate.Prediction.PredictedSizeBytes - (long)previousSizeFloor.Value) + : null; + entry.EffectivePredictedKld = GetEffectivePredictedKld(entry.Candidate); + + selectedEntries.Add(entry); + selectedFamilyKeys.Add(entry.Signature.Key); + return true; + } + + // Critical invariant: the phase's original deterministic-eligible primary owns attempt #1. + // Diversity and the size ladder are fallback ordering only and must never re-rank this row. + AddEntry(originalPrimary, "primary-original-phase-order", previousSizeFloor: null); + sizeFloor = originalPrimary.Candidate.Prediction.PredictedSizeBytes; + sizeFloorStart = sizeFloor; + + while (selectedEntries.Count < attemptLimit) + { + ulong previousFloor = sizeFloor; + + var unusedAtOrAbove = entries + .Skip(1) + .Where(x => IsSelectable(x, selectedConfigKeys) && !selectedFamilyKeys.Contains(x.Signature.Key) && x.Candidate.Prediction.PredictedSizeBytes >= sizeFloor) + .OrderBy(GetEffectivePredictedKld) + .ThenByDescending(x => x.Candidate.Prediction.PredictedSizeBytes) + .ThenBy(x => x.RawRank) + .ToList(); + + var next = unusedAtOrAbove.FirstOrDefault(); + if (next != null && AddEntry(next, "fallback-ladder-unused-family", previousFloor)) + { + sizeFloor = Math.Max(sizeFloor, next.Candidate.Prediction.PredictedSizeBytes); + continue; + } + + int unusedFamiliesRemaining = families.Count(f => !selectedFamilyKeys.Contains(f.Key) && f.Members.Any(m => IsSelectable(m, selectedConfigKeys))); + int unusedBelowCount = entries.Skip(1).Count(x => IsSelectable(x, selectedConfigKeys) && !selectedFamilyKeys.Contains(x.Signature.Key) && x.Candidate.Prediction.PredictedSizeBytes < sizeFloor); + + if (unusedFamiliesRemaining > 0) + { + overflowCount++; + notesForFallback.Add( + $"Diversity ladder overflow: attemptSlot={selectedEntries.Count + 1:N0}; sizeFloorBytes={sizeFloor:N0}; unusedFamiliesRemaining={unusedFamiliesRemaining:N0}; candidatesAtOrAboveFloor=0; overflowCandidatesBelowFloor={unusedBelowCount:N0}; reason=no-unused-family-candidate-at-or-above-size-floor."); + + next = entries + .Skip(1) + .Where(x => IsSelectable(x, selectedConfigKeys) && !selectedFamilyKeys.Contains(x.Signature.Key) && x.Candidate.Prediction.PredictedSizeBytes < sizeFloor) + .OrderByDescending(x => x.Candidate.Prediction.PredictedSizeBytes) + .ThenBy(GetEffectivePredictedKld) + .ThenBy(x => x.RawRank) + .FirstOrDefault(); + + if (next != null && AddEntry(next, "fallback-overflow-unused-family", previousFloor)) + { + // Deliberately monotone: an overflow candidate below the floor cannot drag the safety floor down. + sizeFloor = Math.Max(sizeFloor, next.Candidate.Prediction.PredictedSizeBytes); + continue; + } + } + + familyExhaustionCount++; + notesForFallback.Add( + $"Diversity ladder family exhaustion: attemptSlot={selectedEntries.Count + 1:N0}; selectedFamilies={selectedFamilyKeys.Count:N0}; candidateFamilies={families.Count:N0}; reason=all-distinct-families-exhausted."); + + next = entries + .Skip(1) + .Where(x => IsSelectable(x, selectedConfigKeys) && x.Candidate.Prediction.PredictedSizeBytes >= sizeFloor) + .OrderBy(GetEffectivePredictedKld) + .ThenByDescending(x => x.Candidate.Prediction.PredictedSizeBytes) + .ThenBy(x => x.RawRank) + .FirstOrDefault(); + + if (next != null && AddEntry(next, "fallback-ladder-used-family", previousFloor)) + { + sizeFloor = Math.Max(sizeFloor, next.Candidate.Prediction.PredictedSizeBytes); + continue; + } + + int belowFloorCount = entries.Skip(1).Count(x => IsSelectable(x, selectedConfigKeys) && x.Candidate.Prediction.PredictedSizeBytes < sizeFloor); + overflowCount++; + notesForFallback.Add( + $"Diversity ladder overflow: attemptSlot={selectedEntries.Count + 1:N0}; sizeFloorBytes={sizeFloor:N0}; unusedFamiliesRemaining=0; candidatesAtOrAboveFloor=0; overflowCandidatesBelowFloor={belowFloorCount:N0}; reason=no-remaining-candidate-at-or-above-size-floor."); + + next = entries + .Skip(1) + .Where(x => IsSelectable(x, selectedConfigKeys) && x.Candidate.Prediction.PredictedSizeBytes < sizeFloor) + .OrderByDescending(x => x.Candidate.Prediction.PredictedSizeBytes) + .ThenBy(GetEffectivePredictedKld) + .ThenBy(x => x.RawRank) + .FirstOrDefault(); + + if (next != null && AddEntry(next, "fallback-overflow-used-family", previousFloor)) + { + // Deliberately monotone: do not lower the floor after overflow. + sizeFloor = Math.Max(sizeFloor, next.Candidate.Prediction.PredictedSizeBytes); + continue; + } + + break; + } + + var selected = selectedEntries + .Take(attemptLimit) + .Select((entry, index) => DecorateSelectedCandidate(entry, families, index + 1, entry.SelectionMode)) + .ToList(); + + AssertPrimaryPreserved(phaseName, originalPrimaryConfigKey, selected); + + bool exhaustedDistinctFamilies = selected.Count > selected.Select(x => x.CandidateTheoryFamilyKey).Distinct(StringComparer.Ordinal).Count(); + var notes = new List + { + BuildDiversityNote("enabled", phaseName, rankedCandidates.Count, attemptLimit, families.Count, selected.Count, overflowCount), + $"primarySelectionSource=original-phase-order; primaryRawRankBeforeDiversity={originalPrimary.RawRank:N0}; primaryConfigKey={originalPrimaryConfigKey}; primaryPredictedSizeBytes={originalPrimary.Candidate.Prediction.PredictedSizeBytes:N0}; primaryEffectivePredictedKld={GetEffectivePredictedKld(originalPrimary.Candidate):0.000000}; primaryPredictionRank={FormatNullableRank(originalPrimary.Candidate.Prediction.PredictedRank)}; primaryWasExcluded=false.", + $"fallbackSelectionStrategy=family-size-ladder; fallbackCount={Math.Max(0, selected.Count - 1):N0}; sizeFloorStartBytes={sizeFloorStart.GetValueOrDefault():N0}; sizeFloorEndBytes={sizeFloor:N0}; overflowCount={overflowCount:N0}; familyExhaustionCount={familyExhaustionCount:N0}." + }; + notes.AddRange(originalOrderNotes); + notes.AddRange(notesForFallback); + + if (exhaustedDistinctFamilies) + notes.Add("Distinct candidate theory families were exhausted before the attempt limit; remaining fallback slots were filled from already-selected families using the same monotone size-floor ladder."); + if (overflowCount > 0) + notes.Add("Diversity ladder overflow occurred: at least one selected retry was below the monotone size floor because no same/larger alternative was available in the preferred family bucket."); + if (selected.Count > 0 && overflowCount >= Math.Max(1, selected.Count / 2)) + notes.Add("WARNING: Diversity ladder overflow selected many candidates below the size floor; candidate pool may not contain enough safer alternatives."); + if (families.Count >= rankedCandidates.Count * 0.90 && rankedCandidates.Count >= 25) + notes.Add("WARNING: Diversity warning: candidate family key may be too fine-grained; most scanned candidates formed unique families."); + + return new ValidationCandidateSelectionResult + { + Candidates = selected, + Mode = "enabled", + DiversityEnabled = true, + SelectionStrategy = "original-primary-plus-family-size-ladder-fallbacks", + CandidateFamilyCount = families.Count, + SelectedFamilyCount = selected.Select(x => x.CandidateTheoryFamilyKey).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct(StringComparer.Ordinal).Count(), + SelectedFamilyKeys = selected.Select(x => x.CandidateTheoryFamilyKey).Where(x => !string.IsNullOrWhiteSpace(x)).Distinct(StringComparer.Ordinal).ToList(), + OverflowCount = overflowCount, + SizeFloorStartBytes = sizeFloorStart, + SizeFloorEndBytes = sizeFloor, + Notes = notes + }; + } + + private static List BuildCandidateTheoryFamilies(IReadOnlyList entries) + { + var families = entries + .GroupBy(x => x.Signature.Key, StringComparer.Ordinal) + .Select(g => new CandidateTheoryFamily + { + Key = g.Key, + Display = g.First().Signature.Display, + Members = g.OrderBy(x => x.RawRank).ToList() + }) + .OrderBy(x => x.Members[0].RawRank) + .ToList(); + + for (int familyIndex = 0; familyIndex < families.Count; familyIndex++) + { + families[familyIndex].Rank = familyIndex + 1; + foreach (var member in families[familyIndex].Members) + member.Family = families[familyIndex]; + + for (int memberIndex = 0; memberIndex < families[familyIndex].Members.Count; memberIndex++) + families[familyIndex].Members[memberIndex].MemberRank = memberIndex + 1; + } + + return families; + } + + private static void AssertPrimaryPreserved( + string phaseName, + string originalPrimaryConfigKey, + IReadOnlyList selected) + { + if (selected.Count == 0) + return; + + string selectedPrimaryConfigKey = TensorConfigIdentity.ToKey(selected[0].Prediction.Config); + if (string.Equals(originalPrimaryConfigKey, selectedPrimaryConfigKey, StringComparison.Ordinal)) + return; + + string message = + $"CRITICAL SELECTION INVARIANT FAILED: {phaseName} fallback ordering changed attempt #1. " + + $"originalPrimary={originalPrimaryConfigKey}; selectedPrimary={selectedPrimaryConfigKey}. " + + "Family diversity / size ladder may only reorder fallback attempts after the phase-original primary candidate."; + + AnsiConsole.MarkupLine($"[red]{Markup.Escape(message)}[/]"); + throw new InvalidOperationException(message); + } + + private static IReadOnlyList BuildOriginalPhaseOrderPreviewNotes( + string phaseName, + IReadOnlyList rankedCandidates, + int previewLimit) + { + var notes = new List + { + $"originalOrderPreview phase={phaseName}; showingTop={Math.Min(previewLimit, rankedCandidates.Count):N0}/{rankedCandidates.Count:N0}; these rows are in deterministic-eligible original phase order before fallback diversity." + }; + + int order = 0; + foreach (var candidate in rankedCandidates.Take(previewLimit)) + { + order++; + notes.Add( + $"originalOrderPreview rawOrder={order:N0}; config={TensorConfigIdentity.ToKey(candidate.Prediction.Config)}; size={candidate.Prediction.PredictedSizeBytes:N0}; predKld={GetEffectivePredictedKld(candidate):0.000000}; predictionRank={FormatNullableRank(candidate.Prediction.PredictedRank)}."); + } + + return notes; + } + + private static void PrintOriginalPhaseOrderPreview( + string phaseName, + BenchmarkSnapshotRecord higherDamageSmaller, + BenchmarkSnapshotRecord lowerDamageLarger, + IReadOnlyList rankedCandidates, + int previewLimit) + { + int count = Math.Min(previewLimit, rankedCandidates.Count); + if (count == 0) + return; + + string anchorText = ReferenceEquals(higherDamageSmaller, lowerDamageLarger) || + TensorConfigIdentity.ToKey(higherDamageSmaller.Config) == TensorConfigIdentity.ToKey(lowerDamageLarger.Config) + ? higherDamageSmaller.DisplayName + : $"{higherDamageSmaller.DisplayName} -> {lowerDamageLarger.DisplayName}"; + + AnsiConsole.MarkupLine($"[grey]Original phase order preview ({Markup.Escape(phaseName)}) for {Markup.Escape(anchorText)}:[/] [cyan]{count:N0}/{rankedCandidates.Count:N0}[/]"); + int order = 0; + foreach (var candidate in rankedCandidates.Take(previewLimit)) + { + order++; + AnsiConsole.MarkupLine( + $"[grey] rawOrder={order:N0} config={Markup.Escape(TensorConfigIdentity.ToKey(candidate.Prediction.Config))} " + + $"size={candidate.Prediction.PredictedSizeBytes:N0} predKld={GetEffectivePredictedKld(candidate):0.000000} predictionRank={Markup.Escape(FormatNullableRank(candidate.Prediction.PredictedRank))}[/]"); + } + } + + private static string FormatNullableRank(ulong? rank) => rank.HasValue ? rank.Value.ToString("N0") : "n/a"; + + private static bool IsSelectable(CandidateFamilyEntry entry, HashSet selectedConfigKeys) + { + string key = TensorConfigIdentity.ToKey(entry.Candidate.Prediction.Config); + return !selectedConfigKeys.Contains(key); + } + + private static HybridSelectionCandidate DecorateSelectedCandidate( + CandidateFamilyEntry entry, + IReadOnlyList families, + int attemptOrder, + string diversityMode) + { + var family = entry.Family ?? families.First(x => string.Equals(x.Key, entry.Signature.Key, StringComparison.Ordinal)); + int memberRank = entry.MemberRank > 0 ? entry.MemberRank : Math.Max(1, family.Members.FindIndex(x => ReferenceEquals(x, entry)) + 1); + double effectivePredictedKld = double.IsNaN(entry.EffectivePredictedKld) ? GetEffectivePredictedKld(entry.Candidate) : entry.EffectivePredictedKld; + string previousFloorText = entry.PreviousSizeFloorBytes.HasValue ? entry.PreviousSizeFloorBytes.Value.ToString("N0") : "n/a"; + string deltaText = entry.SizeDeltaVsFloorBytes.HasValue ? entry.SizeDeltaVsFloorBytes.Value.ToString("N0") : "n/a"; + + string selectionStrategy = diversityMode switch + { + "primary-original-phase-order" => "original-phase-primary", + "disabled" or "not-needed" or "disabled-low-bit-only" => "original-phase-order", + _ => "family-size-ladder-fallback" + }; + + var notes = entry.Candidate.CandidateSelectionNotes + .Concat(new[] + { + $"diversity={diversityMode}; selectionStrategy={selectionStrategy}; rawRank={entry.RawRank}; familyRank={family.Rank}; familyMemberRank={memberRank}; previousSizeFloorBytes={previousFloorText}; selectedSizeBytes={entry.Candidate.Prediction.PredictedSizeBytes:N0}; sizeDeltaVsFloorBytes={deltaText}; effectivePredictedKld={effectivePredictedKld:0.000000}; familyKey={entry.Signature.Key}; familyDisplay={entry.Signature.Display}" + }) + .ToList(); + + return CloneCandidateWithSelectionMetadata( + entry.Candidate, + attemptOrder, + entry.RawRank, + entry.Signature.Key, + entry.Signature.Display, + family.Rank, + memberRank, + diversityMode, + notes); + } + + private static HybridSelectionCandidate CloneCandidateWithSelectionMetadata( + HybridSelectionCandidate candidate, + int attemptOrder, + int rawSelectionRank, + string familyKey, + string familyDisplay, + int familyRank, + int familyMemberRank, + string diversityMode, + IReadOnlyList notes, + long? poolSize = null, + long? windowCandidateCount = null, + long? lineBeatingCandidateCount = null, + int? fetchedCandidateCount = null, + int? candidatesAfterBrutalityCount = null, + int? candidateAttemptLimit = null, + int? phaseWindowIndex = null, + int? phaseWindowCount = null) + { + return new HybridSelectionCandidate + { + Prediction = candidate.Prediction, + Reason = candidate.Reason, + LowerDamageAnchor = candidate.LowerDamageAnchor, + HigherDamageAnchor = candidate.HigherDamageAnchor, + LowerDamagePredictionAnchor = candidate.LowerDamagePredictionAnchor, + HigherDamagePredictionAnchor = candidate.HigherDamagePredictionAnchor, + PredictionWindowMinSizeBytes = candidate.PredictionWindowMinSizeBytes, + PredictionWindowMaxSizeBytes = candidate.PredictionWindowMaxSizeBytes, + WindowMinSizeBytes = candidate.WindowMinSizeBytes, + WindowMaxSizeBytes = candidate.WindowMaxSizeBytes, + LinearExpectedKld = candidate.LinearExpectedKld, + PredictedGainOverLine = candidate.PredictedGainOverLine, + AttemptOrder = attemptOrder, + WindowLabel = candidate.WindowLabel, + CandidatePoolSize = poolSize ?? candidate.CandidatePoolSize, + WindowCandidateCount = windowCandidateCount ?? candidate.WindowCandidateCount, + LineBeatingCandidateCount = lineBeatingCandidateCount ?? candidate.LineBeatingCandidateCount, + FetchedCandidateCount = fetchedCandidateCount ?? candidate.FetchedCandidateCount, + CandidatesAfterBrutalityCount = candidatesAfterBrutalityCount ?? candidate.CandidatesAfterBrutalityCount, + CandidateAttemptLimit = candidateAttemptLimit ?? candidate.CandidateAttemptLimit, + PhaseWindowIndex = phaseWindowIndex ?? candidate.PhaseWindowIndex, + PhaseWindowCount = phaseWindowCount ?? candidate.PhaseWindowCount, + RawSelectionRank = rawSelectionRank, + CandidateTheoryFamilyKey = familyKey, + CandidateTheoryFamilyDisplay = familyDisplay, + CandidateTheoryFamilyRank = familyRank, + CandidateTheoryFamilyMemberRank = familyMemberRank, + DiversityMode = diversityMode, + CandidateSelectionNotes = notes + }; + } + + private static CandidateTheorySignature BuildCandidateTheorySignature( + HybridSelectionCandidate candidate, + BenchmarkSnapshotRecord higherDamageSmaller, + BenchmarkSnapshotRecord lowerDamageLarger, + PredictedAnchorRow? higherDamagePredictionAnchor, + PredictedAnchorRow? lowerDamagePredictionAnchor, + IReadOnlyList activeGroups) + { + var config = candidate.Prediction.Config; + var baseQuant = BaselineQuants.FromId(config.BaseQuant); + int anchorBit = Math.Min(higherDamageSmaller.Quant.BaseQuant.BitRange, lowerDamageLarger.Quant.BaseQuant.BitRange); + + var lowRisk = new List(); + var protectedGroups = new List(); + var external = new List(); + var sensitivity = new List(); + int sixPlus = 0; + int five = 0; + int four = 0; + int threeOrLess = 0; + + foreach (var group in activeGroups.OrderBy(x => x.UniqueId)) + { + var effective = GetEffectiveGroupBaseline(config, group); + string placement = $"{group.ShortCode}={effective.Names[0]}"; + + if (effective.BitRange >= 6) + sixPlus++; + else if (effective.BitRange == 5) + five++; + else if (effective.BitRange == 4) + four++; + else + threeOrLess++; + + if (effective.BitRange <= anchorBit - 1 || effective.BitRange <= 3) + lowRisk.Add(placement); + + if ((anchorBit <= 5 && effective.BitRange >= 6) || effective.BitRange >= anchorBit + 1) + protectedGroups.Add(placement); + + if (effective.IsCustomBaseline || effective.IsExternalRepositoryBaseline) + external.Add(placement); + + if (IsHighSensitivityGroup(group) && effective.UniqueId != baseQuant.UniqueId) + sensitivity.Add(placement); + } + + string anchorBand = $"anchor={higherDamagePredictionAnchor?.DisplayName ?? higherDamageSmaller.DisplayName}->{lowerDamagePredictionAnchor?.DisplayName ?? lowerDamageLarger.DisplayName}@{anchorBit}b"; + string bulk = $"bulk:6p={sixPlus},5={five},4={four},3m={threeOrLess}"; + + var components = new List + { + anchorBand, + $"base={baseQuant.Names[0]}", + bulk + }; + + if (lowRisk.Count > 0) + components.Add("risk:" + string.Join(",", lowRisk.OrderBy(x => x, StringComparer.Ordinal))); + if (protectedGroups.Count > 0) + components.Add("protect:" + string.Join(",", protectedGroups.OrderBy(x => x, StringComparer.Ordinal))); + if (external.Count > 0) + components.Add("external:" + string.Join(",", external.OrderBy(x => x, StringComparer.Ordinal))); + if (sensitivity.Count > 0) + components.Add("sensitive:" + string.Join(",", sensitivity.OrderBy(x => x, StringComparer.Ordinal))); + + string key = string.Join("|", components); + string display = string.Join("|", components.Where(x => !x.StartsWith("anchor=", StringComparison.Ordinal))); + return new CandidateTheorySignature { Key = key, Display = display }; + } + + private static BaselineQuants GetEffectiveGroupBaseline(TensorConfig config, TensorGroup group) + { + byte stored = group.UniqueId switch + { + 0 => config.Embeddings, + 1 => config.LmHead, + 2 => config.AttnQ, + 3 => config.AttnKV, + 4 => config.AttnOutput, + 5 => config.FfnUpGate, + 6 => config.FfnDown, + 7 => config.MoeExperts, + 8 => config.MoeRouter, + _ => BaselineQuants.TensorConfigNullSlotValue + }; + + return BaselineQuants.IsNullTensorConfigGroupSlot(stored) + ? BaselineQuants.FromId(config.BaseQuant) + : BaselineQuants.DecodeTensorConfigGroupSlotToBaseline(stored); + } + + private static IReadOnlyList GetActiveTensorGroups() + { + var unusedIds = Cache.UnusedTensorGroups.Select(x => x.UniqueId).ToHashSet(); + return TReg.All.Where(x => !unusedIds.Contains(x.UniqueId)).OrderBy(x => x.UniqueId).ToList(); + } + + private static bool IsHighSensitivityGroup(TensorGroup group) => + group.UniqueId == TReg.Embeddings.UniqueId || + group.UniqueId == TReg.LmHead.UniqueId || + group.UniqueId == TReg.AttnQ.UniqueId || + group.UniqueId == TReg.AttnKV.UniqueId || + group.UniqueId == TReg.FfnDown.UniqueId; + + + private static double GetEffectivePredictedKld(CandidateFamilyEntry entry) => GetEffectivePredictedKld(entry.Candidate); + + private static double GetEffectivePredictedKld(HybridSelectionCandidate candidate) => + double.IsNaN(candidate.Prediction.PredictedKld) ? double.PositiveInfinity : candidate.Prediction.PredictedKld; + + private static string BuildDiversityNote(string mode, string phaseName, int candidateCount, int attemptLimit, int familyCount, int selectedCount, int overflowCount) => + mode switch + { + "enabled" => $"Diversity enabled for {phaseName}: selectionStrategy=family-size-ladder; selected {selectedCount:N0}/{attemptLimit:N0} validation attempts from {familyCount:N0} candidate theory families across {candidateCount:N0} filtered scan candidates; phase-original primary candidate is preserved as attempt 1; only fallback attempts prefer same/larger predicted size before explicit overflow; overflowCount={overflowCount:N0}.", + "not-needed" => $"Diversity not needed for {phaseName}: filtered candidate count {candidateCount:N0} <= attempt limit {attemptLimit:N0}; candidates kept in raw predicted order.", + "disabled-low-bit-only" => $"Diversity skipped for {phaseName}: candidate_selection.diversity_low_bit_only=true and this anchor/window was not Q4-ish or below.", + _ => $"Diversity disabled for {phaseName}; candidates kept in raw predicted order." + }; + + private static void PrintSelectionLadderNotes(IReadOnlyList notes) + { + foreach (var note in notes) + { + if (note.Contains("WARNING", StringComparison.OrdinalIgnoreCase) || + note.Contains("Diversity ladder overflow", StringComparison.OrdinalIgnoreCase) || + note.Contains("Diversity ladder family exhaustion", StringComparison.OrdinalIgnoreCase)) + { + AnsiConsole.MarkupLine($"[yellow] {Markup.Escape(note)}[/]"); + } + else if (note.Contains("selectionStrategy=family-size-ladder", StringComparison.OrdinalIgnoreCase)) + { + AnsiConsole.MarkupLine($"[grey] {Markup.Escape(note)}[/]"); + } + } + } + + private static void PrintSelectedCandidateFamilySummary(IReadOnlyList candidates) + { + foreach (var candidate in candidates.Take(DiagnosticPreviewDisplayCount)) + { + if (string.IsNullOrWhiteSpace(candidate.CandidateTheoryFamilyDisplay)) + continue; + + string ladderNote = candidate.CandidateSelectionNotes.FirstOrDefault(x => x.Contains("selectionStrategy=family-size-ladder", StringComparison.Ordinal)) ?? string.Empty; + string previousFloor = ExtractSelectionNoteValue(ladderNote, "previousSizeFloorBytes") ?? "n/a"; + string deltaVsFloor = ExtractSelectionNoteValue(ladderNote, "sizeDeltaVsFloorBytes") ?? "n/a"; + string effectiveKld = ExtractSelectionNoteValue(ladderNote, "effectivePredictedKld") ?? candidate.Prediction.PredictedKld.ToString("0.000000"); + + AnsiConsole.MarkupLine( + $"[grey] selected attempt={candidate.AttemptOrder:N0}/{candidate.CandidateAttemptLimit:N0} rawRank={candidate.RawSelectionRank:N0} " + + $"selectionMode={Markup.Escape(candidate.DiversityMode)} familyRank={candidate.CandidateTheoryFamilyRank:N0} memberRank={candidate.CandidateTheoryFamilyMemberRank:N0} " + + $"previousSizeFloorBytes={Markup.Escape(previousFloor)} selectedSizeBytes={candidate.Prediction.PredictedSizeBytes:N0} sizeDeltaVsFloorBytes={Markup.Escape(deltaVsFloor)} effectivePredictedKld={Markup.Escape(effectiveKld)}[/]"); + AnsiConsole.MarkupLine($"[grey] familyKey=[/][cyan]{Markup.Escape(candidate.CandidateTheoryFamilyKey)}[/]"); + AnsiConsole.MarkupLine($"[grey] familyDisplay=[/][cyan]{Markup.Escape(candidate.CandidateTheoryFamilyDisplay)}[/]"); + } + } + + private static string? ExtractSelectionNoteValue(string note, string key) + { + if (string.IsNullOrWhiteSpace(note)) + return null; + + string prefix = key + "="; + int start = note.IndexOf(prefix, StringComparison.Ordinal); + if (start < 0) + return null; + + start += prefix.Length; + int end = note.IndexOf(';', start); + return end < 0 ? note[start..].Trim() : note[start..end].Trim(); + } + + private static BrutalityAnalysis AnalyzeNearLowerAnchorBrutality(HybridSelectionCandidate candidate) + { + var higherDamagePredictionAnchor = candidate.HigherDamagePredictionAnchor; + var lowerDamagePredictionAnchor = candidate.LowerDamagePredictionAnchor; + if (higherDamagePredictionAnchor == null || lowerDamagePredictionAnchor == null) + { + return new BrutalityAnalysis + { + Passed = true, + FractionFromSmallAnchor = 1d, + RequiredGain = Config.SelectionMinimumKldImprovementEpsilon, + Explanation = "Brutality skipped because prediction anchor metadata is missing; no predicted-vs-real comparison was performed." + }; + } + + ulong span = lowerDamagePredictionAnchor.PredictedSizeBytes > higherDamagePredictionAnchor.PredictedSizeBytes + ? lowerDamagePredictionAnchor.PredictedSizeBytes - higherDamagePredictionAnchor.PredictedSizeBytes + : 0; + + if (span == 0) + { + return new BrutalityAnalysis + { + Passed = true, + FractionFromSmallAnchor = 1d, + RequiredGain = Config.SelectionMinimumKldImprovementEpsilon, + Explanation = "Brutality passed because prediction-anchor span is zero." + }; + } + + ulong distanceFromSmall = candidate.Prediction.PredictedSizeBytes > higherDamagePredictionAnchor.PredictedSizeBytes + ? candidate.Prediction.PredictedSizeBytes - higherDamagePredictionAnchor.PredictedSizeBytes + : 0; + + double fraction = distanceFromSmall / (double)span; + double requiredGain = Math.Max( + Config.SelectionMinimumKldImprovementEpsilon, + Math.Abs(higherDamagePredictionAnchor.PredictedKld - lowerDamagePredictionAnchor.PredictedKld) * + Config.SelectionNearAnchorRequiredKldGainFractionOfPairGap); + + bool passed = fraction > Config.SelectionNearLowerAnchorBrutalZoneFractionOfPairSpan || + candidate.PredictedGainOverLine >= requiredGain; + + string explanation = passed + ? fraction > Config.SelectionNearLowerAnchorBrutalZoneFractionOfPairSpan + ? $"Brutality passed in prediction space because candidate is outside brutal zone (fraction={fraction:0.###} > {Config.SelectionNearLowerAnchorBrutalZoneFractionOfPairSpan:0.###})." + : $"Brutality passed in prediction space because predicted gain {candidate.PredictedGainOverLine:0.########} >= required gain {requiredGain:0.########}." + : $"Brutality rejected in prediction space because candidate is inside brutal zone (fraction={fraction:0.###} <= {Config.SelectionNearLowerAnchorBrutalZoneFractionOfPairSpan:0.###}) and predicted gain {candidate.PredictedGainOverLine:0.########} < required gain {requiredGain:0.########}."; + + return new BrutalityAnalysis + { + Passed = passed, + FractionFromSmallAnchor = fraction, + RequiredGain = requiredGain, + Explanation = explanation + }; + } + + private static bool PassesNearLowerAnchorBrutality(HybridSelectionCandidate candidate) => + AnalyzeNearLowerAnchorBrutality(candidate).Passed; + + private List ApplyMeaningfulSpacing( + IReadOnlyList snapshots, + List eliminations) + { + if (snapshots.Count <= 2) + return snapshots.ToList(); + + var ordered = snapshots + .OrderBy(x => x.SizeBytes) + .ThenBy(x => x.Kld) + .ToList(); + + ulong minSize = ordered.Min(x => x.SizeBytes); + ulong maxSize = ordered.Max(x => x.SizeBytes); + ulong globalSpan = maxSize > minSize ? maxSize - minSize : 0; + + if (globalSpan == 0) + return _finalEliminator.Eliminate(ordered).Survivors.ToList(); + + ulong minGap = (ulong)Math.Round(globalSpan * Config.SelectionMinimumNeighborGapFractionOfGlobalSpan, MidpointRounding.AwayFromZero); + if (minGap == 0) + return _finalEliminator.Eliminate(ordered).Survivors.ToList(); + + var kept = new List(); + + foreach (var snap in ordered) + { + var tooClose = kept + .Where(x => Distance(x.SizeBytes, snap.SizeBytes) < minGap) + .OrderBy(x => Distance(x.SizeBytes, snap.SizeBytes)) + .FirstOrDefault(); + + if (tooClose == null) + { + kept.Add(snap); + continue; + } + + var winner = ChooseSpacingWinner(tooClose, snap); + var loser = ReferenceEquals(winner, tooClose) ? snap : tooClose; + + if (!ReferenceEquals(winner, tooClose)) + { + kept.Remove(tooClose); + kept.Add(winner); + } + + eliminations.Add(new BaselineEliminationRecord + { + Eliminated = loser, + Eliminator = winner, + Reason = $"meaningful spacing collapse; size gap below {minGap:N0} bytes" + }); + } + + return _finalEliminator.Eliminate(kept).Survivors.ToList(); + } + + private static BenchmarkSnapshotRecord ChooseSpacingWinner(BenchmarkSnapshotRecord left, BenchmarkSnapshotRecord right) + { + if (Dominates(left, right)) + return left; + + if (Dominates(right, left)) + return right; + + return left.Kld.CompareTo(right.Kld) switch + { + < 0 => left, + > 0 => right, + _ => left.SizeBytes <= right.SizeBytes ? left : right + }; + } + + private List MergeAndDominanceFilter( + IReadOnlyList current, + IReadOnlyList additions, + List eliminations, + string reason) + { + if (additions.Count == 0) + return current.ToList(); + + var merged = current + .Concat(additions) + .DistinctBy(x => TensorConfigIdentity.ToKey(x.Config)) + .ToList(); + + var result = _finalEliminator.Eliminate(merged); + + foreach (var eliminated in result.Eliminated) + { + var eliminator = result.Survivors.FirstOrDefault(x => Dominates(x, eliminated)); + if (eliminator == null) + continue; + + eliminations.Add(new BaselineEliminationRecord + { + Eliminated = eliminated, + Eliminator = eliminator, + Reason = reason + }); + } + + return result.Survivors.ToList(); + } + + private static bool ShouldSkipAnchorReplacement(BenchmarkSnapshotRecord anchor) + { + return !Config.SelectionAllowEightBitAnchorReplacements && + anchor.Quant.BaseQuant.BitRange >= 8 && + !anchor.Quant.BaseQuant.IsHighPrecisionExactAlias; + } + + private static List BuildAdjacentPairs(IReadOnlyList anchors) + { + var ordered = anchors + .OrderBy(x => x.Kld) + .ThenBy(x => x.SizeBytes) + .ToList(); + + var result = new List(); + + for (int i = 0; i < ordered.Count - 1; i++) + { + var lowerDamage = ordered[i]; + var higherDamage = ordered[i + 1]; + + if (higherDamage.SizeBytes >= lowerDamage.SizeBytes) + continue; + + result.Add(new AdjacentAnchorPair + { + LowerDamageLarger = lowerDamage, + HigherDamageSmaller = higherDamage + }); + } + + return result; + } + + private static int EstimateInteriorWindowCount(AdjacentAnchorPair pair, IReadOnlyList fractions) + { + if (pair.LowerDamageLarger.SizeBytes <= pair.HigherDamageSmaller.SizeBytes) + return 0; + + ulong span = pair.LowerDamageLarger.SizeBytes - pair.HigherDamageSmaller.SizeBytes; + ulong cursor = pair.HigherDamageSmaller.SizeBytes; + int count = 0; + + for (int i = 0; i < fractions.Count; i++) + { + double fraction = fractions[i]; + if (fraction <= 0d) + continue; + + ulong width = (ulong)Math.Round(span * fraction, MidpointRounding.AwayFromZero); + if (width == 0) + continue; + + ulong max = Math.Min(pair.LowerDamageLarger.SizeBytes, cursor + width); + if (max <= cursor) + continue; + + count++; + cursor = max; + if (cursor >= pair.LowerDamageLarger.SizeBytes) + break; + } + + return count; + } + + private static bool BeatsLinearKldLine( + ulong candidateSize, + double candidateKld, + BenchmarkSnapshotRecord higherDamageSmaller, + BenchmarkSnapshotRecord lowerDamageLarger) + { + double expected = InterpolateKldLine(candidateSize, higherDamageSmaller, lowerDamageLarger); + return candidateKld + Config.SelectionMinimumKldImprovementEpsilon < expected; + } + + private static double InterpolateKldLine( + ulong candidateSize, + BenchmarkSnapshotRecord higherDamageSmaller, + BenchmarkSnapshotRecord lowerDamageLarger) + { + ulong smallSize = higherDamageSmaller.SizeBytes; + ulong largeSize = lowerDamageLarger.SizeBytes; + + if (largeSize <= smallSize) + return Math.Min(higherDamageSmaller.Kld, lowerDamageLarger.Kld); + + double t = Math.Clamp((candidateSize - smallSize) / (double)(largeSize - smallSize), 0d, 1d); + return higherDamageSmaller.Kld + ((lowerDamageLarger.Kld - higherDamageSmaller.Kld) * t); + } + + private static ValidationMetrics ComputeValidationMetrics(HybridSelectionCandidate candidate, BenchmarkSnapshotRecord snapshot) + { + double line = InterpolateKldLine(snapshot.SizeBytes, candidate.HigherDamageAnchor, candidate.LowerDamageAnchor); + double gain = line - snapshot.Kld; + bool insideWindow = snapshot.SizeBytes >= candidate.WindowMinSizeBytes && snapshot.SizeBytes <= candidate.WindowMaxSizeBytes; + bool beatsLine = snapshot.Kld + Config.SelectionMinimumKldImprovementEpsilon < line; + long sizeMissBytes = 0; + + if (snapshot.SizeBytes < candidate.WindowMinSizeBytes) + sizeMissBytes = (long)candidate.WindowMinSizeBytes - (long)snapshot.SizeBytes; + else if (snapshot.SizeBytes > candidate.WindowMaxSizeBytes) + sizeMissBytes = (long)snapshot.SizeBytes - (long)candidate.WindowMaxSizeBytes; + + return new ValidationMetrics + { + ActualLineKld = line, + ActualGainOverLine = gain, + KldMiss = snapshot.Kld + Config.SelectionMinimumKldImprovementEpsilon - line, + SizeMissBytes = sizeMissBytes, + InsideWindow = insideWindow, + BeatsLine = beatsLine + }; + } + + private static string BuildDetailedFailureMessage(HybridSelectionCandidate candidate, BenchmarkSnapshotRecord snapshot, string prefix) + { + var metrics = ComputeValidationMetrics(candidate, snapshot); + string sizeText = metrics.SizeMissBytes == 0 ? "inside size window" : $"missed size window by {metrics.SizeMissBytes:N0} bytes"; + string kldText = metrics.KldMiss <= 0 ? "beat required KLD line" : $"missed KLD line by {metrics.KldMiss:0.000000}"; + + return $"{prefix}; actual size={snapshot.SizeBytes:N0} ({ToGiB(snapshot.SizeBytes):0.00} GiB), actual KLD={snapshot.Kld:0.000000}, line={metrics.ActualLineKld:0.000000}, gain={metrics.ActualGainOverLine:0.000000}, {sizeText}, {kldText}"; + } + + private static void PrintCandidatePredictionLine(HybridSelectionCandidate candidate) + { + AnsiConsole.MarkupLine( + $"[grey] predicted:[/] size={candidate.Prediction.PredictedSizeBytes:N0} bytes ({ToGiB(candidate.Prediction.PredictedSizeBytes):0.00} GiB), " + + $"kld={candidate.Prediction.PredictedKld:0.000000}, line={candidate.LinearExpectedKld:0.000000}, gain={candidate.PredictedGainOverLine:0.000000}, " + + $"rank={candidate.Prediction.PredictedRank}, confidence={candidate.Prediction.PredictionConfidence:0.###}"); + AnsiConsole.MarkupLine( + $"[grey] prediction anchors/window:[/] {Markup.Escape(candidate.HigherDamagePredictionAnchor?.DisplayName ?? "n/a")} -> {Markup.Escape(candidate.LowerDamagePredictionAnchor?.DisplayName ?? "n/a")}, " + + $"predWindow={candidate.PredictionWindowMinSizeBytes:N0}..{candidate.PredictionWindowMaxSizeBytes:N0}, realWindow={candidate.WindowMinSizeBytes:N0}..{candidate.WindowMaxSizeBytes:N0}"); + AnsiConsole.MarkupLine( + $"[grey] selection context:[/] pool={candidate.CandidatePoolSize:N0}, windowRows={candidate.WindowCandidateCount:N0}, lineBeat={candidate.LineBeatingCandidateCount:N0}, " + + $"fetched={candidate.FetchedCandidateCount:N0}, afterBrutality={candidate.CandidatesAfterBrutalityCount:N0}, attemptLimit={candidate.CandidateAttemptLimit:N0}"); + if (!string.IsNullOrWhiteSpace(candidate.CandidateTheoryFamilyDisplay)) + { + string ladderNote = candidate.CandidateSelectionNotes.FirstOrDefault(x => x.Contains("selectionStrategy=family-size-ladder", StringComparison.Ordinal)) ?? string.Empty; + string previousFloor = ExtractSelectionNoteValue(ladderNote, "previousSizeFloorBytes") ?? "n/a"; + string deltaVsFloor = ExtractSelectionNoteValue(ladderNote, "sizeDeltaVsFloorBytes") ?? "n/a"; + string effectiveKld = ExtractSelectionNoteValue(ladderNote, "effectivePredictedKld") ?? candidate.Prediction.PredictedKld.ToString("0.000000"); + AnsiConsole.MarkupLine( + $"[grey] diversity family:[/] selectionMode={Markup.Escape(candidate.DiversityMode)}, rawRank={candidate.RawSelectionRank:N0}, " + + $"familyRank={candidate.CandidateTheoryFamilyRank:N0}, memberRank={candidate.CandidateTheoryFamilyMemberRank:N0}, " + + $"previousSizeFloorBytes={Markup.Escape(previousFloor)}, selectedSizeBytes={candidate.Prediction.PredictedSizeBytes:N0}, sizeDeltaVsFloorBytes={Markup.Escape(deltaVsFloor)}, effectivePredictedKld={Markup.Escape(effectiveKld)}"); + AnsiConsole.MarkupLine($"[grey] diversity familyKey:[/] {Markup.Escape(candidate.CandidateTheoryFamilyKey)}"); + AnsiConsole.MarkupLine($"[grey] diversity familyDisplay:[/] {Markup.Escape(candidate.CandidateTheoryFamilyDisplay)}"); + } + AnsiConsole.MarkupLine($"[grey] bit space:[/] {Markup.Escape(DescribeBitSpace(candidate.Prediction.Config))}"); + } + + private static void PrintCandidateValidationOutcome(HybridSelectionCandidate candidate, BenchmarkSnapshotRecord snapshot, bool accepted, string message) + { + var metrics = ComputeValidationMetrics(candidate, snapshot); + string status = accepted ? "[green]Validated[/]" : "[yellow]Rejected predicted candidate[/]"; + AnsiConsole.MarkupLine( + $"{status}: {Markup.Escape(snapshot.DisplayName)} | actual size={snapshot.SizeBytes:N0} ({ToGiB(snapshot.SizeBytes):0.00} GiB), " + + $"actual KLD={snapshot.Kld:0.000000}, line={metrics.ActualLineKld:0.000000}, gain={metrics.ActualGainOverLine:0.000000}, " + + $"sizeMiss={metrics.SizeMissBytes:N0}, kldMiss={Math.Max(0d, metrics.KldMiss):0.000000}"); + + if (!accepted) + AnsiConsole.MarkupLine($"[yellow] reason:[/] {Markup.Escape(message)}"); + } + + private static void PrintAnchorFrontier(IReadOnlyList anchors, string title) + { + if (anchors.Count == 0) + return; + + var table = new Table().RoundedBorder().BorderColor(Color.Grey); + table.Title = new TableTitle(Markup.Escape(title)); + table.AddColumn("Order"); + table.AddColumn("Anchor"); + table.AddColumn("Provider"); + table.AddColumn("KLD"); + table.AddColumn("Size GiB"); + + int i = 0; + foreach (var anchor in anchors.OrderBy(x => x.Kld).ThenBy(x => x.SizeBytes)) + { + table.AddRow( + (++i).ToString("N0"), + Markup.Escape(anchor.DisplayName), + Markup.Escape(anchor.ProviderName), + anchor.Kld.ToString("0.000000"), + ToGiB(anchor.SizeBytes).ToString("0.00")); + } + + AnsiConsole.Write(table); + } + + private static void PrintPredictionAnchorFrontier( + IReadOnlyList predictedAnchors, + IReadOnlyList realAnchors, + string title) + { + if (predictedAnchors.Count == 0) + { + AnsiConsole.MarkupLine("[yellow]Prediction Anchor Frontier:[/] no scored virtual prediction anchors were found. Final DuckDB preselection will skip anchor-based discovery rather than mix prediction and real spaces."); + return; + } + + var table = new Table().RoundedBorder().BorderColor(Color.Grey); + table.Title = new TableTitle(Markup.Escape(title)); + table.AddColumn("Virtual Anchor"); + table.AddColumn("Canonical Key"); + table.AddColumn("Config Key"); + table.AddColumn("Pred KLD"); + table.AddColumn("Pred Size GiB"); + table.AddColumn("Rank"); + table.AddColumn("Conf"); + table.AddColumn("Isolation Source"); + table.AddColumn("Matching Real Anchor"); + table.AddColumn("Real KLD/Size GiB"); + + foreach (var anchor in predictedAnchors.OrderBy(x => x.PredictedKld).ThenBy(x => x.PredictedSizeBytes)) + { + var real = FindMatchingRealAnchor(anchor, realAnchors); + table.AddRow( + Markup.Escape(anchor.DisplayName), + Markup.Escape(anchor.BaselineCanonicalKey), + Markup.Escape(anchor.ConfigKey), + anchor.PredictedKld.ToString("0.000000"), + ToGiB(anchor.PredictedSizeBytes).ToString("0.00"), + anchor.PredictionRank.ToString("N0"), + anchor.PredictionConfidence.ToString("0.###"), + Markup.Escape(DescribeVirtualAnchorIsolationSource(anchor)), + real == null ? "[grey]none[/]" : Markup.Escape(real.DisplayName), + real == null ? "[grey]n/a[/]" : $"{real.Kld:0.000000} / {ToGiB(real.SizeBytes):0.00}"); + } + + AnsiConsole.Write(table); + + var q8Anchor = predictedAnchors.FirstOrDefault(x => + x.RuntimeBaselineId == BaselineQuants.Q8_0.UniqueId || + string.Equals(NormalizeAnchorKey(x.BaselineCanonicalKey), NormalizeAnchorKey(BaselineQuants.Q8_0.CanonicalKey), StringComparison.Ordinal) || + string.Equals(x.DisplayName, BaselineQuants.Q8_0.Names[0], StringComparison.OrdinalIgnoreCase)); + + if (q8Anchor != null && Math.Abs(q8Anchor.PredictedKld) <= 1e-12d) + { + AnsiConsole.MarkupLine("[yellow]WARNING:[/] Q8_0 virtual prediction anchor has zero predicted KLD. This usually means Q8_0 isolation rows were skipped or missing. Q8_0 must not be treated as native/exact truth in prediction space."); + } + } + + private static string DescribeVirtualAnchorIsolationSource(PredictedAnchorRow anchor) + { + try + { + var baseline = BaselineQuants.FromId(anchor.RuntimeBaselineId); + if (baseline.IsExternalRepositoryBaseline) + return "exact external; fallback disabled"; + + if (BaselineQuants.IsNativeExactAlias(baseline.UniqueId)) + return "native exact"; + + return "standard exact"; + } + catch + { + return "unknown"; + } + } + + private static BenchmarkSnapshotRecord? FindMatchingRealAnchor( + PredictedAnchorRow predictedAnchor, + IReadOnlyList realAnchors) + { + string predictedKey = NormalizeAnchorKey(predictedAnchor.BaselineCanonicalKey); + var byCanonical = realAnchors.FirstOrDefault(x => + string.Equals( + NormalizeAnchorKey(HybridBenchmarkRepository.ResolveSourceBaselineForProvider(x.Quant).CanonicalKey), + predictedKey, + StringComparison.Ordinal)); + + if (byCanonical != null) + return byCanonical; + + return realAnchors.FirstOrDefault(x => + HybridBenchmarkRepository.ResolveSourceBaselineForProvider(x.Quant).UniqueId == predictedAnchor.RuntimeBaselineId); + } + + private static void LogPredictionAndRealPairLines( + BenchmarkSnapshotRecord realHigherDamageSmaller, + BenchmarkSnapshotRecord realLowerDamageLarger, + PredictedAnchorRow predictedHigherDamageSmaller, + PredictedAnchorRow predictedLowerDamageLarger, + string label) + { + AnsiConsole.MarkupLine($"[grey]{Markup.Escape(label)}:[/] [cyan]{Markup.Escape(realHigherDamageSmaller.DisplayName)}[/] -> [cyan]{Markup.Escape(realLowerDamageLarger.DisplayName)}[/]"); + AnsiConsole.MarkupLine($"[grey] Prediction line:[/] {Markup.Escape(predictedHigherDamageSmaller.DisplayName)} size={predictedHigherDamageSmaller.PredictedSizeBytes:N0} kld={predictedHigherDamageSmaller.PredictedKld:0.000000} -> {Markup.Escape(predictedLowerDamageLarger.DisplayName)} size={predictedLowerDamageLarger.PredictedSizeBytes:N0} kld={predictedLowerDamageLarger.PredictedKld:0.000000}"); + AnsiConsole.MarkupLine($"[grey] Real validation line:[/] {Markup.Escape(realHigherDamageSmaller.DisplayName)} size={realHigherDamageSmaller.SizeBytes:N0} kld={realHigherDamageSmaller.Kld:0.000000} -> {Markup.Escape(realLowerDamageLarger.DisplayName)} size={realLowerDamageLarger.SizeBytes:N0} kld={realLowerDamageLarger.Kld:0.000000}"); + } + + private static void PrintCandidatePreviewTable(IReadOnlyList candidates, string title) + { + if (candidates.Count == 0) + return; + + var table = new Table().RoundedBorder().BorderColor(Color.Grey); + table.Title = new TableTitle(Markup.Escape(title)); + table.AddColumn("Attempt"); + table.AddColumn("Candidate"); + table.AddColumn("Pred KLD"); + table.AddColumn("Line"); + table.AddColumn("Gain"); + table.AddColumn("Size GiB"); + table.AddColumn("Rank"); + table.AddColumn("Family"); + table.AddColumn("Bit Space"); + + foreach (var c in candidates.Take(DiagnosticPreviewDisplayCount)) + { + table.AddRow( + c.AttemptOrder.ToString("N0"), + Markup.Escape(HybridBenchmarkRepository.BuildDisplayName(c.Prediction.Quant)), + c.Prediction.PredictedKld.ToString("0.000000"), + c.LinearExpectedKld.ToString("0.000000"), + c.PredictedGainOverLine.ToString("0.000000"), + ToGiB(c.Prediction.PredictedSizeBytes).ToString("0.00"), + c.Prediction.PredictedRank?.ToString("N0") ?? "n/a", + Markup.Escape(string.IsNullOrWhiteSpace(c.CandidateTheoryFamilyDisplay) ? "n/a" : c.CandidateTheoryFamilyDisplay), + Markup.Escape(DescribeBitSpace(c.Prediction.Config))); + } + + AnsiConsole.Write(table); + } + + private static CandidatePreviewLog ToCandidatePreviewLog(HybridSelectionCandidate candidate) => + ToCandidatePreviewLog(candidate, AnalyzeNearLowerAnchorBrutality(candidate)); + + private static CandidatePreviewLog ToCandidatePreviewLog(HybridSelectionCandidate candidate, BrutalityAnalysis brutality) + { + return new CandidatePreviewLog + { + AttemptOrder = candidate.AttemptOrder, + Key = TensorConfigIdentity.ToKey(candidate.Prediction.Config), + DisplayName = HybridBenchmarkRepository.BuildDisplayName(candidate.Prediction.Quant), + PredictedSizeBytes = candidate.Prediction.PredictedSizeBytes, + PredictedSizeGiB = ToGiB(candidate.Prediction.PredictedSizeBytes), + PredictedKld = candidate.Prediction.PredictedKld, + LinearExpectedKld = candidate.LinearExpectedKld, + PredictedGainOverLine = candidate.PredictedGainOverLine, + PredictionConfidence = candidate.Prediction.PredictionConfidence, + PredictionRank = candidate.Prediction.PredictedRank, + RawSelectionRank = candidate.RawSelectionRank, + CandidateTheoryFamilyKey = candidate.CandidateTheoryFamilyKey, + CandidateTheoryFamilyDisplay = candidate.CandidateTheoryFamilyDisplay, + CandidateTheoryFamilyRank = candidate.CandidateTheoryFamilyRank, + CandidateTheoryFamilyMemberRank = candidate.CandidateTheoryFamilyMemberRank, + DiversityMode = candidate.DiversityMode, + BaseQuant = candidate.Prediction.Quant.BaseQuant.Names[0], + BaseBitRange = candidate.Prediction.Quant.BaseQuant.BitRange, + BitSpace = DescribeBitSpace(candidate.Prediction.Config), + OverrideSummary = DescribeOverrides(candidate.Prediction.Config), + BrutalityPassed = brutality.Passed, + BrutalityFractionFromSmallAnchor = brutality.FractionFromSmallAnchor, + BrutalityRequiredGain = brutality.RequiredGain, + BrutalityExplanation = brutality.Explanation + }; + } + + private static object ToAnchorLog(BenchmarkSnapshotRecord anchor) + { + return new + { + key = TensorConfigIdentity.ToKey(anchor.Config), + displayName = anchor.DisplayName, + provider = anchor.ProviderName, + baselineFamily = anchor.BaselineFamily, + sizeBytes = anchor.SizeBytes, + sizeGiB = ToGiB(anchor.SizeBytes), + kld = anchor.Kld, + ppl = anchor.Ppl, + bitRange = anchor.Quant.BaseQuant.BitRange, + quantizeBase = anchor.Quant.BaseQuant.QuantizeBaseArgumentName + }; + } + + private static object? ToPredictionAnchorLog(PredictedAnchorRow? anchor) + { + if (anchor == null) + return null; + + return new + { + configKey = anchor.ConfigKey, + displayName = anchor.DisplayName, + baselineCanonicalKey = anchor.BaselineCanonicalKey, + runtimeBaselineId = anchor.RuntimeBaselineId, + predictedSizeBytes = anchor.PredictedSizeBytes, + predictedSizeGiB = ToGiB(anchor.PredictedSizeBytes), + predictedKld = anchor.PredictedKld, + predictionRank = anchor.PredictionRank, + predictionConfidence = anchor.PredictionConfidence, + isolationSource = DescribeVirtualAnchorIsolationSource(anchor), + isVirtualPredictionAnchor = anchor.IsVirtualPredictionAnchor + }; + } + + private static async Task WriteSelectionPhaseDiagnosticsAsync( + IReadOnlyList phaseDiagnostics, + IReadOnlyList validationFailures, + IReadOnlyList validationAttempts, + CancellationToken ct) + { + string directory = ResolveGgufDirectory(); + Directory.CreateDirectory(directory); + string path = Path.Combine(directory, "magicquant-selection-phase-diagnostics.json"); + string attemptsPath = Path.Combine(directory, "magicquant-selection-validation-attempts.json"); + string missesPath = Path.Combine(directory, "magicquant-selection-validation-misses.json"); + + var payload = new + { + generatedUtc = DateTime.UtcNow, + config = new + { + nearBaselineMaxSizeGrowthPercent = Config.SelectionNearBaselineMaxSizeGrowthPercent, + interiorWindowFractions = Config.SelectionInteriorWindowFractions, + maxCandidatesPerInteriorWindow = Config.SelectionMaxCandidatesPerInteriorWindow, + maxFallbackAttemptsPerAnchor = Config.SelectionMaxFallbackAttemptsPerAnchor, + minimumKldImprovementEpsilon = Config.SelectionMinimumKldImprovementEpsilon, + nearLowerAnchorBrutalZoneFractionOfPairSpan = Config.SelectionNearLowerAnchorBrutalZoneFractionOfPairSpan, + nearAnchorRequiredKldGainFractionOfPairGap = Config.SelectionNearAnchorRequiredKldGainFractionOfPairGap, + allowEightBitAnchorReplacements = Config.SelectionAllowEightBitAnchorReplacements, + diversifyValidationCandidates = Config.SelectionDiversifyValidationCandidates, + diversityScanMultiplier = Config.SelectionDiversityScanMultiplier, + diversityScanMinCandidates = Config.SelectionDiversityScanMinCandidates, + diversityScanMaxCandidates = Config.SelectionDiversityScanMaxCandidates, + diversityLowBitOnly = Config.SelectionDiversityLowBitOnly + }, + totals = new + { + phaseWindowCount = phaseDiagnostics.Count, + selectedForValidation = phaseDiagnostics.Sum(x => x.SelectedForValidationCount), + validationAttempts = validationAttempts.Count, + validationAccepted = validationAttempts.Count(x => x.Accepted), + validationMisses = validationFailures.Count(x => !x.Accepted) + }, + windows = phaseDiagnostics + }; + + var attemptsPayload = validationAttempts.Select(ToValidationAttemptLog).ToList(); + var missesPayload = validationFailures.Where(x => !x.Accepted).Select(ToValidationAttemptLog).ToList(); + + await File.WriteAllTextAsync(path, JsonSerializer.Serialize(payload, JsonOptions), ct); + await File.WriteAllTextAsync(attemptsPath, JsonSerializer.Serialize(attemptsPayload, JsonOptions), ct); + await File.WriteAllTextAsync(missesPath, JsonSerializer.Serialize(missesPayload, JsonOptions), ct); + AnsiConsole.MarkupLine($"[green]Selection phase diagnostics log:[/] {Markup.Escape(path)}"); + AnsiConsole.MarkupLine($"[green]Selection validation attempts log:[/] {Markup.Escape(attemptsPath)}"); + AnsiConsole.MarkupLine($"[green]Selection validation miss log:[/] {Markup.Escape(missesPath)}"); + } + + + private static object ToValidationAttemptLog(CandidateValidationResult attempt) + { + var c = attempt.Candidate; + var snap = attempt.Snapshot; + double? actualLine = null; + double? actualGainOverLine = null; + long? sizeMissBytes = null; + double? kldMiss = null; + bool? actualInsideSizeWindow = null; + bool? actualBeatLine = null; + + if (snap != null) + { + actualLine = InterpolateKldLine(snap.SizeBytes, c.HigherDamageAnchor, c.LowerDamageAnchor); + actualGainOverLine = actualLine.Value - snap.Kld; + actualInsideSizeWindow = snap.SizeBytes >= c.WindowMinSizeBytes && snap.SizeBytes <= c.WindowMaxSizeBytes; + actualBeatLine = snap.Kld + Config.SelectionMinimumKldImprovementEpsilon < actualLine.Value; + + if (snap.SizeBytes < c.WindowMinSizeBytes) + sizeMissBytes = (long)c.WindowMinSizeBytes - (long)snap.SizeBytes; + else if (snap.SizeBytes > c.WindowMaxSizeBytes) + sizeMissBytes = (long)snap.SizeBytes - (long)c.WindowMaxSizeBytes; + else + sizeMissBytes = 0; + + kldMiss = snap.Kld + Config.SelectionMinimumKldImprovementEpsilon - actualLine.Value; + } + + return new + { + accepted = attempt.Accepted, + failureCode = attempt.FailureCode, + message = attempt.Message, + reason = c.Reason.ToString(), + attemptOrder = c.AttemptOrder, + attemptLimit = c.CandidateAttemptLimit, + windowLabel = c.WindowLabel, + phaseWindowIndex = c.PhaseWindowIndex, + phaseWindowCount = c.PhaseWindowCount, + candidateKey = TensorConfigIdentity.ToKey(c.Prediction.Config), + candidateInternalName = HybridBenchmarkRepository.BuildDisplayName(c.Prediction.Quant), + bitSpace = DescribeBitSpace(c.Prediction.Config), + overrideSummary = DescribeOverrides(c.Prediction.Config), + baseQuant = c.Prediction.Quant.BaseQuant.Names[0], + baseBitRange = c.Prediction.Quant.BaseQuant.BitRange, + predicted = new + { + sizeBytes = c.Prediction.PredictedSizeBytes, + sizeGiB = ToGiB(c.Prediction.PredictedSizeBytes), + kld = c.Prediction.PredictedKld, + lineKldAtPredictedSize = c.LinearExpectedKld, + gainOverLine = c.PredictedGainOverLine, + confidence = c.Prediction.PredictionConfidence, + rank = c.Prediction.PredictedRank + }, + selectionContext = new + { + predictionWindowMinSizeBytes = c.PredictionWindowMinSizeBytes, + predictionWindowMaxSizeBytes = c.PredictionWindowMaxSizeBytes, + realValidationWindowMinSizeBytes = c.WindowMinSizeBytes, + realValidationWindowMaxSizeBytes = c.WindowMaxSizeBytes, + candidatePoolSize = c.CandidatePoolSize, + windowCandidateCount = c.WindowCandidateCount, + lineBeatingCandidateCount = c.LineBeatingCandidateCount, + fetchedCandidateCount = c.FetchedCandidateCount, + candidatesAfterBrutalityCount = c.CandidatesAfterBrutalityCount, + candidateAttemptLimit = c.CandidateAttemptLimit, + rawSelectionRank = c.RawSelectionRank, + diversityMode = c.DiversityMode, + selectionStrategy = c.CandidateSelectionNotes.FirstOrDefault(x => x.Contains("selectionStrategy=family-size-ladder", StringComparison.Ordinal)), + candidateTheoryFamilyKey = c.CandidateTheoryFamilyKey, + candidateTheoryFamilyDisplay = c.CandidateTheoryFamilyDisplay, + candidateTheoryFamilyRank = c.CandidateTheoryFamilyRank, + candidateTheoryFamilyMemberRank = c.CandidateTheoryFamilyMemberRank, + notes = c.CandidateSelectionNotes + }, + actual = snap == null + ? null + : new + { + displayName = snap.DisplayName, + sizeBytes = snap.SizeBytes, + sizeGiB = ToGiB(snap.SizeBytes), + kld = snap.Kld, + ppl = snap.Ppl, + lineKldAtActualSize = actualLine, + gainOverLine = actualGainOverLine, + insideSizeWindow = actualInsideSizeWindow, + beatLine = actualBeatLine, + sizeMissBytes, + kldMiss, + positiveKldShortfall = kldMiss.HasValue ? Math.Max(0d, kldMiss.Value) : (double?)null + }, + anchors = new + { + realHigherDamageSmaller = ToAnchorLog(c.HigherDamageAnchor), + realLowerDamageLarger = ToAnchorLog(c.LowerDamageAnchor), + predictionHigherDamageSmaller = ToPredictionAnchorLog(c.HigherDamagePredictionAnchor), + predictionLowerDamageLarger = ToPredictionAnchorLog(c.LowerDamagePredictionAnchor) + } + }; + } + + private static string DescribeBitSpace(TensorConfig config) + { + var baseQuant = BaselineQuants.FromId(config.BaseQuant); + var overrides = DescribeOverrides(config); + return string.IsNullOrWhiteSpace(overrides) + ? $"base={baseQuant.Names[0]}({baseQuant.BitRange}b); overrides=inherit-all" + : $"base={baseQuant.Names[0]}({baseQuant.BitRange}b); overrides={overrides}"; + } + + private static string DescribeOverrides(TensorConfig config) + { + var parts = new List(); + AddOverride(parts, "E", config.Embeddings); + AddOverride(parts, "H", config.LmHead); + AddOverride(parts, "Q", config.AttnQ); + AddOverride(parts, "K", config.AttnKV); + AddOverride(parts, "O", config.AttnOutput); + AddOverride(parts, "U", config.FfnUpGate); + AddOverride(parts, "D", config.FfnDown); + AddOverride(parts, "X", config.MoeExperts); + AddOverride(parts, "R", config.MoeRouter); + return string.Join(", ", parts); + } + + private static void AddOverride(List parts, string groupToken, byte storedSlot) + { + if (BaselineQuants.IsNullTensorConfigGroupSlot(storedSlot)) + return; + + var baseline = BaselineQuants.DecodeTensorConfigGroupSlotToBaseline(storedSlot); + parts.Add($"{groupToken}:{baseline.Names[0]}({baseline.BitRange}b)"); + } + + private static string ResolveGgufDirectory() + { + if (!string.IsNullOrWhiteSpace(Cache.ModelMagicQuantDirectory)) + return Path.Combine(Cache.ModelMagicQuantDirectory!, "GGUF"); + + if (!string.IsNullOrWhiteSpace(Cache.MagicQuantDirectory)) + return Path.Combine(Cache.MagicQuantDirectory!, "GGUF"); + + return Path.Combine(Directory.GetCurrentDirectory(), "GGUF"); + } + + private async Task CountPredictedRowsInIntersectedSizeWindowAsync( + ulong predictionMin, + ulong predictionMax, + ulong realMin, + ulong realMax, + CancellationToken ct) + { + var window = IntersectSizeWindows(predictionMin, predictionMax, realMin, realMax); + if (window == null) + return 0; + + return await _predictedStore.CountPredictedHybridCandidatesInSizeWindowAsync(window.Value.Min, window.Value.Max, ct); + } + + private static (ulong Min, ulong Max)? IntersectSizeWindows( + ulong firstMin, + ulong firstMax, + ulong secondMin, + ulong secondMax) + { + ulong min = Math.Max(firstMin, secondMin); + ulong max = Math.Min(firstMax, secondMax); + return max < min ? null : (min, max); + } + + private static ulong AddPercent(ulong bytes, double percent) + { + if (percent <= 0d) + return bytes; + + double multiplier = 1d + (percent / 100d); + double result = bytes * multiplier; + if (result >= ulong.MaxValue) + return ulong.MaxValue; + + return (ulong)Math.Round(result, MidpointRounding.AwayFromZero); + } + + private static ulong Distance(ulong left, ulong right) => left >= right ? left - right : right - left; + + private static string NormalizeAnchorKey(string? value) => (value ?? string.Empty).Trim().ToLowerInvariant(); + + private static string NormalizePublicEliminationReason(string reason) + { + if (string.IsNullOrWhiteSpace(reason)) + return string.Empty; + + if (reason.Contains("dominance", StringComparison.OrdinalIgnoreCase)) + return "dominance"; + + if (reason.Contains("spacing", StringComparison.OrdinalIgnoreCase)) + return "spacing"; + + if (reason.Contains("strict", StringComparison.OrdinalIgnoreCase)) + return "strict-dominance"; + + return reason.Trim().ToLowerInvariant(); + } + + private static bool Dominates(BenchmarkSnapshotRecord better, BenchmarkSnapshotRecord worse) + { + bool sameOrSmaller = better.SizeBytes <= worse.SizeBytes; + bool strictlyLowerKld = better.Kld + Config.SelectionMinimumKldImprovementEpsilon < worse.Kld; + return sameOrSmaller && strictlyLowerKld; + } + + private static double ToGiB(ulong bytes) => bytes / 1024d / 1024d / 1024d; + + private sealed class ValidationCandidateSelectionResult + { + public IReadOnlyList Candidates { get; init; } = Array.Empty(); + public string Mode { get; init; } = string.Empty; + public bool DiversityEnabled { get; init; } + public string SelectionStrategy { get; init; } = string.Empty; + public int CandidateFamilyCount { get; init; } + public int SelectedFamilyCount { get; init; } + public IReadOnlyList SelectedFamilyKeys { get; init; } = Array.Empty(); + public string DiversitySelectionStrategy { get; init; } = string.Empty; + public int DiversityOverflowCount { get; init; } + public ulong? DiversitySizeFloorStartBytes { get; init; } + public ulong? DiversitySizeFloorEndBytes { get; init; } + public int OverflowCount { get; init; } + public ulong? SizeFloorStartBytes { get; init; } + public ulong? SizeFloorEndBytes { get; init; } + public IReadOnlyList Notes { get; init; } = Array.Empty(); + } + + private sealed class CandidateTheorySignature + { + public string Key { get; init; } = string.Empty; + public string Display { get; init; } = string.Empty; + } + + private sealed class CandidateFamilyEntry + { + public HybridSelectionCandidate Candidate { get; init; } = default!; + public CandidateTheorySignature Signature { get; init; } = new(); + public int RawRank { get; init; } + public int MemberRank { get; set; } + public CandidateTheoryFamily? Family { get; set; } + public string SelectionMode { get; set; } = string.Empty; + public ulong? PreviousSizeFloorBytes { get; set; } + public long? SizeDeltaVsFloorBytes { get; set; } + public double EffectivePredictedKld { get; set; } = double.NaN; + } + + private sealed class CandidateTheoryFamily + { + public string Key { get; init; } = string.Empty; + public string Display { get; init; } = string.Empty; + public int Rank { get; set; } + public List Members { get; init; } = new(); + } + + private sealed class AdjacentAnchorPair + { + public BenchmarkSnapshotRecord LowerDamageLarger { get; init; } = default!; + public BenchmarkSnapshotRecord HigherDamageSmaller { get; init; } = default!; + } + + private sealed class BrutalityAnalysis + { + public bool Passed { get; init; } + public double FractionFromSmallAnchor { get; init; } + public double RequiredGain { get; init; } + public string Explanation { get; init; } = string.Empty; + } + + private sealed class ValidationMetrics + { + public double ActualLineKld { get; init; } + public double ActualGainOverLine { get; init; } + public double KldMiss { get; init; } + public long SizeMissBytes { get; init; } + public bool InsideWindow { get; init; } + public bool BeatsLine { get; init; } + } + + private sealed class SelectionPhaseDiagnostic + { + public string Phase { get; init; } = string.Empty; + public string WindowLabel { get; init; } = string.Empty; + public int PhaseWindowIndex { get; init; } + public int PhaseWindowCount { get; init; } + public object? HigherDamageSmaller { get; init; } + public object? LowerDamageLarger { get; init; } + public object? PredictionHigherDamageSmaller { get; init; } + public object? PredictionLowerDamageLarger { get; init; } + public ulong PredictionWindowMinSizeBytes { get; init; } + public ulong PredictionWindowMaxSizeBytes { get; init; } + public ulong WindowMinSizeBytes { get; init; } + public ulong WindowMaxSizeBytes { get; init; } + public double WindowSizeGiB { get; init; } + public long CandidatePoolSize { get; init; } + public long PredictedPoolCount { get; init; } + public long DeterministicEligibleCount { get; init; } + public long RejectedByRealSizeWindow { get; init; } + public long RejectedByOtherPhaseDeterministicRules { get; init; } + public long WindowCandidateCount { get; init; } + public long LineBeatingCandidateCount { get; init; } + public int FetchedCandidateCount { get; init; } + public int CandidatesAfterBrutalityCount { get; init; } + public int SelectedForValidationCount { get; init; } + public int CandidateAttemptLimit { get; init; } + public int QueryFetchLimit { get; init; } + public bool DiversityEnabled { get; init; } + public string DiversityMode { get; init; } = string.Empty; + public int DiversityScanLimit { get; init; } + public int DiversityScanFetched { get; init; } + public int CandidateFamilyCount { get; init; } + public int SelectedFamilyCount { get; init; } + public IReadOnlyList SelectedFamilyKeys { get; init; } = Array.Empty(); + public string DiversitySelectionStrategy { get; init; } = string.Empty; + public int DiversityOverflowCount { get; init; } + public ulong? DiversitySizeFloorStartBytes { get; init; } + public ulong? DiversitySizeFloorEndBytes { get; init; } + public IReadOnlyList TopCandidates { get; init; } = Array.Empty(); + public IReadOnlyList RejectedByBrutalityPreview { get; init; } = Array.Empty(); + public IReadOnlyList Notes { get; init; } = Array.Empty(); + } + + private sealed class CandidatePreviewLog + { + public int AttemptOrder { get; init; } + public string Key { get; init; } = string.Empty; + public string DisplayName { get; init; } = string.Empty; + public ulong PredictedSizeBytes { get; init; } + public double PredictedSizeGiB { get; init; } + public double PredictedKld { get; init; } + public double LinearExpectedKld { get; init; } + public double PredictedGainOverLine { get; init; } + public double PredictionConfidence { get; init; } + public ulong? PredictionRank { get; init; } + public int RawSelectionRank { get; init; } + public string CandidateTheoryFamilyKey { get; init; } = string.Empty; + public string CandidateTheoryFamilyDisplay { get; init; } = string.Empty; + public int CandidateTheoryFamilyRank { get; init; } + public int CandidateTheoryFamilyMemberRank { get; init; } + public string DiversityMode { get; init; } = string.Empty; + public string BaseQuant { get; init; } = string.Empty; + public byte BaseBitRange { get; init; } + public string BitSpace { get; init; } = string.Empty; + public string OverrideSummary { get; init; } = string.Empty; + public bool BrutalityPassed { get; init; } + public double BrutalityFractionFromSmallAnchor { get; init; } + public double BrutalityRequiredGain { get; init; } + public string BrutalityExplanation { get; init; } = string.Empty; + } +} \ No newline at end of file diff --git a/src/MagicQuant/Services/PredictionValidationService.cs b/src/MagicQuant/Services/PredictionValidationService.cs new file mode 100644 index 0000000..1d16d1d --- /dev/null +++ b/src/MagicQuant/Services/PredictionValidationService.cs @@ -0,0 +1,356 @@ +using System.Globalization; +using System.Text; +using MagicQuant.Models; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; +using Spectre.Console; + +namespace MagicQuant.Services; + +/// +/// Offline validator for the rank-safe isolation predictor. +/// It scores every currently benchmarked general-category combo in the active +/// scoped-model/imatrix bucket, compares predicted KLD to real KLD, and reports +/// rank/order accuracy. +/// +public sealed class PredictionValidationService +{ + private readonly HybridBenchmarkRepository _repository; + private readonly RankSafeKldPredictionService _predictionService; + + public PredictionValidationService( + HybridBenchmarkRepository repository, + RankSafeKldPredictionService predictionService) + { + _repository = repository; + _predictionService = predictionService; + } + + public async Task ExportAsync( + string outputDirectory, + CancellationToken ct = default) + { + Directory.CreateDirectory(outputDirectory); + + var actual = await _repository.LoadAllBenchmarkSnapshotsForCurrentContextAsync( + category: (byte)BenchmarkCategory.General, + strictImatrixContext: true, + ct: ct); + + if (actual.Count == 0) + throw new InvalidOperationException("No category=General benchmark snapshots were found for the active scoped model/imatrix context."); + + var predictions = await _predictionService.PredictAsync(actual.Select(x => x.Config).ToList(), ct); + + var actualByKey = actual.ToDictionary(x => TensorConfigIdentity.ToKey(x.Config), StringComparer.Ordinal); + var rows = predictions.Rows + .Where(x => x.IsPredictable) + .Where(x => actualByKey.ContainsKey(TensorConfigIdentity.ToKey(x.Config))) + .ToList(); + + foreach (var row in rows) + { + var snap = actualByKey[TensorConfigIdentity.ToKey(row.Config)]; + row.ActualKld = snap.Kld; + row.ActualPpl = snap.Ppl; + row.ActualSizeBytes = snap.SizeBytes; + } + + AssignRanks(rows); + + var summary = BuildSummary(rows); + string csvPath = Path.Combine(outputDirectory, "prediction_validation_general.csv"); + string markdownPath = Path.Combine(outputDirectory, "prediction_validation_general.md"); + + await File.WriteAllTextAsync(csvPath, BuildCsv(rows), ct); + await File.WriteAllTextAsync(markdownPath, BuildMarkdown(summary, rows, predictions.Notes), ct); + + PrintSummary(summary, csvPath, markdownPath); + + return new PredictionValidationExportResult + { + Summary = summary, + CsvPath = csvPath, + MarkdownPath = markdownPath, + Rows = rows + .OrderByDescending(x => x.AbsoluteKldError) + .ThenByDescending(x => RankDistance(x)) + .ToList() + }; + } + + private static void AssignRanks(IReadOnlyList rows) + { + int actualRank = 1; + foreach (var row in rows.OrderBy(x => x.ActualKld).ThenBy(x => x.ActualSizeBytes ?? ulong.MaxValue)) + row.ActualRank = actualRank++; + + ulong predictedRank = 1; + foreach (var row in rows.OrderBy(x => x.PredictedKld).ThenBy(x => x.PredictedSizeBytes)) + row.PredictedRank = predictedRank++; + } + + private static RankSafeValidationSummary BuildSummary(IReadOnlyList rows) + { + if (rows.Count == 0) + return new RankSafeValidationSummary(); + + double mae = rows.Average(x => x.AbsoluteKldError); + double rmse = Math.Sqrt(rows.Average(x => x.SignedKldError * x.SignedKldError)); + double maxAbs = rows.Max(x => x.AbsoluteKldError); + double meanSigned = rows.Average(x => x.SignedKldError); + + long concordant = 0; + long discordant = 0; + long tiedPred = 0; + + for (int i = 0; i < rows.Count; i++) + { + for (int j = i + 1; j < rows.Count; j++) + { + double actualDiff = rows[i].ActualKld - rows[j].ActualKld; + double predDiff = rows[i].PredictedKld - rows[j].PredictedKld; + + int actualSign = Math.Sign(actualDiff); + int predSign = Math.Sign(predDiff); + + if (predSign == 0) + { + tiedPred++; + continue; + } + + if (actualSign == 0 || actualSign == predSign) + concordant++; + else + discordant++; + } + } + + long denominator = concordant + discordant; + double pairwise = denominator == 0 ? 100d : concordant * 100d / denominator; + + int ShiftWithin(int maxShift) => + rows.Count(x => x.ActualRank.HasValue && x.PredictedRank.HasValue && + RankDistance(x) <= (ulong)maxShift); + + return new RankSafeValidationSummary + { + RowCount = rows.Count, + PredictableCount = rows.Count, + Mae = mae, + Rmse = rmse, + MaxAbsoluteError = maxAbs, + MeanSignedError = meanSigned, + PairwiseAccuracyPercent = pairwise, + ConcordantPairs = concordant, + DiscordantPairs = discordant, + TiedPredictedPairs = tiedPred, + ExactRankMatches = ShiftWithin(0), + WithinOneRank = ShiftWithin(1), + WithinTwoRanks = ShiftWithin(2), + WithinFiveRanks = ShiftWithin(5), + WithinTenRanks = ShiftWithin(10), + WithinTwentyRanks = ShiftWithin(20) + }; + } + + private static string BuildCsv(IReadOnlyList rows) + { + var sb = new StringBuilder(); + sb.AppendLine("config_key,display_name,is_hybrid,base_quant,is_size_predictable,predicted_kld,actual_kld,abs_error,signed_error,predicted_rank,actual_rank,rank_shift,predicted_size_bytes,actual_size_bytes,size_abs_error_bytes,size_abs_error_percent,predicted_ppl,actual_ppl,effective_groups,notes"); + + foreach (var row in rows + .OrderByDescending(x => x.AbsoluteKldError) + .ThenByDescending(x => RankDistance(x))) + { + long shift = RankShift(row); + sb.Append(Csv(TensorConfigIdentity.ToKey(row.Config))).Append(','); + sb.Append(Csv(HybridBenchmarkRepository.BuildDisplayName(row.Quant))).Append(','); + sb.Append(row.IsHybrid ? "true" : "false").Append(','); + sb.Append(Csv(row.Quant.BaseQuant.Names[0])).Append(','); + sb.Append(row.IsSizePredictable ? "true" : "false").Append(','); + sb.Append(Format(row.PredictedKld)).Append(','); + sb.Append(Format(row.ActualKld)).Append(','); + sb.Append(Format(row.AbsoluteKldError)).Append(','); + sb.Append(Format(row.SignedKldError)).Append(','); + sb.Append(row.PredictedRank?.ToString(CultureInfo.InvariantCulture) ?? "").Append(','); + sb.Append(row.ActualRank?.ToString(CultureInfo.InvariantCulture) ?? "").Append(','); + sb.Append(shift.ToString(CultureInfo.InvariantCulture)).Append(','); + sb.Append(row.PredictedSizeBytes.ToString(CultureInfo.InvariantCulture)).Append(','); + sb.Append(row.ActualSizeBytes?.ToString(CultureInfo.InvariantCulture) ?? "").Append(','); + long sizeError = row.ActualSizeBytes.HasValue ? (long)row.PredictedSizeBytes - (long)row.ActualSizeBytes.Value : 0L; + double sizeErrorPct = row.ActualSizeBytes.HasValue && row.ActualSizeBytes.Value > 0 + ? Math.Abs(sizeError) * 100d / row.ActualSizeBytes.Value + : double.NaN; + sb.Append(row.ActualSizeBytes.HasValue ? Math.Abs(sizeError).ToString(CultureInfo.InvariantCulture) : "").Append(','); + sb.Append(row.ActualSizeBytes.HasValue ? Format(sizeErrorPct) : "").Append(','); + sb.Append(Format(row.PredictedPpl)).Append(','); + sb.Append(Format(row.ActualPpl)).Append(','); + sb.Append(Csv(BuildEffectiveGroupSummary(row.Config))).Append(','); + sb.Append(Csv(string.Join(" | ", row.Notes))); + sb.AppendLine(); + } + + return sb.ToString(); + } + + private static string BuildMarkdown( + RankSafeValidationSummary summary, + IReadOnlyList rows, + IReadOnlyList notes) + { + var sb = new StringBuilder(); + sb.AppendLine("# MagicQuant Rank-Safe Prediction Validation"); + sb.AppendLine(); + sb.AppendLine("This report compares predicted KLD to real category=General benchmark KLD for the active scoped model/imatrix bucket."); + sb.AppendLine(); + + sb.AppendLine("## Summary"); + sb.AppendLine(); + sb.AppendLine("| Metric | Value |"); + sb.AppendLine("|---|---:|"); + sb.AppendLine($"| Rows | {summary.RowCount:N0} |"); + sb.AppendLine($"| MAE | {summary.Mae:0.000000} |"); + sb.AppendLine($"| RMSE | {summary.Rmse:0.000000} |"); + sb.AppendLine($"| Max abs error | {summary.MaxAbsoluteError:0.000000} |"); + sb.AppendLine($"| Mean signed error | {summary.MeanSignedError:0.000000} |"); + sb.AppendLine($"| Pairwise order accuracy | {summary.PairwiseAccuracyPercent:0.0000}% |"); + sb.AppendLine($"| Concordant pairs | {summary.ConcordantPairs:N0} |"); + sb.AppendLine($"| Discordant pairs | {summary.DiscordantPairs:N0} |"); + sb.AppendLine($"| Tied predicted pairs | {summary.TiedPredictedPairs:N0} |"); + + var sizeRows = rows.Where(x => x.ActualSizeBytes.HasValue && x.IsSizePredictable).ToList(); + if (sizeRows.Count > 0) + { + double sizeMaePercent = sizeRows.Average(x => Math.Abs((long)x.PredictedSizeBytes - (long)x.ActualSizeBytes!.Value) * 100d / x.ActualSizeBytes!.Value); + double sizeMaxPercent = sizeRows.Max(x => Math.Abs((long)x.PredictedSizeBytes - (long)x.ActualSizeBytes!.Value) * 100d / x.ActualSizeBytes!.Value); + sb.AppendLine($"| Size-safe rows | {sizeRows.Count:N0} |"); + sb.AppendLine($"| Size MAE % | {sizeMaePercent:0.0000}% |"); + sb.AppendLine($"| Size max abs % | {sizeMaxPercent:0.0000}% |"); + } + sb.AppendLine(); + + sb.AppendLine("## Rank movement"); + sb.AppendLine(); + sb.AppendLine("| Window | Count | Percent |"); + sb.AppendLine("|---|---:|---:|"); + AppendRankWindow(sb, "Exact", summary.ExactRankMatches, summary.RowCount); + AppendRankWindow(sb, "Within 1", summary.WithinOneRank, summary.RowCount); + AppendRankWindow(sb, "Within 2", summary.WithinTwoRanks, summary.RowCount); + AppendRankWindow(sb, "Within 5", summary.WithinFiveRanks, summary.RowCount); + AppendRankWindow(sb, "Within 10", summary.WithinTenRanks, summary.RowCount); + AppendRankWindow(sb, "Within 20", summary.WithinTwentyRanks, summary.RowCount); + sb.AppendLine(); + + if (notes.Count > 0) + { + sb.AppendLine("## Prediction notes"); + sb.AppendLine(); + foreach (var note in notes) + sb.AppendLine($"- {note}"); + sb.AppendLine(); + } + + sb.AppendLine("## Largest size misses"); + sb.AppendLine(); + sb.AppendLine("| Name | Size Safe | Pred Size GB | Actual Size GB | Abs Error MB | Error % | Groups |"); + sb.AppendLine("|---|---:|---:|---:|---:|---:|---|"); + foreach (var row in rows + .Where(x => x.ActualSizeBytes.HasValue) + .OrderByDescending(x => Math.Abs((long)x.PredictedSizeBytes - (long)x.ActualSizeBytes!.Value)) + .Take(50)) + { + long absBytes = Math.Abs((long)row.PredictedSizeBytes - (long)row.ActualSizeBytes!.Value); + double pct = row.ActualSizeBytes.Value == 0 ? 0d : absBytes * 100d / row.ActualSizeBytes.Value; + double mb = absBytes / 1024d / 1024d; + sb.AppendLine($"| {EscapePipe(HybridBenchmarkRepository.BuildDisplayName(row.Quant))} | {(row.IsSizePredictable ? "yes" : "no")} | {ToGb(row.PredictedSizeBytes)} | {ToGb(row.ActualSizeBytes.Value)} | {mb:0.00} | {pct:0.0000}% | {EscapePipe(BuildEffectiveGroupSummary(row.Config))} |"); + } + sb.AppendLine(); + + sb.AppendLine("## Largest KLD misses"); + sb.AppendLine(); + sb.AppendLine("| Name | Predicted KLD | Actual KLD | Abs Error | Pred Rank | Actual Rank | Shift | Size Pred GB | Size Actual GB | Groups |"); + sb.AppendLine("|---|---:|---:|---:|---:|---:|---:|---:|---:|---|"); + + foreach (var row in rows + .OrderByDescending(x => x.AbsoluteKldError) + .ThenByDescending(x => RankDistance(x)) + .Take(100)) + { + long shift = RankShift(row); + sb.AppendLine( + $"| {EscapePipe(HybridBenchmarkRepository.BuildDisplayName(row.Quant))} | {row.PredictedKld:0.000000} | {row.ActualKld:0.000000} | {row.AbsoluteKldError:0.000000} | " + + $"{row.PredictedRank} | {row.ActualRank} | {shift:+#;-#;0} | {ToGb(row.PredictedSizeBytes)} | {ToGb(row.ActualSizeBytes ?? 0)} | {EscapePipe(BuildEffectiveGroupSummary(row.Config))} |"); + } + + return sb.ToString(); + } + + private static void PrintSummary(RankSafeValidationSummary summary, string csvPath, string markdownPath) + { + AnsiConsole.Write(new Rule("[yellow]Prediction Validation[/]") { Justification = Justify.Left }); + AnsiConsole.MarkupLine($"[green]Rows:[/] [cyan]{summary.RowCount:N0}[/]"); + AnsiConsole.MarkupLine($"[green]MAE:[/] [cyan]{summary.Mae:0.000000}[/] [green]RMSE:[/] [cyan]{summary.Rmse:0.000000}[/] [green]MaxAbs:[/] [cyan]{summary.MaxAbsoluteError:0.000000}[/]"); + AnsiConsole.MarkupLine($"[green]Pairwise order accuracy:[/] [cyan]{summary.PairwiseAccuracyPercent:0.0000}%[/] [grey]discordant={summary.DiscordantPairs:N0} tied-pred={summary.TiedPredictedPairs:N0}[/]"); + AnsiConsole.MarkupLine($"[green]CSV:[/] [blue]{Markup.Escape(csvPath)}[/]"); + AnsiConsole.MarkupLine($"[green]Markdown:[/] [blue]{Markup.Escape(markdownPath)}[/]"); + } + + private static void AppendRankWindow(StringBuilder sb, string label, int count, int total) + { + double pct = total == 0 ? 0d : count * 100d / total; + sb.AppendLine($"| {label} | {count:N0} | {pct:0.00}% |"); + } + + private static string BuildEffectiveGroupSummary(TensorConfig config) + { + return string.Join("; ", + RankSafeKldPredictionService.EnumerateEffectiveBaselines(config) + .Select(x => + { + var baseline = BaselineQuants.FromId(x.EffectiveBaselineId); + return $"{x.Group.Name}={baseline.Names[0]}"; + })); + } + + private static ulong RankDistance(RankSafePredictionRow row) + { + if (!row.PredictedRank.HasValue || !row.ActualRank.HasValue) + return 0UL; + + ulong actual = (ulong)Math.Max(0, row.ActualRank.Value); + return row.PredictedRank.Value >= actual + ? row.PredictedRank.Value - actual + : actual - row.PredictedRank.Value; + } + + private static long RankShift(RankSafePredictionRow row) + { + if (!row.PredictedRank.HasValue || !row.ActualRank.HasValue) + return 0L; + + long predicted = row.PredictedRank.Value > long.MaxValue + ? long.MaxValue + : (long)row.PredictedRank.Value; + + return predicted - row.ActualRank.Value; + } + + private static string Format(double value) => + double.IsNaN(value) || double.IsInfinity(value) + ? "" + : value.ToString("0.000000########", CultureInfo.InvariantCulture); + + private static string Csv(string value) + { + value ??= string.Empty; + if (!value.Contains(',') && !value.Contains('"') && !value.Contains('\n') && !value.Contains('\r')) + return value; + + return "\"" + value.Replace("\"", "\"\"") + "\""; + } + + private static string ToGb(ulong bytes) => (bytes / 1024d / 1024d / 1024d).ToString("0.00", CultureInfo.InvariantCulture); + private static string EscapePipe(string value) => (value ?? string.Empty).Replace("|", "\\|"); +} \ No newline at end of file diff --git a/src/MagicQuant/Services/Progress/StageProgressOptions.cs b/src/MagicQuant/Services/Progress/StageProgressOptions.cs new file mode 100644 index 0000000..e761ef7 --- /dev/null +++ b/src/MagicQuant/Services/Progress/StageProgressOptions.cs @@ -0,0 +1,15 @@ +namespace MagicQuant.Services.Progress; + +public sealed class StageProgressOptions +{ + public string StageName { get; init; } = string.Empty; + public int Total { get; init; } + public int MinimumNonSkippedSamplesBeforeEta { get; init; } = 2; + public TimeSpan MinimumPrintInterval { get; init; } = TimeSpan.FromSeconds(15); + public int PrintEveryNFinished { get; init; } = 1; + public bool ShowEta { get; init; } = true; + public bool CountSkippedForEta { get; init; } = false; + public TimeSpan MinimumEtaSampleDuration { get; init; } = TimeSpan.FromSeconds(1); + public bool PrintFinalSummary { get; init; } = true; + public string? UnitLabel { get; init; } +} diff --git a/src/MagicQuant/Services/Progress/StageProgressSnapshot.cs b/src/MagicQuant/Services/Progress/StageProgressSnapshot.cs new file mode 100644 index 0000000..d9a78ba --- /dev/null +++ b/src/MagicQuant/Services/Progress/StageProgressSnapshot.cs @@ -0,0 +1,13 @@ +namespace MagicQuant.Services.Progress; + +public readonly record struct StageProgressSnapshot( + string StageName, + int Total, + DateTime StartedUtc, + int Completed, + int Skipped, + int Failed, + int Finished, + DateTime CapturedUtc, + DateTime LastPrintedUtc, + int LastPrintedFinished); diff --git a/src/MagicQuant/Services/Progress/StageProgressTracker.cs b/src/MagicQuant/Services/Progress/StageProgressTracker.cs new file mode 100644 index 0000000..ebaffa6 --- /dev/null +++ b/src/MagicQuant/Services/Progress/StageProgressTracker.cs @@ -0,0 +1,184 @@ +using Spectre.Console; + +namespace MagicQuant.Services.Progress; + +public sealed class StageProgressTracker +{ + private readonly StageProgressOptions _options; + private readonly object _printSync = new(); + + private int _completed; + private int _skipped; + private int _failed; + private int _etaSamples; + private int _lastPrintedFinished; + private DateTime _lastPrintedUtc; + + public StageProgressTracker(StageProgressOptions options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + + if (_options.Total < 0) + throw new ArgumentOutOfRangeException(nameof(options.Total), "Total cannot be negative."); + + StageName = string.IsNullOrWhiteSpace(_options.StageName) ? "Stage" : _options.StageName.Trim(); + Total = _options.Total; + StartedUtc = DateTime.UtcNow; + _lastPrintedUtc = StartedUtc; + } + + public string StageName { get; } + public int Total { get; } + public DateTime StartedUtc { get; } + + public StageProgressSnapshot Snapshot + { + get + { + int completed = Volatile.Read(ref _completed); + int skipped = Volatile.Read(ref _skipped); + int failed = Volatile.Read(ref _failed); + int finished = completed + skipped + failed; + return new StageProgressSnapshot( + StageName, + Total, + StartedUtc, + completed, + skipped, + failed, + finished, + DateTime.UtcNow, + _lastPrintedUtc, + Volatile.Read(ref _lastPrintedFinished)); + } + } + + public void ReportFinished( + SampleProcessState state, + string? itemName = null, + TimeSpan? duration = null, + bool? countForEtaOverride = null) + { + switch (state) + { + case SampleProcessState.Completed: + Interlocked.Increment(ref _completed); + break; + case SampleProcessState.Skipped: + Interlocked.Increment(ref _skipped); + break; + default: + Interlocked.Increment(ref _failed); + break; + } + + bool countForEta = countForEtaOverride ?? ShouldCountForEta(state, duration); + if (countForEta) + Interlocked.Increment(ref _etaSamples); + + MaybePrint(state, itemName); + } + + private bool ShouldCountForEta(SampleProcessState state, TimeSpan? duration) + { + if (!duration.HasValue || duration.Value < _options.MinimumEtaSampleDuration) + return false; + + return state switch + { + SampleProcessState.Completed => true, + SampleProcessState.Failed => true, + SampleProcessState.Skipped => _options.CountSkippedForEta, + _ => false + }; + } + + private void MaybePrint(SampleProcessState justFinishedState, string? itemName) + { + var now = DateTime.UtcNow; + + lock (_printSync) + { + int completed = Volatile.Read(ref _completed); + int skipped = Volatile.Read(ref _skipped); + int failed = Volatile.Read(ref _failed); + int finished = completed + skipped + failed; + + bool isFinal = Total > 0 && finished >= Total; + bool intervalElapsed = now - _lastPrintedUtc >= _options.MinimumPrintInterval; + bool countThresholdHit = finished - _lastPrintedFinished >= Math.Max(1, _options.PrintEveryNFinished); + bool rapidSkipStorm = justFinishedState == SampleProcessState.Skipped && + finished - _lastPrintedFinished < Math.Max(1, _options.PrintEveryNFinished) && + !intervalElapsed && + !isFinal; + + if (!isFinal && !intervalElapsed && (!countThresholdHit || rapidSkipStorm)) + return; + + if (!isFinal && finished == _lastPrintedFinished) + return; + + string escapedStage = Markup.Escape(StageName); + + if (!_options.ShowEta) + { + var unitLabel = string.IsNullOrWhiteSpace(_options.UnitLabel) + ? "items finished" + : _options.UnitLabel.Trim(); + + AnsiConsole.MarkupLine($"[grey][[progress]][/] {escapedStage}: [cyan]{finished}[/]/[cyan]{Total}[/] {Markup.Escape(unitLabel)}"); + } + else + { + var elapsed = now - StartedUtc; + string elapsedText = FormatDuration(elapsed); + + int etaSampleCount = Volatile.Read(ref _etaSamples); + string etaText = "ETA warming up..."; + string estFinishText = "est finish UTC n/a"; + + if (elapsed.TotalSeconds > 0 && etaSampleCount >= Math.Max(1, _options.MinimumNonSkippedSamplesBeforeEta)) + { + double rate = etaSampleCount / elapsed.TotalSeconds; + int remaining = Math.Max(0, Total - finished); + if (rate > 0) + { + var estimatedRemaining = TimeSpan.FromSeconds(remaining / rate); + var estimatedFinishUtc = now + estimatedRemaining; + etaText = $"ETA {FormatEta(estimatedRemaining)}"; + estFinishText = $"est finish UTC {estimatedFinishUtc:yyyy-MM-dd HH:mm}"; + } + } + + string maybeItem = string.IsNullOrWhiteSpace(itemName) + ? string.Empty + : $" | item={Markup.Escape(itemName)}"; + + AnsiConsole.MarkupLine( + $"[grey][[progress]][/] {escapedStage}: [cyan]{finished}[/]/[cyan]{Total}[/] done | completed=[green]{completed}[/] skipped=[yellow]{skipped}[/] failed=[red]{failed}[/] | elapsed={elapsedText} | {etaText} | {estFinishText}{maybeItem}"); + } + + _lastPrintedUtc = now; + _lastPrintedFinished = finished; + } + } + + private static string FormatDuration(TimeSpan duration) + { + if (duration.TotalDays >= 1) + return $"{(int)duration.TotalDays}d {duration.Hours:00}h {duration.Minutes:00}m"; + + return $"{duration.Hours:00}h {duration.Minutes:00}m {duration.Seconds:00}s"; + } + + private static string FormatEta(TimeSpan duration) + { + if (duration.TotalDays >= 1) + return $"{(int)duration.TotalDays}d {duration.Hours:00}h {duration.Minutes:00}m"; + + if (duration.TotalHours >= 1) + return $"{duration.Hours:00}h {duration.Minutes:00}m"; + + return $"{duration.Minutes:00}m"; + } +} diff --git a/src/MagicQuant/Services/QuantDatabaseService.cs b/src/MagicQuant/Services/QuantDatabaseService.cs new file mode 100644 index 0000000..daf6ad1 --- /dev/null +++ b/src/MagicQuant/Services/QuantDatabaseService.cs @@ -0,0 +1,1000 @@ +using System.Collections.Immutable; +using System.Diagnostics; +using System.Numerics; +using DuckDB.NET.Data; +using MagicQuant.Helpers; +using MagicQuant.Models; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models; +using Microsoft.EntityFrameworkCore; +using Spectre.Console; + +namespace MagicQuant.Services; + +public class QuantDatabaseService +{ + private const string TableName = CombinationDuckDbSchema.TableName; + + private static readonly string[] ExpectedColumnTypes = CombinationDuckDbSchema.ExpectedColumnTypes; + + private static string CreateTableSql => CombinationDuckDbSchema.CreateTableSql; + + private static async Task ConfigureFastLoadSessionAsync(DuckDBConnection connection, CancellationToken ct) + { + // These are safe session-level tweaks for this write-heavy workload. + // We do not care about insertion order for tensor combo staging. + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = "SET preserve_insertion_order = false;"; + await cmd.ExecuteNonQueryAsync(ct); + } + + // Let DuckDB use the available machine parallelism. + int threadCount = Math.Max(1, Environment.ProcessorCount); + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = $"SET threads = {threadCount};"; + await cmd.ExecuteNonQueryAsync(ct); + } + } + + private static async Task RecreateTableAsync(DuckDBConnection connection, CancellationToken ct) + { + using var createCmd = connection.CreateCommand(); + createCmd.CommandText = CreateTableSql; + await createCmd.ExecuteNonQueryAsync(ct); + } + + public async Task GetRemainingCombinationCountAsync(CancellationToken ct = default) + { + using var connection = new DuckDBConnection(ConnectionString); + await connection.OpenAsync(ct); + await ConfigureFastLoadSessionAsync(connection, ct); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = $"SELECT COUNT(*) FROM {TableName};"; + + return ToInt64(await cmd.ExecuteScalarAsync(ct)); + } + + public async Task> GetRemainingTensorConfigsAsync(CancellationToken ct = default) + { + using var connection = new DuckDBConnection(ConnectionString); + await connection.OpenAsync(ct); + await ConfigureFastLoadSessionAsync(connection, ct); + long count = await GetRowCountAsync(connection, ct); + if (count > Config.MaxInMemoryCombinationLoadRows) + throw new InvalidOperationException($"Refusing to load {count:N0} DuckDB tensor configs into memory. Use SQL-native filtering/streaming instead."); + + var results = new List(); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@" + SELECT + BaseQuant, + Embeddings, + LmHead, + AttnQ, + AttnKV, + AttnOutput, + FfnUpGate, + FfnDown, + MoeExperts, + MoeRouter + FROM {TableName} + ORDER BY + BaseQuant, + Embeddings, + LmHead, + AttnQ, + AttnKV, + AttnOutput, + FfnUpGate, + FfnDown, + MoeExperts, + MoeRouter;"; + + using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + results.Add(new TensorConfig( + baseQuant: Convert.ToByte(reader.GetValue(0)), + embeddings: Convert.ToByte(reader.GetValue(1)), + lmHead: Convert.ToByte(reader.GetValue(2)), + attnQ: Convert.ToByte(reader.GetValue(3)), + attnKV: Convert.ToByte(reader.GetValue(4)), + attnOutput: Convert.ToByte(reader.GetValue(5)), + ffnUpGate: Convert.ToByte(reader.GetValue(6)), + ffnDown: Convert.ToByte(reader.GetValue(7)), + moeExperts: Convert.ToByte(reader.GetValue(8)), + moeRouter: Convert.ToByte(reader.GetValue(9)) + )); + } + + return results; + } + + private string ConnectionString => $"Data Source={CombinationDatabasePathService.GetPath()}"; + + public async Task InitializeAsync(bool forceRebuild = false, CancellationToken ct = default) + { + var duckDbDirectory = CombinationDatabasePathService.GetDirectory(); + Directory.CreateDirectory(duckDbDirectory); + + using var connection = new DuckDBConnection(ConnectionString); + await connection.OpenAsync(ct); + await ConfigureFastLoadSessionAsync(connection, ct); + + BigInteger expectedTotal = ComboCounter.CountAll(); + bool tableShapeOk = await HasExpectedTableShapeAsync(connection, ct); + long currentDbCount = tableShapeOk + ? await GetRowCountAsync(connection, ct) + : -1; + long currentNormalDbCount = tableShapeOk + ? await GetNormalRowCountAsync(connection, ct) + : -1; + + AnsiConsole.MarkupLine( + $"[bold]DuckDB Check:[/] Current Rows: [cyan]{currentDbCount:N0}[/] | Normal Candidate Rows: [cyan]{currentNormalDbCount:N0}[/] | Expected Normal Rows: [yellow]{expectedTotal:N0}[/]"); + + if (!tableShapeOk) + AnsiConsole.MarkupLine("[yellow]DuckDB table shape is missing or stale. Rebuild required.[/]"); + + if (forceRebuild || !tableShapeOk || new BigInteger(currentNormalDbCount) != expectedTotal) + { + AnsiConsole.MarkupLine("[bold red]DuckDB empty, mismatch, forced, or stale.[/] Initializing/Rebuilding..."); + await RebuildDatabaseAsync(connection, expectedTotal, ct); + } + else + { + var virtualAnchorStats = await AppendVirtualPredictionAnchorRowsAsync(connection, ct); + AnsiConsole.MarkupLine( + $"[bold green]DuckDB is synchronized and ready.[/] [grey]Virtual anchors inserted={virtualAnchorStats.InsertedRows:N0}, marked={virtualAnchorStats.MarkedExistingRows:N0}[/]"); + } + } + + public async Task RebuildAsync(CancellationToken ct = default) + { + await InitializeAsync(forceRebuild: true, ct: ct); + } + + public async Task PrunePredictedLargerThanQ8Async( + RequiredSampleGenerationResult fullPlan, + CancellationToken ct = default) + { + using var connection = new DuckDBConnection(ConnectionString); + await connection.OpenAsync(ct); + await ConfigureFastLoadSessionAsync(connection, ct); + + var predictionContext = await BuildPredictionContextAsync(fullPlan, ct); + + if (predictionContext == null) + { + AnsiConsole.MarkupLine("[yellow]Predicted-size pruning skipped: prediction context was incomplete.[/]"); + return 0; + } + + if (Config.ManualMaxPredictedSizeBytes <= 0 && predictionContext.ShouldSkipPureQ8CeilingPruning) + { + if (!string.IsNullOrWhiteSpace(predictionContext.SkipPureQ8CeilingReason)) + { + AnsiConsole.MarkupLine($"[green]Predicted-size pruning removed 0 combinations.[/] [grey]{Markup.Escape(predictionContext.SkipPureQ8CeilingReason!)}[/]"); + } + else + { + AnsiConsole.MarkupLine("[green]Predicted-size pruning removed 0 combinations.[/]"); + } + + return 0; + } + + long beforeCount = await GetRowCountAsync(connection, ct); + + ulong sizeCeilingBytes = Config.ManualMaxPredictedSizeBytes > 0 + ? Config.ManualMaxPredictedSizeBytes + : predictionContext.PureQ8BaseSize; + await BuildPredictedSizeLookupTablesAsync(connection, predictionContext, ct); + + using (var pruneCmd = connection.CreateCommand()) + { + pruneCmd.CommandText = $@" +DROP TABLE IF EXISTS tensor_configs_pruned; +CREATE TABLE tensor_configs_pruned AS +SELECT t.* +FROM {TableName} t +JOIN temp_base_predicted_size b ON b.BaseQuant = t.BaseQuant +LEFT JOIN temp_group_size_delta de ON de.BaseQuant = t.BaseQuant AND de.GroupName = 'Embeddings' AND de.StoredSlot = t.Embeddings +LEFT JOIN temp_group_size_delta dl ON dl.BaseQuant = t.BaseQuant AND dl.GroupName = 'LmHead' AND dl.StoredSlot = t.LmHead +LEFT JOIN temp_group_size_delta daq ON daq.BaseQuant = t.BaseQuant AND daq.GroupName = 'AttnQ' AND daq.StoredSlot = t.AttnQ +LEFT JOIN temp_group_size_delta dakv ON dakv.BaseQuant = t.BaseQuant AND dakv.GroupName = 'AttnKV' AND dakv.StoredSlot = t.AttnKV +LEFT JOIN temp_group_size_delta dao ON dao.BaseQuant = t.BaseQuant AND dao.GroupName = 'AttnOutput' AND dao.StoredSlot = t.AttnOutput +LEFT JOIN temp_group_size_delta dfu ON dfu.BaseQuant = t.BaseQuant AND dfu.GroupName = 'FfnUpGate' AND dfu.StoredSlot = t.FfnUpGate +LEFT JOIN temp_group_size_delta dfd ON dfd.BaseQuant = t.BaseQuant AND dfd.GroupName = 'FfnDown' AND dfd.StoredSlot = t.FfnDown +LEFT JOIN temp_group_size_delta dme ON dme.BaseQuant = t.BaseQuant AND dme.GroupName = 'MoeExperts' AND dme.StoredSlot = t.MoeExperts +LEFT JOIN temp_group_size_delta dmr ON dmr.BaseQuant = t.BaseQuant AND dmr.GroupName = 'MoeRouter' AND dmr.StoredSlot = t.MoeRouter +WHERE CAST(b.BaseSizeBytes AS BIGINT) + + COALESCE(de.DeltaBytes, 0) + COALESCE(dl.DeltaBytes, 0) + COALESCE(daq.DeltaBytes, 0) + + COALESCE(dakv.DeltaBytes, 0) + COALESCE(dao.DeltaBytes, 0) + COALESCE(dfu.DeltaBytes, 0) + + COALESCE(dfd.DeltaBytes, 0) + COALESCE(dme.DeltaBytes, 0) + COALESCE(dmr.DeltaBytes, 0) + <= CAST({sizeCeilingBytes} AS BIGINT); +DROP TABLE {TableName}; +ALTER TABLE tensor_configs_pruned RENAME TO {TableName};"; + await pruneCmd.ExecuteNonQueryAsync(ct); + } + + long afterCount = await GetRowCountAsync(connection, ct); + long removed = beforeCount - afterCount; + + string ceilingLabel = Config.ManualMaxPredictedSizeBytes > 0 + ? $"manual ceiling {Config.ManualMaxPredictedSizeBytes:N0} bytes" + : "pure Q8"; + + AnsiConsole.MarkupLine($"[yellow]Predicted-size pruning removed:[/] [red]{removed:N0}[/] combo(s) larger than {ceilingLabel}."); + return removed; + } + + public async Task PruneHighPrecisionHybridCandidatesAsync(CancellationToken ct = default) + { + if (RuntimeSearchSpace.AllowHighPrecisionHybrids) + return 0; + + using var connection = new DuckDBConnection(ConnectionString); + await connection.OpenAsync(ct); + await ConfigureFastLoadSessionAsync(connection, ct); + + long beforeCount = await GetRowCountAsync(connection, ct); + byte bf16Stored = BaselineQuants.EncodeTensorConfigGroupSlotBaselineId(BaselineQuants.BF16_Hybrid.UniqueId); + byte f16Stored = BaselineQuants.EncodeTensorConfigGroupSlotBaselineId(BaselineQuants.F16_Hybrid.UniqueId); + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = $@" +DROP TABLE IF EXISTS tensor_configs_pruned; +CREATE TABLE tensor_configs_pruned AS +SELECT * +FROM {TableName} +WHERE Embeddings NOT IN ({bf16Stored}, {f16Stored}) + AND LmHead NOT IN ({bf16Stored}, {f16Stored}) + AND AttnQ NOT IN ({bf16Stored}, {f16Stored}) + AND AttnKV NOT IN ({bf16Stored}, {f16Stored}) + AND AttnOutput NOT IN ({bf16Stored}, {f16Stored}) + AND FfnUpGate NOT IN ({bf16Stored}, {f16Stored}) + AND FfnDown NOT IN ({bf16Stored}, {f16Stored}) + AND MoeExperts NOT IN ({bf16Stored}, {f16Stored}) + AND MoeRouter NOT IN ({bf16Stored}, {f16Stored}); +DROP TABLE {TableName}; +ALTER TABLE tensor_configs_pruned RENAME TO {TableName};"; + await cmd.ExecuteNonQueryAsync(ct); + } + long afterCount = await GetRowCountAsync(connection, ct); + return beforeCount - afterCount; + } + + private async Task HasExpectedTableShapeAsync(DuckDBConnection connection, CancellationToken ct) + { + using var existsCmd = connection.CreateCommand(); + existsCmd.CommandText = $"SELECT COUNT(*) FROM information_schema.tables WHERE table_name = '{TableName}'"; + long exists = (long)(await existsCmd.ExecuteScalarAsync(ct) ?? 0L); + + if (exists == 0) + return false; + + var actual = new List(); + + using var shapeCmd = connection.CreateCommand(); + shapeCmd.CommandText = $@" + SELECT lower(data_type) + FROM information_schema.columns + WHERE table_name = '{TableName}' + ORDER BY ordinal_position;"; + + using var reader = await shapeCmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + actual.Add(Convert.ToString(reader.GetValue(0)) ?? string.Empty); + } + + if (actual.Count != ExpectedColumnTypes.Length) + return false; + + for (int i = 0; i < ExpectedColumnTypes.Length; i++) + { + if (!string.Equals(actual[i], ExpectedColumnTypes[i], StringComparison.Ordinal)) + return false; + } + + return true; + } + + private async Task GetRowCountAsync(DuckDBConnection connection, CancellationToken ct) + { + using var countCmd = connection.CreateCommand(); + countCmd.CommandText = $"SELECT COUNT(*) FROM {TableName}"; + return ToInt64(await countCmd.ExecuteScalarAsync(ct)); + } + + private async Task GetNormalRowCountAsync(DuckDBConnection connection, CancellationToken ct) + { + using var countCmd = connection.CreateCommand(); + countCmd.CommandText = $"SELECT COUNT(*) FROM {TableName} WHERE COALESCE(IsVirtualPredictionAnchor, FALSE) = FALSE"; + return ToInt64(await countCmd.ExecuteScalarAsync(ct)); + } + + private async Task RebuildDatabaseAsync( + DuckDBConnection connection, + BigInteger expectedTotal, + CancellationToken ct) + { + AnsiConsole.MarkupLine($"[yellow]Starting SQL-native tensor combination generation for {expectedTotal:N0} rows...[/]"); + + await RecreateTableAsync(connection, ct); + await ConfigureFastLoadSessionAsync(connection, ct); + + BigInteger insertedGrandTotal = BigInteger.Zero; + var overallSw = Stopwatch.StartNew(); + + foreach (var baseline in RuntimeSearchSpace.GetActiveCombinationBaselines()) + { + ct.ThrowIfCancellationRequested(); + + var baseSw = Stopwatch.StartNew(); + string baseName = baseline.Names.FirstOrDefault() ?? baseline.UniqueId.ToString(); + long before = await GetRowCountAsync(connection, ct); + BigInteger expectedForBase = await InsertBaselineCombinationsSqlAsync(connection, baseline, ct); + long after = await GetRowCountAsync(connection, ct); + BigInteger delta = new(after - before); + if (delta != expectedForBase) + throw new InvalidOperationException($"Baseline {baseName} inserted {delta} rows, expected {expectedForBase}."); + insertedGrandTotal += delta; + + baseSw.Stop(); + + double rowsPerSec = baseSw.Elapsed.TotalSeconds <= 0 + ? 0 : (double)(long)expectedForBase / baseSw.Elapsed.TotalSeconds; + + AnsiConsole.MarkupLine( + $"[bold green]Base complete:[/] {Markup.Escape(baseName)} " + + $"[grey]| Inserted:[/] {expectedForBase:N0} rows " + + $"[grey]| Time:[/] {baseSw.Elapsed.TotalMinutes:N2} min " + + $"[grey]| Rate:[/] {rowsPerSec:N0} rows/sec"); + } + + var virtualAnchorStats = await AppendVirtualPredictionAnchorRowsAsync(connection, ct); + + overallSw.Stop(); + + long finalCount = await GetRowCountAsync(connection, ct); + BigInteger expectedIncludingVirtualRows = expectedTotal + new BigInteger(virtualAnchorStats.InsertedRows); + + double finalRate = overallSw.Elapsed.TotalSeconds <= 0 + ? 0 + : (double)(long)insertedGrandTotal / overallSw.Elapsed.TotalSeconds; + + AnsiConsole.MarkupLine( + $"[bold green]DuckDB rebuild complete.[/] " + + $"[grey]| Inserted tracked:[/] {insertedGrandTotal:N0} " + + $"[grey]| Virtual anchors inserted:[/] {virtualAnchorStats.InsertedRows:N0} " + + $"[grey]| Virtual anchors marked:[/] {virtualAnchorStats.MarkedExistingRows:N0} " + + $"[grey]| Final row count:[/] {finalCount:N0} " + + $"[grey]| Time:[/] {overallSw.Elapsed.TotalMinutes:N2} min " + + $"[grey]| Avg rate:[/] {finalRate:N0} rows/sec"); + if (new BigInteger(finalCount) != expectedIncludingVirtualRows) + throw new InvalidOperationException($"Final tensor_configs row count mismatch. actual={finalCount:N0}, expected={expectedIncludingVirtualRows:N0} (normal={expectedTotal:N0}, virtual-inserted={virtualAnchorStats.InsertedRows:N0})."); + } + + private static async Task AppendVirtualPredictionAnchorRowsAsync( + DuckDBConnection connection, + CancellationToken ct) + { + var activeGroups = GetVirtualPredictionAnchorActiveGroups(); + if (activeGroups.Count == 0) + { + AnsiConsole.MarkupLine("[yellow]Virtual prediction anchors skipped:[/] no active tensor groups were available."); + return new VirtualAnchorInsertStats(); + } + + var carrier = ChooseVirtualPredictionAnchorCarrier(); + var anchorBaselines = GetVirtualPredictionAnchorBaselines() + .Where(x => !BaselineQuants.IsNativeExactAlias(x.UniqueId)) + .OrderByDescending(x => x.BitRange) + .ThenBy(x => x.ExplicitCandidateSortOrder) + .ThenBy(x => x.UniqueId) + .ToList(); + + if (anchorBaselines.Count == 0) + { + AnsiConsole.MarkupLine("[yellow]Virtual prediction anchors skipped:[/] no active baseline identities were available."); + return new VirtualAnchorInsertStats(); + } + + int inserted = 0; + int markedExisting = 0; + var preview = new List(); + + foreach (var baseline in anchorBaselines) + { + ct.ThrowIfCancellationRequested(); + + var quant = HybridQuant.CreateLearnedCandidateBlanket( + baseQuant: carrier, + groups: activeGroups, + candidateBaseline: baseline); + + var config = (TensorConfig)quant; + bool rowAlreadyExisted = await VirtualAnchorRowExistsAsync(connection, config, ct); + await UpsertVirtualAnchorRowAsync(connection, config, baseline, ct); + + if (rowAlreadyExisted) + markedExisting++; + else + inserted++; + + if (preview.Count < 12) + preview.Add($"{baseline.Names[0]} -> {TensorConfigIdentity.ToKey(config)}"); + } + + string groupList = string.Join(", ", activeGroups.Select(x => x.Name)); + AnsiConsole.MarkupLine( + $"[green]Virtual prediction anchors staged:[/] inserted=[cyan]{inserted:N0}[/], marked-existing=[cyan]{markedExisting:N0}[/], " + + $"carrier=[cyan]{Markup.Escape(carrier.Names[0])}[/], groups=[cyan]{Markup.Escape(groupList)}[/]"); + + foreach (var item in preview) + AnsiConsole.MarkupLine($" [grey]- {Markup.Escape(item)}[/]"); + + if (anchorBaselines.Count > preview.Count) + AnsiConsole.MarkupLine($" [grey]- ... {anchorBaselines.Count - preview.Count:N0} more virtual anchors[/]"); + + return new VirtualAnchorInsertStats + { + InsertedRows = inserted, + MarkedExistingRows = markedExisting + }; + } + + private static IReadOnlyList GetVirtualPredictionAnchorActiveGroups() + { + return TReg.All + .Where(x => !Cache.UnusedTensorGroups.Any(u => u.UniqueId == x.UniqueId)) + .OrderBy(x => x.UniqueId) + .ToList(); + } + + private static IReadOnlyList GetVirtualPredictionAnchorBaselines() + { + bool hasUsableImatrix = RuntimeSearchSpace.HasUsableImatrix(); + + return BaselineQuants.GetLearningBaselines(hasUsableImatrix) + .Concat(BaselineQuants.GetGroupCombinationCandidates(hasUsableImatrix, RuntimeSearchSpace.AllowHighPrecisionHybrids)) + .Concat(BaselineQuants.GetCombinationCarrierBaselines(hasUsableImatrix)) + .Where(x => !BaselineQuants.IsNativeExactAlias(x.UniqueId)) + .GroupBy(x => NormalizeAnchorKey(x.CanonicalKey), StringComparer.Ordinal) + .Select(g => g.OrderBy(x => x.UniqueId).First()) + .OrderBy(x => x.UniqueId) + .ToList(); + } + + private static BaselineQuants ChooseVirtualPredictionAnchorCarrier() + { + var activeCarriers = RuntimeSearchSpace.GetActiveCombinationBaselines() + .OrderBy(x => x.UniqueId) + .ToList(); + + if (activeCarriers.Count == 0) + return BaselineQuants.Q8_0; + + if (activeCarriers.Count == 1) + return activeCarriers[0]; + + var q8 = activeCarriers.FirstOrDefault(x => x.UniqueId == BaselineQuants.Q8_0.UniqueId); + if (q8 != null) + return q8; + + return activeCarriers + .OrderByDescending(x => x.BitRange) + .ThenByDescending(x => x.ExplicitCandidateSortOrder) + .ThenBy(x => x.UniqueId) + .First(); + } + + private static async Task VirtualAnchorRowExistsAsync( + DuckDBConnection connection, + TensorConfig config, + CancellationToken ct) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = $"SELECT COUNT(*) FROM {TableName} WHERE {BuildSlotPredicateSql(config)};"; + var value = await cmd.ExecuteScalarAsync(ct); + return ToInt64(value) > 0; + } + + private static async Task UpsertVirtualAnchorRowAsync( + DuckDBConnection connection, + TensorConfig config, + BaselineQuants baseline, + CancellationToken ct) + { + if (await VirtualAnchorRowExistsAsync(connection, config, ct)) + { + using var update = connection.CreateCommand(); + update.CommandText = $@" +UPDATE {TableName} +SET IsProtectedAnchor = TRUE, + IsVirtualPredictionAnchor = TRUE, + AnchorBaselineRuntimeId = {baseline.UniqueId}, + AnchorBaselineCanonicalKey = {SqlString(baseline.CanonicalKey)}, + AnchorDisplayName = {SqlString(baseline.Names.FirstOrDefault() ?? baseline.CanonicalKey)} +WHERE {BuildSlotPredicateSql(config)};"; + await update.ExecuteNonQueryAsync(ct); + return; + } + + using var insert = connection.CreateCommand(); + insert.CommandText = $@" +INSERT INTO {TableName} +({CombinationDuckDbSchema.SlotColumnList}, IsProtectedAnchor, IsVirtualPredictionAnchor, AnchorBaselineRuntimeId, AnchorBaselineCanonicalKey, AnchorDisplayName) +VALUES ({config.BaseQuant}, {config.Embeddings}, {config.LmHead}, {config.AttnQ}, {config.AttnKV}, {config.AttnOutput}, {config.FfnUpGate}, {config.FfnDown}, {config.MoeExperts}, {config.MoeRouter}, TRUE, TRUE, {baseline.UniqueId}, {SqlString(baseline.CanonicalKey)}, {SqlString(baseline.Names.FirstOrDefault() ?? baseline.CanonicalKey)});"; + await insert.ExecuteNonQueryAsync(ct); + } + + private static string BuildSlotPredicateSql(TensorConfig config) + { + return $"BaseQuant = {config.BaseQuant} AND Embeddings = {config.Embeddings} AND LmHead = {config.LmHead} AND AttnQ = {config.AttnQ} AND AttnKV = {config.AttnKV} AND AttnOutput = {config.AttnOutput} AND FfnUpGate = {config.FfnUpGate} AND FfnDown = {config.FfnDown} AND MoeExperts = {config.MoeExperts} AND MoeRouter = {config.MoeRouter}"; + } + + private static string SqlString(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + return "NULL"; + + return $"'{value.Replace("'", "''")}'"; + } + + private static long ToInt64(object? value) + { + if (value is null || value is DBNull) + return 0L; + + if (value is BigInteger big) + return (long)big; + + return Convert.ToInt64(value); + } + + private static string NormalizeAnchorKey(string? value) => (value ?? string.Empty).Trim().ToLowerInvariant(); + + private sealed class VirtualAnchorInsertStats + { + public int InsertedRows { get; init; } + public int MarkedExistingRows { get; init; } + } + + private async Task BulkAppendAsync( + DuckDBConnection connection, + IReadOnlyCollection rows, + string label, + CancellationToken ct) + { + // Emergency/small debug use only. Do NOT use for full search-space generation or trillion-scale pruning. + // Insert only the ten tensor slot columns; DuckDB prediction columns intentionally remain NULL + // until DuckDbPredictionMaterializationService scores/ranks the transient search space. + if (rows.Count == 0) + return; + + await ConfigureFastLoadSessionAsync(connection, ct); + + using var tx = connection.BeginTransaction(); + using var insert = connection.CreateCommand(); + insert.CommandText = $@" +INSERT INTO {TableName} ({CombinationDuckDbSchema.SlotColumnList}) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; + + foreach (var row in rows) + { + ct.ThrowIfCancellationRequested(); + + insert.Parameters.Clear(); + insert.Parameters.Add(new DuckDBParameter { Value = row.BaseQuant }); + insert.Parameters.Add(new DuckDBParameter { Value = row.Embeddings }); + insert.Parameters.Add(new DuckDBParameter { Value = row.LmHead }); + insert.Parameters.Add(new DuckDBParameter { Value = row.AttnQ }); + insert.Parameters.Add(new DuckDBParameter { Value = row.AttnKV }); + insert.Parameters.Add(new DuckDBParameter { Value = row.AttnOutput }); + insert.Parameters.Add(new DuckDBParameter { Value = row.FfnUpGate }); + insert.Parameters.Add(new DuckDBParameter { Value = row.FfnDown }); + insert.Parameters.Add(new DuckDBParameter { Value = row.MoeExperts }); + insert.Parameters.Add(new DuckDBParameter { Value = row.MoeRouter }); + + await insert.ExecuteNonQueryAsync(ct); + } + + tx.Commit(); + } + + private async Task BuildPredictionContextAsync( + RequiredSampleGenerationResult fullPlan, + CancellationToken ct) + { + await using var db = new MagicQuantContext(); + + var model = await db.AiModelHashes + .FirstOrDefaultAsync(x => x.UniqueHash == Cache.CurrentModelId, ct); + + if (model == null) + return null; + + var imatrixDefinitionId = await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, model.Id, createIfMissing: false, ct); + + var pureQ8 = await LoadSnapshotByQuantAsync( + db, + model.Id, + imatrixDefinitionId, + HybridQuant.CreatePureBaseline(BaselineQuants.Q8_0), + ct); + + if (pureQ8 == null) + return null; + + var activeCombinationBaselines = RuntimeSearchSpace.GetActiveCombinationBaselines().ToList(); + + var carrierBaseOnlyPlans = fullPlan.Plans + .Where(x => x.Kind == RequiredSampleKind.BaseOnlyIsolation) + .Where(x => x.TestedBaselineId.HasValue) + .Where(x => x.Key.StartsWith("carrier-baseonly:", StringComparison.Ordinal) || + x.Key.StartsWith("baseonly:", StringComparison.Ordinal)) + .Where(x => + { + var baseline = BaselineQuants.FromId(x.TestedBaselineId!.Value); + return baseline.IsCombinationCarrierCandidate; + }) + .GroupBy(x => x.TestedBaselineId!.Value) + .Select(g => g.First()) + .ToList(); + + var loadedCarrierSnapshots = new List<(byte BaselineId, BenchmarkRow Snapshot)>(); + foreach (var plan in carrierBaseOnlyPlans) + { + var snap = await LoadSnapshotByQuantAsync(db, model.Id, imatrixDefinitionId, plan.Quant, ct); + if (snap != null) + loadedCarrierSnapshots.Add((plan.TestedBaselineId!.Value, snap)); + } + + ulong representativeCarrierBaseOnlySize = loadedCarrierSnapshots.Count > 0 + ? loadedCarrierSnapshots + .OrderByDescending(x => BaselineQuants.FromId(x.BaselineId).BitRange) + .ThenByDescending(x => BaselineQuants.FromId(x.BaselineId).ExplicitCandidateSortOrder) + .Select(x => x.Snapshot.SizeBytes) + .First() + : pureQ8.SizeBytes; + + bool carrierBaseOnlyTruthCollapsed = loadedCarrierSnapshots.Count > 1 && + loadedCarrierSnapshots + .Select(x => x.Snapshot.SizeBytes) + .Distinct() + .Count() == 1; + + string? skipPureQ8PruneReason = null; + if (carrierBaseOnlyTruthCollapsed && activeCombinationBaselines.Count == 1 && Config.ManualMaxPredictedSizeBytes <= 0) + { + var safeCarrier = activeCombinationBaselines[0]; + skipPureQ8PruneReason = + $"Skipped pure-Q8 size pruning because base-carrier isolation truth collapsed across carriers and the search already resolved to the single deterministic safe carrier '{safeCarrier.Names[0]}'."; + } + + var pureBaselineSizes = new Dictionary(); + foreach (var baseline in RuntimeSearchSpace.GetActiveCombinationBaselines()) + { + var snap = await LoadSnapshotByQuantAsync(db, model.Id, imatrixDefinitionId, HybridQuant.CreatePureBaseline(baseline), ct); + if (snap != null) + pureBaselineSizes[baseline.UniqueId] = snap.SizeBytes; + } + + pureBaselineSizes[BaselineQuants.Q8_0.UniqueId] = pureQ8.SizeBytes; + + var sizeByGroupAndCandidate = new Dictionary<(byte GroupId, byte CandidateId), ulong>(); + + var groupPlans = fullPlan.Plans + .Where(x => x.Kind == RequiredSampleKind.GroupIsolationProbe || x.Kind == RequiredSampleKind.GroupIsolationContinuation) + .Where(x => x.TestedBaselineId == BaselineQuants.Q8_0.UniqueId) + .ToList(); + + foreach (var plan in groupPlans) + { + if (!plan.TargetGroupId.HasValue || !plan.TestedCandidateId.HasValue) + continue; + + var snap = await LoadSnapshotByQuantAsync(db, model.Id, imatrixDefinitionId, plan.Quant, ct); + if (snap == null) + continue; + + sizeByGroupAndCandidate[(plan.TargetGroupId!.Value, plan.TestedCandidateId!.Value)] = snap.SizeBytes; + } + + return new PredictionContext( + pureQ8BaseSize: pureQ8.SizeBytes, + pureBaselineSizes: pureBaselineSizes, + carrierBaseOnlySize: representativeCarrierBaseOnlySize, + sizesByGroupAndCandidate: sizeByGroupAndCandidate, + shouldSkipPureQ8CeilingPruning: !string.IsNullOrWhiteSpace(skipPureQ8PruneReason), + skipPureQ8CeilingReason: skipPureQ8PruneReason); + } + + private static async Task LoadSnapshotByQuantAsync( + MagicQuantContext db, + uint modelId, + int? imatrixDefinitionId, + HybridQuant quant, + CancellationToken ct) + { + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + var lookup = (TensorConfig)quant; + + var row = await db.AiBenchmarks + .Join(db.TensorCombos, + b => b.TensorComboId, + c => c.Id, + (b, c) => new { b, c }) + .FirstOrDefaultAsync(x => + x.b.ArchitectureFamilyId == architectureFamilyId && + x.b.TensorGroupProfileId == tensorGroupProfileId && + x.b.AiModelHashId == modelId && + x.b.ImatrixDefinitionId == imatrixDefinitionId && + x.c.BaseQuant == lookup.BaseQuant && + x.c.Embeddings == lookup.Embeddings && + x.c.LmHead == lookup.LmHead && + x.c.AttnQ == lookup.AttnQ && + x.c.AttnKV == lookup.AttnKV && + x.c.AttnOutput == lookup.AttnOutput && + x.c.FfnUpGate == lookup.FfnUpGate && + x.c.FfnDown == lookup.FfnDown && + x.c.MoeExperts == lookup.MoeExperts && + x.c.MoeRouter == lookup.MoeRouter, + ct); + + if (row == null) + return null; + + return new BenchmarkRow { SizeBytes = row.b.SizeBytes }; + } + + private static async Task InsertBaselineCombinationsSqlAsync(DuckDBConnection connection, BaselineQuants baseline, CancellationToken ct) + { + var allowed = ComboLogic.GetAllowedCandidateIdsPerGroup(baseline); + if (allowed.Length != 9 || allowed.Any(x => x == null || x.Length == 0)) + throw new InvalidOperationException($"Invalid allowed candidate dimensions for baseline {baseline.Names[0]}."); + await CreateTempDimensionTableAsync(connection, "temp_dim_embeddings", allowed[0], ct); + await CreateTempDimensionTableAsync(connection, "temp_dim_lm_head", allowed[1], ct); + await CreateTempDimensionTableAsync(connection, "temp_dim_attn_q", allowed[2], ct); + await CreateTempDimensionTableAsync(connection, "temp_dim_attn_kv", allowed[3], ct); + await CreateTempDimensionTableAsync(connection, "temp_dim_attn_output", allowed[4], ct); + await CreateTempDimensionTableAsync(connection, "temp_dim_ffn_up_gate", allowed[5], ct); + await CreateTempDimensionTableAsync(connection, "temp_dim_ffn_down", allowed[6], ct); + await CreateTempDimensionTableAsync(connection, "temp_dim_moe_experts", allowed[7], ct); + await CreateTempDimensionTableAsync(connection, "temp_dim_moe_router", allowed[8], ct); + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@" +INSERT INTO {TableName} (BaseQuant,Embeddings,LmHead,AttnQ,AttnKV,AttnOutput,FfnUpGate,FfnDown,MoeExperts,MoeRouter) +SELECT CAST({baseline.UniqueId} AS UTINYINT), e.v, lh.v, aq.v, akv.v, ao.v, fu.v, fd.v, me.v, mr.v +FROM temp_dim_embeddings e +CROSS JOIN temp_dim_lm_head lh +CROSS JOIN temp_dim_attn_q aq +CROSS JOIN temp_dim_attn_kv akv +CROSS JOIN temp_dim_attn_output ao +CROSS JOIN temp_dim_ffn_up_gate fu +CROSS JOIN temp_dim_ffn_down fd +CROSS JOIN temp_dim_moe_experts me +CROSS JOIN temp_dim_moe_router mr;"; + await cmd.ExecuteNonQueryAsync(ct); + return ProductOfDimensionLengths(allowed); + } + + private static async Task BuildPredictedSizeLookupTablesAsync(DuckDBConnection connection, PredictionContext predictionContext, CancellationToken ct) + { + using (var create = connection.CreateCommand()) + { + create.CommandText = @"DROP TABLE IF EXISTS temp_base_predicted_size; +DROP TABLE IF EXISTS temp_group_size_delta; +CREATE TEMP TABLE temp_base_predicted_size (BaseQuant UTINYINT, BaseSizeBytes UBIGINT); +CREATE TEMP TABLE temp_group_size_delta (BaseQuant UTINYINT, GroupName VARCHAR, StoredSlot UTINYINT, DeltaBytes BIGINT);"; + await create.ExecuteNonQueryAsync(ct); + } + string[] groupNames = ["Embeddings","LmHead","AttnQ","AttnKV","AttnOutput","FfnUpGate","FfnDown","MoeExperts","MoeRouter"]; + foreach (var baseline in RuntimeSearchSpace.GetActiveCombinationBaselines()) + { + using (var b = connection.CreateCommand()) + { + b.CommandText = $"INSERT INTO temp_base_predicted_size VALUES ({baseline.UniqueId}, {predictionContext.GetBaseSizeForSql(baseline.UniqueId)});"; + await b.ExecuteNonQueryAsync(ct); + } + var allowed = ComboLogic.GetAllowedCandidateIdsPerGroup(baseline); + for (int i = 0; i < groupNames.Length; i++) + foreach (byte slot in allowed[i]) + { + long delta = predictionContext.GetRelativeSizeDeltaForSql((byte)(i + 1), baseline.UniqueId, slot); + using var d = connection.CreateCommand(); + d.CommandText = $"INSERT INTO temp_group_size_delta VALUES ({baseline.UniqueId}, '{groupNames[i]}', {slot}, {delta});"; + await d.ExecuteNonQueryAsync(ct); + } + } + } + + private sealed class BenchmarkRow + { + public ulong SizeBytes { get; set; } + } + + private static string BuildValuesSql(IReadOnlyList values) => + $"(VALUES {string.Join(", ", values.Select(v => $"({v})"))}) AS t(v)"; + + private static async Task CreateTempDimensionTableAsync(DuckDBConnection connection, string tableName, IReadOnlyList values, CancellationToken ct) + { + if (values.Count == 0) + throw new InvalidOperationException($"Dimension {tableName} had zero candidates."); + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@"DROP TABLE IF EXISTS {tableName}; +CREATE TEMP TABLE {tableName} AS +SELECT CAST(v AS UTINYINT) AS v +FROM {BuildValuesSql(values)};"; + await cmd.ExecuteNonQueryAsync(ct); + } + + private static BigInteger ProductOfDimensionLengths(ImmutableArray allowed) + { + BigInteger product = BigInteger.One; + foreach (var dim in allowed) + product *= dim.Length; + return product; + } + + private sealed class PredictionContext + { + private readonly Dictionary _pureBaselineSizes; + private readonly Dictionary<(byte GroupId, byte CandidateId), ulong> _sizesByGroupAndCandidate; + + public ulong PureQ8BaseSize { get; } + public ulong CarrierBaseOnlySize { get; } + public bool ShouldSkipPureQ8CeilingPruning { get; } + public string? SkipPureQ8CeilingReason { get; } + + public PredictionContext( + ulong pureQ8BaseSize, + Dictionary pureBaselineSizes, + ulong carrierBaseOnlySize, + Dictionary<(byte GroupId, byte CandidateId), ulong> sizesByGroupAndCandidate, + bool shouldSkipPureQ8CeilingPruning, + string? skipPureQ8CeilingReason) + { + PureQ8BaseSize = pureQ8BaseSize; + CarrierBaseOnlySize = carrierBaseOnlySize; + ShouldSkipPureQ8CeilingPruning = shouldSkipPureQ8CeilingPruning; + SkipPureQ8CeilingReason = skipPureQ8CeilingReason; + _pureBaselineSizes = pureBaselineSizes; + _sizesByGroupAndCandidate = sizesByGroupAndCandidate; + } + + public ulong Predict(TensorConfig config) + { + long total = (long)GetBaseSizeForSql(config.BaseQuant); + total += GetRelativeSizeDeltaForSql(TReg.Embeddings.UniqueId, config.BaseQuant, config.Embeddings); + total += GetRelativeSizeDeltaForSql(TReg.LmHead.UniqueId, config.BaseQuant, config.LmHead); + total += GetRelativeSizeDeltaForSql(TReg.AttnQ.UniqueId, config.BaseQuant, config.AttnQ); + total += GetRelativeSizeDeltaForSql(TReg.AttnKV.UniqueId, config.BaseQuant, config.AttnKV); + total += GetRelativeSizeDeltaForSql(TReg.AttnOutput.UniqueId, config.BaseQuant, config.AttnOutput); + total += GetRelativeSizeDeltaForSql(TReg.FfnUpGate.UniqueId, config.BaseQuant, config.FfnUpGate); + total += GetRelativeSizeDeltaForSql(TReg.FfnDown.UniqueId, config.BaseQuant, config.FfnDown); + total += GetRelativeSizeDeltaForSql(TReg.MoeExperts.UniqueId, config.BaseQuant, config.MoeExperts); + total += GetRelativeSizeDeltaForSql(TReg.MoeRouter.UniqueId, config.BaseQuant, config.MoeRouter); + + if (total < 0) + total = 0; + + return (ulong)total; + } + + public ulong GetBaseSizeForSql(byte baseQuant) + { + if (_pureBaselineSizes.TryGetValue(baseQuant, out var directBase)) + return directBase; + + if (TryGetDisabledSurrogateBaselineId(baseQuant, out var disabledSurrogateId) && + _pureBaselineSizes.ContainsKey(disabledSurrogateId)) + { + /* + * Deprecated surrogate fallback, intentionally disabled: + * + * return _pureBaselineSizes[disabledSurrogateId]; + * + * This made external/custom carriers inherit standard-family base size in the + * SQL pre-pruning path. The RankSafe materializer now requires exact external + * base-only truth, and this older helper should fail the same way. + */ + throw new InvalidOperationException( + $"Missing exact pure/base size for external baseline {FormatBaselineForSql(baseQuant)} (id '{baseQuant}'), " + + $"but disabled surrogate {FormatBaselineForSql(disabledSurrogateId)} (id '{disabledSurrogateId}') exists. " + + "SQL size prediction fallback is disabled to prevent external/custom collapse."); + } + + throw new InvalidOperationException( + $"Missing pure/base size for baseline {FormatBaselineForSql(baseQuant)} (id '{baseQuant}'). " + + "SQL size prediction no longer falls back to Q8_0 because missing size truth should stop the run."); + } + + public long GetRelativeSizeDeltaForSql(byte groupId, byte baseQuant, byte storedSlot) + { + if (storedSlot == 0) + return 0; + + byte decoded = BaselineQuants.DecodeTensorConfigGroupSlotToBaselineId(storedSlot); + if (BaselineQuants.IsNativeExactAlias(decoded)) + return 0; + + if (decoded == baseQuant) + return 0; + + ulong candidateSize = GetExactGroupSizeOrThrow(groupId, decoded, "candidate"); + ulong baseSize = GetExactGroupSizeOrThrow(groupId, baseQuant, "base"); + return (long)candidateSize - (long)baseSize; + } + + private ulong GetExactGroupSizeOrThrow(byte groupId, byte baselineId, string role) + { + if (_sizesByGroupAndCandidate.TryGetValue((groupId, baselineId), out var exactSize)) + return exactSize; + + if (TryGetDisabledSurrogateBaselineId(baselineId, out var disabledSurrogateId) && + _sizesByGroupAndCandidate.ContainsKey((groupId, disabledSurrogateId))) + { + /* + * Deprecated surrogate fallback, intentionally disabled: + * + * return _sizesByGroupAndCandidate[(groupId, disabledSurrogateId)]; + * + * Group-size deltas must be based on the exact runtime baseline id. Re-enabling + * this would collapse external/custom group assignments into their standard + * family before DuckDB ranking ever sees them. + */ + throw new InvalidOperationException( + $"Missing exact group-size isolation for {role} baseline {FormatBaselineForSql(baselineId)} (id '{baselineId}') " + + $"in tensor group id '{groupId}', but disabled surrogate {FormatBaselineForSql(disabledSurrogateId)} (id '{disabledSurrogateId}') exists. " + + "Regenerate the exact isolated sample instead of using SQL size fallback."); + } + + throw new InvalidOperationException( + $"Missing group-size isolation for {role} baseline {FormatBaselineForSql(baselineId)} (id '{baselineId}') in tensor group id '{groupId}'. " + + "SQL size prediction no longer returns zero for missing isolation truth."); + } + + private static bool TryGetDisabledSurrogateBaselineId(byte baselineId, out byte surrogateBaselineId) + { + surrogateBaselineId = baselineId; + + if (BaselineQuants.IsNativeExactAlias(baselineId)) + return false; + + var baseline = BaselineQuants.FromId(baselineId); + if (!baseline.IsExternalRepositoryBaseline) + return false; + + var builtIn = BaselineQuants.ResolveBuiltInStandardBaseline(baseline.QuantizeBaseArgumentName) + ?? BaselineQuants.ResolveBuiltInStandardBaseline(baseline.Names[0]); + + if (builtIn == null || builtIn.UniqueId == baselineId) + return false; + + surrogateBaselineId = builtIn.UniqueId; + return true; + } + + private static string FormatBaselineForSql(byte baselineId) + { + try + { + return BaselineQuants.FromId(baselineId).Names[0]; + } + catch + { + return $"id {baselineId}"; + } + } + } +} \ No newline at end of file diff --git a/src/MagicQuant/Services/QuantFidelityComparerService.cs b/src/MagicQuant/Services/QuantFidelityComparerService.cs new file mode 100644 index 0000000..5f3d76c --- /dev/null +++ b/src/MagicQuant/Services/QuantFidelityComparerService.cs @@ -0,0 +1,484 @@ +using MagicQuant.Models; +using MQ.DB; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; + +namespace MagicQuant.Services; + +public sealed class QuantFidelityComparerService +{ + private static readonly TensorGroup[] OrderedGroups = + [ + TReg.Embeddings, + TReg.LmHead, + TReg.AttnQ, + TReg.AttnKV, + TReg.AttnOutput, + TReg.FfnUpGate, + TReg.FfnDown, + TReg.MoeExperts, + TReg.MoeRouter + ]; + + public IReadOnlyList ActiveGroups => OrderedGroups + .Where(g => !Cache.UnusedTensorGroups.Any(u => u.UniqueId == g.UniqueId)) + .OrderBy(g => g.UniqueId) + .ToList(); + + public IReadOnlyList InactiveGroups => OrderedGroups + .Where(g => Cache.UnusedTensorGroups.Any(u => u.UniqueId == g.UniqueId)) + .OrderBy(g => g.UniqueId) + .ToList(); + + public AnomalyMovementAnalysis Analyze(TensorConfig reference, TensorConfig candidate) + { + var changed = new List(); + int upgrades = 0; + int downgrades = 0; + int same = 0; + int unknown = 0; + int lateral = 0; + int net = 0; + + foreach (var group in ActiveGroups) + { + byte referenceStored = GetStoredSlot(reference, group); + byte candidateStored = GetStoredSlot(candidate, group); + byte referenceQuant = EffectiveQuantId(reference, group); + byte candidateQuant = EffectiveQuantId(candidate, group); + var movement = Compare(referenceQuant, candidateQuant); + + switch (movement) + { + case QuantMovementKind.Upgrade: + upgrades++; + break; + case QuantMovementKind.Downgrade: + downgrades++; + break; + case QuantMovementKind.Same: + same++; + break; + case QuantMovementKind.LateralOrEquivalent: + lateral++; + break; + case QuantMovementKind.Unknown: + unknown++; + break; + } + + net += EffectiveTier(candidateQuant) - EffectiveTier(referenceQuant); + + if (movement != QuantMovementKind.Same || referenceStored != candidateStored) + { + changed.Add(new AnomalyChangedGroup + { + Group = group, + ReferenceQuantId = referenceQuant, + CandidateQuantId = candidateQuant, + ReferenceStoredSlot = referenceStored, + CandidateStoredSlot = candidateStored, + Movement = movement + }); + } + } + + AnomalyMovementClassification classification; + if (downgrades > 0 && upgrades == 0 && unknown == 0) + classification = AnomalyMovementClassification.MonotoneDowngrade; + else if (downgrades > 0 && upgrades > 0) + classification = AnomalyMovementClassification.MixedTrade; + else if (upgrades > 0 && downgrades == 0) + classification = AnomalyMovementClassification.MonotoneUpgrade; + else if (lateral > 0 && downgrades == 0 && upgrades == 0) + classification = AnomalyMovementClassification.LateralOrProviderEquivalent; + else if (changed.Count == 0) + classification = AnomalyMovementClassification.NoMovement; + else + classification = AnomalyMovementClassification.Unknown; + + return new AnomalyMovementAnalysis + { + Classification = classification, + ChangedGroups = changed, + UpgradeCount = upgrades, + DowngradeCount = downgrades, + SameCount = same, + UnknownCount = unknown, + LateralCount = lateral, + NetBitDelta = net + }; + } + + public QuantMovementKind Compare(byte referenceQuantId, byte candidateQuantId) + { + if (referenceQuantId == candidateQuantId) + return QuantMovementKind.Same; + + int referenceTier = EffectiveTier(referenceQuantId); + int candidateTier = EffectiveTier(candidateQuantId); + + if (referenceTier < 0 || candidateTier < 0) + return QuantMovementKind.Unknown; + + if (candidateTier == referenceTier) + return QuantMovementKind.LateralOrEquivalent; + + return candidateTier < referenceTier + ? QuantMovementKind.Downgrade + : QuantMovementKind.Upgrade; + } + + /// + /// Builds an explicit contextual quantized blanket: base=referenceQuantId and every + /// active tensor group is explicitly stored as that same learned quant. Inactive + /// groups remain NULL so dense/MoE architecture differences are preserved. + /// + /// This is the anomaly-world equivalent of the old exact blanket, except it is + /// intentionally quantized context, not BF16/F16/native isolation truth. + /// + public TensorConfig CreateActivatedContextBlanket(byte referenceQuantId) + { + if (BaselineQuants.IsNativeExactAlias(referenceQuantId)) + { + throw new InvalidOperationException( + $"SkippedInvalidContextualAnomalyProbe: reason=BF16ExactIsolationSample referenceQuant={SafeName(referenceQuantId)}"); + } + + byte stored = BaselineQuants.EncodeTensorConfigGroupSlotBaselineId(referenceQuantId); + var config = new TensorConfig( + baseQuant: referenceQuantId, + embeddings: BaselineQuants.TensorConfigNullSlotValue, + lmHead: BaselineQuants.TensorConfigNullSlotValue, + attnQ: BaselineQuants.TensorConfigNullSlotValue, + attnKV: BaselineQuants.TensorConfigNullSlotValue, + attnOutput: BaselineQuants.TensorConfigNullSlotValue, + ffnUpGate: BaselineQuants.TensorConfigNullSlotValue, + ffnDown: BaselineQuants.TensorConfigNullSlotValue, + moeExperts: BaselineQuants.TensorConfigNullSlotValue, + moeRouter: BaselineQuants.TensorConfigNullSlotValue); + + foreach (var group in ActiveGroups.OrderBy(g => g.UniqueId)) + config = WithStoredSlot(config, group, stored); + + return config; + } + + /// + /// Builds the contextual higher-bit twin for anomaly detection. + /// + /// This is intentionally NOT the BF16/exact isolation reference used by normal + /// tensor-group learning. For anomaly smoke/probes, the reference is an explicit + /// activated quantized context: base=Q8 means every active group is explicitly Q8; + /// base=Q6 means every active group is explicitly Q6; and so on. + /// + public TensorConfig BuildBaseContextTwin(TensorConfig candidate) => CreateActivatedContextBlanket(candidate.BaseQuant); + + /// + /// Converts a normal generated DuckDB row into the explicit anomaly context shape. + /// This is allowed for prediction-space smoke only: sparse generated rows are not + /// treated as historical truth and are never persisted as anomaly probes/rules. + /// + public bool TryNormalizeSparseDuckRowToActivatedContext( + TensorConfig source, + out TensorConfig activated, + out bool hadSparseActiveGroups, + out string reason) + { + activated = default; + hadSparseActiveGroups = false; + + if (BaselineQuants.IsNativeExactAlias(source.BaseQuant)) + { + reason = $"BF16ExactIsolationSample: base={SafeName(source.BaseQuant)}"; + return false; + } + + if (HasIsolationDisplayMarker(source)) + { + reason = $"BF16ExactIsolationSample: displayName={HybridBenchmarkRepository.BuildDisplayName((HybridQuant)source)}"; + return false; + } + + TensorConfig result; + try + { + result = CreateActivatedContextBlanket(source.BaseQuant); + } + catch (Exception ex) + { + reason = ex.Message; + return false; + } + + foreach (var group in ActiveGroups) + { + byte stored = GetStoredSlot(source, group); + if (BaselineQuants.IsNullTensorConfigGroupSlot(stored)) + { + hadSparseActiveGroups = true; + continue; + } + + byte quantId = BaselineQuants.DecodeTensorConfigGroupSlotToBaselineId(stored); + if (BaselineQuants.IsNativeExactAlias(quantId)) + { + reason = $"BF16ExactIsolationSample: {group.Name}={SafeName(quantId)}"; + return false; + } + + result = WithStoredSlot(result, group, stored); + } + + foreach (var group in InactiveGroups) + { + if (!BaselineQuants.IsNullTensorConfigGroupSlot(GetStoredSlot(result, group))) + { + result = WithStoredSlot(result, group, BaselineQuants.TensorConfigNullSlotValue); + } + } + + if (!TryValidateContextualAnomalyConfig(result, out reason)) + return false; + + activated = result; + reason = hadSparseActiveGroups + ? "Sparse DuckDB prediction row normalized into explicit activated contextual anomaly vector." + : string.Empty; + return true; + } + + public bool IsNativeExactQuantId(byte quantId) => BaselineQuants.IsNativeExactAlias(quantId); + + public bool IsContextualQuantizedConfig(TensorConfig config) => TryValidateContextualAnomalyConfig(config, out _); + + public bool TryValidateContextualAnomalyConfig(TensorConfig config, out string reason) + { + if (BaselineQuants.IsNativeExactAlias(config.BaseQuant)) + { + reason = $"BF16ExactIsolationSample: base={SafeName(config.BaseQuant)}"; + return false; + } + + if (HasIsolationDisplayMarker(config)) + { + reason = $"BF16ExactIsolationSample: display name contains BF16/F16/native/exact marker ({HybridBenchmarkRepository.BuildDisplayName((HybridQuant)config)})"; + return false; + } + + foreach (var group in ActiveGroups) + { + byte stored = GetStoredSlot(config, group); + if (BaselineQuants.IsNullTensorConfigGroupSlot(stored)) + { + reason = $"SparseActiveGroup: group={group.Name} shortCode={group.ShortCode}"; + return false; + } + + byte quantId = BaselineQuants.DecodeTensorConfigGroupSlotToBaselineId(stored); + if (BaselineQuants.IsNativeExactAlias(quantId)) + { + reason = $"BF16ExactIsolationSample: {group.Name}={SafeName(quantId)}"; + return false; + } + } + + foreach (var group in InactiveGroups) + { + byte stored = GetStoredSlot(config, group); + if (!BaselineQuants.IsNullTensorConfigGroupSlot(stored)) + { + byte quantId = BaselineQuants.DecodeTensorConfigGroupSlotToBaselineId(stored); + if (BaselineQuants.IsNativeExactAlias(quantId)) + { + reason = $"BF16ExactIsolationSample: inactive {group.Name}={SafeName(quantId)}"; + return false; + } + } + } + + reason = string.Empty; + return true; + } + + public void EnsureAllActiveGroupsExplicit(TensorConfig config, string purpose) + { + if (TryValidateContextualAnomalyConfig(config, out _)) + return; + + TryValidateContextualAnomalyConfig(config, out var reason); + throw new InvalidOperationException( + $"SkippedInvalidContextualAnomalyProbe: purpose={purpose} reason={reason} config={TensorConfigIdentity.ToKey(config)} name={HybridBenchmarkRepository.BuildDisplayName((HybridQuant)config)}"); + } + + public bool TryDescribeNativeExactActiveState(TensorConfig config, out string reason) + { + if (BaselineQuants.IsNativeExactAlias(config.BaseQuant)) + { + reason = $"base={SafeName(config.BaseQuant)}"; + return true; + } + + foreach (var group in ActiveGroups) + { + byte stored = GetStoredSlot(config, group); + if (BaselineQuants.IsNullTensorConfigGroupSlot(stored)) + continue; + + byte effective = BaselineQuants.DecodeTensorConfigGroupSlotToBaselineId(stored); + if (BaselineQuants.IsNativeExactAlias(effective)) + { + reason = $"{group.ShortCode}={SafeName(effective)}"; + return true; + } + } + + reason = string.Empty; + return false; + } + + public bool HasIsolationDisplayMarker(TensorConfig config) + { + string displayName = HybridBenchmarkRepository.BuildDisplayName((HybridQuant)config); + return displayName.Contains("-B16", StringComparison.OrdinalIgnoreCase) || + displayName.Contains("BF16", StringComparison.OrdinalIgnoreCase) || + displayName.Contains("F16", StringComparison.OrdinalIgnoreCase) || + displayName.Contains("NATIVE", StringComparison.OrdinalIgnoreCase) || + displayName.Contains("EXACT", StringComparison.OrdinalIgnoreCase); + } + + public bool IsContextualQuantizedRule(AnomalyInteractionRule rule) + { + if (BaselineQuants.IsNativeExactAlias(rule.ReferenceQuantId)) + return false; + + foreach (var state in rule.GroupStates) + { + if (BaselineQuants.IsNativeExactAlias(state.CandidateQuantId) || + BaselineQuants.IsNativeExactAlias(state.ReferenceQuantId)) + { + return false; + } + } + + return true; + } + + public byte EffectiveQuantId(TensorConfig config, TensorGroup group) + { + byte stored = GetStoredSlot(config, group); + return BaselineQuants.IsNullTensorConfigGroupSlot(stored) + ? config.BaseQuant + : BaselineQuants.DecodeTensorConfigGroupSlotToBaselineId(stored); + } + + public byte GetStoredSlot(TensorConfig config, TensorGroup group) + { + return group.UniqueId switch + { + var id when id == TReg.Embeddings.UniqueId => config.Embeddings, + var id when id == TReg.LmHead.UniqueId => config.LmHead, + var id when id == TReg.AttnQ.UniqueId => config.AttnQ, + var id when id == TReg.AttnKV.UniqueId => config.AttnKV, + var id when id == TReg.AttnOutput.UniqueId => config.AttnOutput, + var id when id == TReg.FfnUpGate.UniqueId => config.FfnUpGate, + var id when id == TReg.FfnDown.UniqueId => config.FfnDown, + var id when id == TReg.MoeExperts.UniqueId => config.MoeExperts, + var id when id == TReg.MoeRouter.UniqueId => config.MoeRouter, + _ => throw new InvalidOperationException($"Unknown tensor group id '{group.UniqueId}'.") + }; + } + + public TensorConfig WithStoredSlot(TensorConfig config, TensorGroup group, byte storedSlot) + { + return new TensorConfig( + config.BaseQuant, + group.UniqueId == TReg.Embeddings.UniqueId ? storedSlot : config.Embeddings, + group.UniqueId == TReg.LmHead.UniqueId ? storedSlot : config.LmHead, + group.UniqueId == TReg.AttnQ.UniqueId ? storedSlot : config.AttnQ, + group.UniqueId == TReg.AttnKV.UniqueId ? storedSlot : config.AttnKV, + group.UniqueId == TReg.AttnOutput.UniqueId ? storedSlot : config.AttnOutput, + group.UniqueId == TReg.FfnUpGate.UniqueId ? storedSlot : config.FfnUpGate, + group.UniqueId == TReg.FfnDown.UniqueId ? storedSlot : config.FfnDown, + group.UniqueId == TReg.MoeExperts.UniqueId ? storedSlot : config.MoeExperts, + group.UniqueId == TReg.MoeRouter.UniqueId ? storedSlot : config.MoeRouter); + } + + public string BuildChangedGroupHash(IEnumerable groups) + { + string key = string.Join("|", groups + .OrderBy(x => x.Group.UniqueId) + .Select(x => $"{x.Group.UniqueId}:{x.ReferenceQuantId}->{x.CandidateQuantId}")); + + return TensorConfigIdentity.StableHash(key); + } + + public string DescribeGroups(IEnumerable groups) + { + return string.Join(" + ", groups + .OrderBy(x => x.Group.UniqueId) + .Select(x => $"{x.Group.ShortCode}={SafeName(x.CandidateQuantId)} from {SafeName(x.ReferenceQuantId)}")); + } + + public string ReferenceContextKey(TensorConfig reference) + { + return string.Join("|", ActiveGroups.Select(g => $"{g.UniqueId}:{EffectiveQuantId(reference, g)}")); + } + + public Dictionary BuildEffectiveGroupVector(TensorConfig config) + { + return ActiveGroups + .OrderBy(g => g.UniqueId) + .ToDictionary( + g => g.Name, + g => SafeName(EffectiveQuantId(config, g)), + StringComparer.Ordinal); + } + + public IReadOnlyList BuildInactiveGroupList() => InactiveGroups + .OrderBy(g => g.UniqueId) + .Select(g => g.Name) + .ToList(); + + public bool HasAllActiveGroupsExplicit(TensorConfig config) + { + foreach (var group in ActiveGroups) + { + if (BaselineQuants.IsNullTensorConfigGroupSlot(GetStoredSlot(config, group))) + return false; + } + + return true; + } + + public int EffectiveTier(byte quantId) + { + if (BaselineQuants.IsNativeExactAlias(quantId)) + return 160; + + var baseline = BaselineQuants.FromId(quantId); + string name = baseline.Names[0].ToUpperInvariant(); + + if (name.Contains("Q8") || baseline.BitRange >= 8) return 80; + if (name.Contains("Q6") || baseline.BitRange == 6) return 60; + if (name.Contains("Q5") || baseline.BitRange == 5) return 50; + if (name.Contains("Q4") || name.Contains("IQ4") || baseline.BitRange == 4) return 40; + if (name.Contains("Q3") || name.Contains("IQ3") || baseline.BitRange == 3) return 30; + if (name.Contains("Q2") || name.Contains("IQ2") || baseline.BitRange == 2) return 20; + if (name.Contains("Q1") || name.Contains("IQ1") || baseline.BitRange == 1) return 10; + + return baseline.BitRange > 0 ? baseline.BitRange * 10 : -1; + } + + private static string SafeName(byte quantId) + { + try + { + return BaselineQuants.FromId(quantId).Names[0]; + } + catch + { + return $"id:{quantId}"; + } + } +} diff --git a/src/MagicQuant/Services/QuantizationConcurrencyPlan.cs b/src/MagicQuant/Services/QuantizationConcurrencyPlan.cs new file mode 100644 index 0000000..4ddcd5e --- /dev/null +++ b/src/MagicQuant/Services/QuantizationConcurrencyPlan.cs @@ -0,0 +1,16 @@ +namespace MagicQuant.Services; + +/// CPU and storage writer limits, independent of model IO and benchmark execution. +public sealed record QuantizationConcurrencyPlan(int ReservedThreads, int UsableThreads, + int NaturalConcurrency, int Concurrency, int ThreadsPerProcess) +{ + public static QuantizationConcurrencyPlan Create(int threadCount, int scratchWriterCapacity) + { + const int minimumThreadsPerProcess = 4; + int reserved = threadCount switch { >= 8 => 2, >= 4 => 1, _ => 0 }; + int usable = Math.Max(1, threadCount - reserved); + int natural = Math.Max(1, usable / minimumThreadsPerProcess); + int concurrent = Math.Max(1, Math.Min(natural, scratchWriterCapacity)); + return new(reserved, usable, natural, concurrent, Math.Max(1, usable / concurrent)); + } +} diff --git a/src/MagicQuant/Services/QuantizationService.cs b/src/MagicQuant/Services/QuantizationService.cs new file mode 100644 index 0000000..df4cccd --- /dev/null +++ b/src/MagicQuant/Services/QuantizationService.cs @@ -0,0 +1,3178 @@ +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Runtime.InteropServices; +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; +using MagicQuant.Helpers; +using MagicQuant.Models.Learning; +using MagicQuant.Services.Learning; +using MagicQuant.Services.Progress; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; +using Microsoft.EntityFrameworkCore; +using Spectre.Console; + +namespace MagicQuant.Services; + +public enum SampleProcessState +{ + Completed = 1, + Skipped = 2, + Failed = 3 +} + +public sealed class SampleProcessingRecord +{ + public RequiredSamplePlan Plan { get; set; } = default!; + public SampleProcessState State { get; set; } + public string ModelName { get; set; } = string.Empty; + public Guid? TensorComboId { get; set; } + public Guid? BenchmarkId { get; set; } + public string? Error { get; set; } +} + +public sealed class SampleProcessingSummary +{ + public int Requested { get; set; } + public int Completed { get; set; } + public int Skipped { get; set; } + public int Failed { get; set; } + public List Records { get; set; } = new(); +} + +public class QuantizationService +{ + private readonly BenchmarkService _benchmarker; + private readonly ModelArtifactPathService _paths; + private readonly ScratchStorageService _scratchStorage; + private readonly PythonManager _python; + private readonly GgufMetadataReader _ggufMetadataReader; + private readonly SemaphoreSlim _cpuQuantLock; + private readonly int _quantThreadsPerProcess; + private readonly int _maxConcurrentQuantizations; + private readonly ImatrixService _imatrixService; + private readonly HuggingFaceBaselineService _huggingFaceBaselineService; + private readonly TensorGroupingAuditService _tensorGroupingAuditService; + private readonly TensorLearningDiagnosticWriter _tensorLearningDiagnosticWriter; + + private const byte UnknownTensorGroupId = 255; + + private static readonly Lazy> QuantAliasLookup = + new(BuildQuantAliasLookup, LazyThreadSafetyMode.ExecutionAndPublication); + + public QuantizationService(BenchmarkService benchmarker) + { + _benchmarker = benchmarker ?? throw new ArgumentNullException(nameof(benchmarker)); + _python = _benchmarker._pyManager; + _ggufMetadataReader = new GgufMetadataReader(_python); + + if (string.IsNullOrWhiteSpace(Cache.ModelMagicQuantDirectory)) + throw new Exception( + "Cache.ModelMagicQuantDirectory not set. The pipeline must set this before quantization starts."); + + if (string.IsNullOrWhiteSpace(Cache.ModelDirectory)) + throw new Exception("Cache.ModelDirectory not set. The pipeline must set this before quantization starts."); + + if (string.IsNullOrWhiteSpace(Cache.LlamaBin)) + throw new Exception("Cache.LlamaBin not set. Initialization must complete before quantization starts."); + + _paths = new ModelArtifactPathService(); + _scratchStorage = new ScratchStorageService(_paths); + _imatrixService = new ImatrixService(); + _huggingFaceBaselineService = new HuggingFaceBaselineService(_python); + _tensorGroupingAuditService = new TensorGroupingAuditService(); + _tensorLearningDiagnosticWriter = new TensorLearningDiagnosticWriter(); + + Directory.CreateDirectory(_paths.GgufDir); + Directory.CreateDirectory(_paths.BenchDir); + Directory.CreateDirectory(_paths.QuantizationLogsDir); + + int threadCount = Cache.SysInfo?.ThreadCount ?? Environment.ProcessorCount; + + int scratchWriterCapacity = _scratchStorage.WriterCapacity; + var cpuPlan = QuantizationConcurrencyPlan.Create(threadCount, scratchWriterCapacity); + int reservedThreads = cpuPlan.ReservedThreads; + int usableThreads = cpuPlan.UsableThreads; + int naturalConcurrentQuantizations = cpuPlan.NaturalConcurrency; + _maxConcurrentQuantizations = cpuPlan.Concurrency; + _quantThreadsPerProcess = cpuPlan.ThreadsPerProcess; + + _cpuQuantLock = new SemaphoreSlim( + _maxConcurrentQuantizations, + _maxConcurrentQuantizations); + + AnsiConsole.MarkupLine( + $"[grey]Quantization CPU plan:[/] " + + $"threads={threadCount}, reserved={reservedThreads}, usable={usableThreads}, " + + $"naturalConcurrent={naturalConcurrentQuantizations}, " + + $"scratchWriterCapacity={scratchWriterCapacity}, " + + $"concurrent={_maxConcurrentQuantizations}, " + + $"threads/process={_quantThreadsPerProcess}"); + } + + public static void ValidateQuantNameNormalizationOrThrow() + { + var collisions = TensorWeightScheme.All + .Where(x => !x.Names.IsDefaultOrEmpty) + .SelectMany(s => s.Names.Select(name => new + { + SchemeId = s.UniqueId, + Canonical = s.Names[0], + Alias = CanonicalizeQuantToken(name) + })) + .GroupBy(x => x.Alias, StringComparer.Ordinal) + .Where(g => g.Select(x => x.SchemeId).Distinct().Count() > 1) + .Select(g => $"{g.Key} => {string.Join(", ", g.Select(x => x.Canonical).Distinct(StringComparer.Ordinal))}") + .ToList(); + + if (collisions.Count > 0) + { + throw new InvalidOperationException( + "Quant alias registry has conflicting aliases across TensorWeightScheme definitions: " + + string.Join(" | ", collisions)); + } + } + + // ---------------------------------------------------------------- + // Batch processing + // ---------------------------------------------------------------- + + public Task ProcessHybridBatchAsync( + IReadOnlyCollection quants, + CancellationToken ct = default) + => ProcessHybridBatchAsync(quants, progressOptions: null, ct); + + public async Task ProcessHybridBatchAsync( + IReadOnlyCollection quants, + StageProgressOptions? progressOptions, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (quants == null) + throw new ArgumentNullException(nameof(quants)); + + var shimmedPlans = quants + .Select((quant, index) => new RequiredSamplePlan + { + Kind = RequiredSampleKind.GroupIsolationContinuation, + Key = $"legacy:{index}", + Description = "Legacy batch item", + Quant = quant + }) + .ToList(); + + return await ProcessHybridBatchAsync(shimmedPlans, progressOptions, ct); + } + + public Task ProcessHybridBatchAsync( + IReadOnlyCollection plans, + CancellationToken ct = default) + => ProcessHybridBatchAsync(plans, progressOptions: null, ct); + + public async Task ProcessHybridBatchAsync( + IReadOnlyCollection plans, + StageProgressOptions? progressOptions, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (plans == null) + throw new ArgumentNullException(nameof(plans)); + + if (plans.Count == 0) + { + return new SampleProcessingSummary + { + Requested = 0, + Records = new List() + }; + } + + var records = new ConcurrentBag(); + var stageProgress = progressOptions != null && progressOptions.Total > 0 + ? new StageProgressTracker(progressOptions) + : null; + + await EnsureBaseModelFileAsync(false); + + var learnableBaselinePlans = plans + .Where(p => IsLearnableBaselineRun(p.Quant)) + .OrderBy(p => p.Quant.BaseQuant.UniqueId) + .ToList(); + + var duplicateLearnableNames = learnableBaselinePlans + .Select(p => GenerateHybridName(p.Quant)) + .GroupBy(x => x, StringComparer.Ordinal) + .Where(g => g.Count() > 1) + .Select(g => g.Key) + .ToList(); + + if (duplicateLearnableNames.Count > 0) + { + throw new InvalidOperationException( + "Duplicate learnable baseline output names were queued in the same batch: " + + string.Join(", ", duplicateLearnableNames)); + } + + int baselineWorkerCount = CalculateBatchWorkerCount(learnableBaselinePlans.Count); + + await Parallel.ForEachAsync( + learnableBaselinePlans, + new ParallelOptions + { + MaxDegreeOfParallelism = baselineWorkerCount, + CancellationToken = ct + }, + async (baselinePlan, token) => + { + records.Add(await ExecutePlanAsync( + baselinePlan, + stageProgress, + allowIndependentGpuTopology: learnableBaselinePlans.Count > 1, + ct: token)); + }); + + var remainingPlans = plans.Except(learnableBaselinePlans).ToList(); + var equivalenceMap = await BuildIsolationDeduplicationPlanAsync(remainingPlans, ct); + + var planByKey = remainingPlans.ToDictionary(p => p.Key, StringComparer.Ordinal); + var primaryGroups = new List<(RequiredSamplePlan Source, List Duplicates)>(); + + foreach (var plan in remainingPlans) + { + ct.ThrowIfCancellationRequested(); + + if (!equivalenceMap.TryGetValue(plan.Key, out var sourceKey) || + string.IsNullOrWhiteSpace(sourceKey) || + string.Equals(sourceKey, plan.Key, StringComparison.Ordinal)) + { + primaryGroups.Add((plan, new List())); + continue; + } + + if (!planByKey.TryGetValue(sourceKey, out _)) + { + primaryGroups.Add((plan, new List())); + } + } + + var groupBySource = primaryGroups.ToDictionary(g => g.Source.Key, g => g, StringComparer.Ordinal); + foreach (var plan in remainingPlans) + { + if (!equivalenceMap.TryGetValue(plan.Key, out var sourceKey) || + string.IsNullOrWhiteSpace(sourceKey) || + string.Equals(sourceKey, plan.Key, StringComparison.Ordinal)) + { + continue; + } + + if (groupBySource.TryGetValue(sourceKey, out var group)) + group.Duplicates.Add(plan); + } + + int workerCount = CalculateBatchWorkerCount(primaryGroups.Count); + + await Parallel.ForEachAsync( + primaryGroups, + new ParallelOptions { MaxDegreeOfParallelism = workerCount, CancellationToken = ct }, + async (group, token) => + { + records.Add(await ExecutePlanAsync( + group.Source, + stageProgress, + allowIndependentGpuTopology: primaryGroups.Count > 1, + ct: token)); + + foreach (var duplicatePlan in group.Duplicates) + { + token.ThrowIfCancellationRequested(); + records.Add(await ExecuteDuplicatePlanAsync( + group.Source, + duplicatePlan, + stageProgress, + allowIndependentGpuTopology: primaryGroups.Count > 1, + ct: token)); + } + }); + + var finalRecords = records.OrderBy(x => x.Plan.Key, StringComparer.Ordinal).ToList(); + + return new SampleProcessingSummary + { + Requested = plans.Count, + Completed = finalRecords.Count(x => x.State == SampleProcessState.Completed), + Skipped = finalRecords.Count(x => x.State == SampleProcessState.Skipped), + Failed = finalRecords.Count(x => x.State == SampleProcessState.Failed), + Records = finalRecords + }; + } + + + private int CalculateBatchWorkerCount(int itemCount) + { + if (itemCount <= 0) + return 1; + + return Math.Max(1, Math.Min( + itemCount, + _maxConcurrentQuantizations + _benchmarker.CurrentParallelSlotCount)); + } + + + private async Task ExecutePlanAsync( + RequiredSamplePlan plan, + StageProgressTracker? progress, + bool allowIndependentGpuTopology, + CancellationToken ct) + { + var record = new SampleProcessingRecord + { + Plan = plan, + ModelName = GenerateHybridName(plan.Quant) + }; + var sw = Stopwatch.StartNew(); + + try + { + var state = await ProcessHybridQuantAsync( + plan.Quant, + allowIndependentGpuTopology, + ct); + sw.Stop(); + record.State = state; + + var identity = await ResolveBenchmarkIdentityAsync(plan.Quant, ct); + record.TensorComboId = identity.TensorComboId; + record.BenchmarkId = identity.BenchmarkId; + + progress?.ReportFinished(state, record.ModelName, sw.Elapsed); + return record; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + sw.Stop(); + record.State = SampleProcessState.Failed; + record.Error = ex.Message; + + AnsiConsole.MarkupLine($"[red]Sample failed:[/] {Markup.Escape(record.ModelName)}"); + AnsiConsole.MarkupLine($"[grey]{Markup.Escape(ex.Message)}[/]"); + + progress?.ReportFinished(SampleProcessState.Failed, record.ModelName, sw.Elapsed); + return record; + } + } + + private async Task ExecuteDuplicatePlanAsync( + RequiredSamplePlan sourcePlan, + RequiredSamplePlan duplicatePlan, + StageProgressTracker? progress, + bool allowIndependentGpuTopology, + CancellationToken ct) + { + var record = new SampleProcessingRecord + { + Plan = duplicatePlan, + ModelName = GenerateHybridName(duplicatePlan.Quant) + }; + var sw = Stopwatch.StartNew(); + + try + { + bool cloned = await CloneEquivalentIsolationBenchmarkAsync(sourcePlan, duplicatePlan, ct); + + if (cloned) + { + var identity = await ResolveBenchmarkIdentityAsync(duplicatePlan.Quant, ct); + record.State = SampleProcessState.Completed; + record.TensorComboId = identity.TensorComboId; + record.BenchmarkId = identity.BenchmarkId; + progress?.ReportFinished( + SampleProcessState.Completed, + record.ModelName, + duration: TimeSpan.Zero, + countForEtaOverride: false); + sw.Stop(); + return record; + } + + sw.Stop(); + return await ExecutePlanAsync( + duplicatePlan, + progress, + allowIndependentGpuTopology, + ct); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + sw.Stop(); + record.State = SampleProcessState.Failed; + record.Error = ex.Message; + + AnsiConsole.MarkupLine($"[red]Sample failed:[/] {Markup.Escape(record.ModelName)}"); + AnsiConsole.MarkupLine($"[grey]{Markup.Escape(ex.Message)}[/]"); + + progress?.ReportFinished(SampleProcessState.Failed, record.ModelName, sw.Elapsed); + return record; + } + } + + private async Task<(Guid? TensorComboId, Guid? BenchmarkId)> ResolveBenchmarkIdentityAsync( + HybridQuant quant, + CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(Cache.CurrentModelId)) + throw new InvalidOperationException("Cache.CurrentModelId is not set."); + + var lookup = BuildTensorLookup(quant); + + await using var db = new MagicQuantContext(); + + var scopedAiModelHashId = await ResolveCurrentScopedAiModelHashIdOrNullAsync(db, ct); + + if (scopedAiModelHashId == null) + return (null, null); + + var imatrixDefinitionId = + await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, scopedAiModelHashId.Value, + createIfMissing: false, ct); + + var comboId = await db.TensorCombos + .AsNoTracking() + .Where(x => + x.BaseQuant == lookup.BaseQuant && + x.Embeddings == lookup.Embeddings && + x.LmHead == lookup.LmHead && + x.AttnQ == lookup.AttnQ && + x.AttnKV == lookup.AttnKV && + x.AttnOutput == lookup.AttnOutput && + x.FfnUpGate == lookup.FfnUpGate && + x.FfnDown == lookup.FfnDown && + x.MoeExperts == lookup.MoeExperts && + x.MoeRouter == lookup.MoeRouter) + .Select(x => x.Id) + .FirstOrDefaultAsync(ct); + + if (comboId == Guid.Empty) + return (null, null); + + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + + var benchmarkId = await db.AiBenchmarks + .AsNoTracking() + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.AiModelHashId == scopedAiModelHashId.Value && + x.ImatrixDefinitionId == imatrixDefinitionId && + x.TensorComboId == comboId) + .Select(x => x.Id) + .FirstOrDefaultAsync(ct); + + return (comboId, benchmarkId == Guid.Empty ? null : benchmarkId); + } + + public async Task ProcessHybridQuantAsync( + HybridQuant quant, + bool allowIndependentGpuTopology = true, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + string modelName = GenerateHybridName(quant); + string modelBenchDir = _paths.GetBenchmarkDir(modelName); + string baseLogitsDir = GetBaseLogitsDirectory(); + + DateTime startedUtc = DateTime.UtcNow; + bool forceBaselineRelearn = ShouldForceBaselineRelearn(quant.BaseQuant); + bool pureExternalBaseline = ShouldDownloadExternalBaselineInsteadOfQuantizing(quant); + bool baselineLearnedTruthExists = await HasLearnedTruthForBaselineAsync(quant.BaseQuant, ct); + + if (!forceBaselineRelearn && baselineLearnedTruthExists && await _benchmarker.TryReuseExistingBenchmarksAsync( + quantConfig: quant, + modelPath: string.Empty, + benchDir: modelBenchDir, + klLogitsDir: baseLogitsDir, + domainsOverride: new[] { "general" })) + { + AnsiConsole.MarkupLine($"[grey]Reused existing benchmark artifacts:[/] {Markup.Escape(modelName)}"); + return SampleProcessState.Skipped; + } + + if (!forceBaselineRelearn && baselineLearnedTruthExists && await BenchmarkExistsAsync(quant, ct)) + { + AnsiConsole.MarkupLine($"[grey]Skipping already completed sample:[/] {Markup.Escape(modelName)}"); + return SampleProcessState.Skipped; + } + + string inputPath = await GetEffectiveInputModelPathAsync( + quant, + forceBaselineRelearn, + baselineLearnedTruthExists, + ct); + + string? disposableExternalBaselinePath = pureExternalBaseline && + IsPathInsideExternalBaselineCacheRoot(inputPath) + ? inputPath + : null; + + try + { + ScratchArtifactKind leaseKind = pureExternalBaseline + ? ScratchArtifactKind.ExternalBaselineRebuild + : quant.BaseQuant.IsExternalRepositoryBaseline + ? ScratchArtifactKind.ExternalBaselineNormalizedSample + : ScratchArtifactKind.QuantizedSample; + + await using var lease = await _scratchStorage.AcquireAsync(leaseKind, modelName, ct: ct); + string benchmarkModelPath = lease.GgufPath; + + try + { + QuantizationExecutionReport? quantizationReport = null; + PreparedExternalBaselineBuild? preparedExternalBaseline = null; + + await _cpuQuantLock.WaitAsync(ct); + try + { + if (pureExternalBaseline) + { + preparedExternalBaseline = await PrepareExternalBaselineRebuildAsync( + quant, + downloadedExternalBaselinePath: inputPath, + rebuiltOutputPath: lease.GgufPath, + logPath: lease.PrimaryLogPath, + metadataWorkingDirectory: lease.LeaseDirectory, + forceBaselineRelearn: forceBaselineRelearn, + ct: ct); + benchmarkModelPath = preparedExternalBaseline.BenchmarkModelPath; + } + else + { + var quantToExecute = quant.BaseQuant.IsExternalRepositoryBaseline + ? CreateEquivalentStandardCarrierQuantForExternalRebuild(quant) + : quant; + + IReadOnlyDictionary? temporaryCarrierOverrides = null; + + if (quant.BaseQuant.IsExternalRepositoryBaseline) + { + temporaryCarrierOverrides = TryLoadAllLearnedTensorMappings( + canonicalBaselineKey: quant.BaseQuant.CanonicalKey, + preferredSourceScheme: quant.BaseQuant.DefaultTensorScheme, + allowDominantFallback: true); + + if (temporaryCarrierOverrides.Count == 0) + { + throw new InvalidOperationException( + $"Missing blanket learned mapping for external/custom baseline '{quant.BaseQuant.Names[0]}'. " + + "External baseline hybrids require learned tensor mappings before sampling. " + + "Use targeted YAML relearn configuration to regenerate only the affected baseline/profile truth."); + } + } + + var effectiveInputPath = quant.BaseQuant.IsExternalRepositoryBaseline + ? await EnsureBaseModelFileAsync() + : inputPath; + + quantizationReport = await RunLlamaQuantizeAsync( + effectiveInputPath, + lease.GgufPath, + quantToExecute, + temporaryCarrierOverrides: temporaryCarrierOverrides, + logPath: lease.PrimaryLogPath, + metadataWorkingDirectory: lease.LeaseDirectory, + ct: ct); + } + } + finally + { + _cpuQuantLock.Release(); + } + + AnsiConsole.MarkupLine($"[yellow]Benchmarking:[/] {Markup.Escape(modelName)}"); + + await _benchmarker.RunAllBenchmarksAsync( + quantConfig: quant, + modelPath: benchmarkModelPath, + benchDir: modelBenchDir, + klLogitsDir: baseLogitsDir, + saveLogits: false, + domainsOverride: new[] { "general" }, + allowIndependentGpuTopology: allowIndependentGpuTopology); + + if (IsLearnableBaselineRun(quant)) + { + if (preparedExternalBaseline?.HasPreparedLearningTruth == true) + await PersistLearnedBaselineTensorMapFromPreparedAsync(quant, preparedExternalBaseline, ct); + else if (!baselineLearnedTruthExists || forceBaselineRelearn) + await LearnAndPersistBaselineTensorMapAsync(quant, benchmarkModelPath, quantizationReport, ct); + } + + await PersistQuantizationRunAsync( + quant: quant, + imatrixDefinitionId: null, + startedUtc: startedUtc, + completedUtc: DateTime.UtcNow, + succeeded: true, + outputModelPath: benchmarkModelPath, + error: null, + ct: ct); + + return SampleProcessState.Completed; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + try + { + await PersistQuantizationRunAsync( + quant: quant, + imatrixDefinitionId: null, + startedUtc: startedUtc, + completedUtc: DateTime.UtcNow, + succeeded: false, + outputModelPath: benchmarkModelPath, + error: ex.ToString(), + ct: ct); + } + catch + { + } + + throw; + } + } + finally + { + await TryCleanupExternalBaselineDownloadArtifactsAsync( + disposableExternalBaselinePath, + "after digestion and benchmarking"); + } + } + + private bool ShouldDownloadExternalBaselineInsteadOfQuantizing(HybridQuant quant) + => quant.BaseQuant.IsExternalRepositoryBaseline && quant.Tensors.Count == 0; + + private static bool ShouldForceBaselineRelearn(BaselineQuants baseline) + { + if (Config.Current.Learning.ForceRelearnArchitectureFamily) + return true; + + if (baseline.IsExternalRepositoryBaseline) + return Config.GetResolvedCustomBaseline(baseline.CanonicalKey)?.ForceRelearn == true; + + return (Config.Current.Learning.ForceRelearnStandardBaselines ?? []) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Select(x => BaselineQuants.ResolveBuiltInStandardBaseline(x.Trim())) + .Any(x => x?.UniqueId == baseline.UniqueId); + } + + private async Task GetEffectiveInputModelPathAsync( + HybridQuant quant, + bool forceRefresh, + bool baselineLearnedTruthExists, + CancellationToken ct) + { + string basePath = await EnsureBaseModelFileAsync(); + if (!quant.BaseQuant.IsExternalRepositoryBaseline) + return basePath; + + // Pure external baselines are downloaded so MagicQuant can learn their tensor truth. + // Any continuation / isolation / hybrid that uses that external baseline must rebuild + // from the native base GGUF instead of requantizing the staged external GGUF. + if (quant.Tensors.Count > 0) + return basePath; + + if (!forceRefresh && baselineLearnedTruthExists) + return basePath; + + string externalPath = GetExternalBaselineCachePath(quant.BaseQuant); + try + { + await _huggingFaceBaselineService.DownloadBaselineAsync(quant.BaseQuant, externalPath, forceRefresh, ct); + await ValidateExternalBaselineTensorParityOrThrow(basePath, externalPath); + return externalPath; + } + catch + { + await TryCleanupExternalBaselineDownloadArtifactsAsync( + externalPath, + "after failed download/validation"); + + throw; + } + } + + private string GetExternalBaselineCachePath(BaselineQuants baseline) + { + string root = _paths.ExternalBaselinesDir; + Directory.CreateDirectory(root); + return _paths.GetExternalBaselineDurablePath(baseline); + } + + private async Task ValidateExternalBaselineTensorParityOrThrow( + string baseModelPath, + string externalBaselinePath) + { + var baseMeta = await ReadTensorMetadataFromGgufAsync(baseModelPath, Path.GetDirectoryName(externalBaselinePath)!); + var externalMeta = + await ReadTensorMetadataFromGgufAsync(externalBaselinePath, Path.GetDirectoryName(externalBaselinePath)!); + + return ExternalBaselineTensorParity.ValidateOrThrow(baseMeta, externalMeta); + } + + private async Task HasLearnedTruthForBaselineAsync(BaselineQuants baseline, CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (baseline.UniqueId == BaselineQuants.NativeSourceUniqueId) + return await HasNativeSourceLearnedTruthAsync(ct); + + if (string.IsNullOrWhiteSpace(Cache.CurrentModelId)) + return false; + + await using var db = new MagicQuantContext(); + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + var baselineDefinition = await BaselineDefinitionResolver.ResolveRequiredDefinitionAsync(db, baseline, ct); + + var query = db.LearnedBaselineTensorQuants + .AsNoTracking() + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.BaselineQuantDefinitionId == baselineDefinition.Id); + + if (baseline.DefaultTensorScheme != null) + query = query.Where(x => x.TensorWeightSchemeId == baseline.DefaultTensorScheme.UniqueId); + + return await query.AnyAsync(ct); + } + + private async Task PrepareExternalBaselineRebuildAsync( + HybridQuant quant, + string downloadedExternalBaselinePath, + string rebuiltOutputPath, + string logPath, + string metadataWorkingDirectory, + bool forceBaselineRelearn, + CancellationToken ct) + { + if (!quant.BaseQuant.IsExternalRepositoryBaseline) + throw new InvalidOperationException( + "PrepareExternalBaselineRebuildAsync was called for a non-external baseline."); + + string nativeBasePath = await EnsureBaseModelFileAsync(); + bool canReuseLearnedTruth = !forceBaselineRelearn && await HasLearnedTruthForBaselineAsync(quant.BaseQuant, ct); + + if (canReuseLearnedTruth) + { + var blanket = TryLoadAllLearnedTensorMappings( + canonicalBaselineKey: quant.BaseQuant.CanonicalKey, + preferredSourceScheme: quant.BaseQuant.DefaultTensorScheme, + allowDominantFallback: false); + + if (blanket.Count == 0) + throw new InvalidOperationException( + $"Custom baseline '{quant.BaseQuant.Names[0]}' was marked as already learned, but no blanket learned tensor mapping could be loaded."); + + if (!File.Exists(rebuiltOutputPath) || forceBaselineRelearn) + { + AnsiConsole.MarkupLine( + $"[cyan]Rebuilding normalized custom baseline from learned truth:[/] {Markup.Escape(quant.BaseQuant.Names[0])}"); + await RunLlamaQuantizeAsync( + nativeBasePath, + rebuiltOutputPath, + quant, + blanket, + logPath: logPath, + metadataWorkingDirectory: metadataWorkingDirectory, + ct: ct); + } + + return new PreparedExternalBaselineBuild + { + BenchmarkModelPath = rebuiltOutputPath, + DownloadedExternalModelPath = downloadedExternalBaselinePath, + HasPreparedLearningTruth = false + }; + } + + AnsiConsole.MarkupLine( + $"[cyan]Learning external baseline truth from downloaded artifact:[/] {Markup.Escape(quant.BaseQuant.Names[0])}"); + var parity = await ValidateExternalBaselineTensorParityOrThrow(nativeBasePath, downloadedExternalBaselinePath); + + var ggufMetadata = parity.ExternalMetadata; + var ggufTruth = ggufMetadata.TensorTypes + .ToDictionary(x => x.Key, x => NormalizeQuantName(x.Value), StringComparer.Ordinal); + + if (ggufTruth.Count == 0) + throw new InvalidOperationException( + $"Downloaded external baseline '{quant.BaseQuant.Names[0]}' produced no readable GGUF tensor truth."); + + var truth = ggufTruth.ToDictionary( + x => x.Key, + x => new LearnedTensorTruth(x.Key, x.Value, LearningSource.GgufOnly), + StringComparer.Ordinal); + + foreach (string tensorName in parity.InheritedOptionalTensorNames) + { + if (!parity.NativeMetadata.TensorTypes.TryGetValue(tensorName, out string? nativeType) || + string.IsNullOrWhiteSpace(nativeType)) + { + throw new InvalidOperationException( + $"Native source did not provide tensor type metadata for omitted optional MTP tensor '{tensorName}'."); + } + + truth[tensorName] = new LearnedTensorTruth( + tensorName, + NormalizeQuantName(nativeType), + LearningSource.InheritedFromNative); + } + + if (parity.InheritedOptionalTensorNames.Count > 0) + { + AnsiConsole.MarkupLine( + $"[yellow]External baseline omits {parity.OmittedNextnLayerCount:N0} declared optional NextN/MTP layer(s) " + + $"({parity.InheritedOptionalTensorNames.Count:N0} tensors).[/] " + + "The normalized rebuild will inherit those optional tensors from the native source; model-trunk parity remains strict."); + } + + truth = truth + .OrderBy(x => x.Key, StringComparer.Ordinal) + .ToDictionary(x => x.Key, x => x.Value, StringComparer.Ordinal); + + var verification = new TensorTruthVerificationResult + { + TruthByTensor = truth + }; + var audit = _tensorGroupingAuditService.Audit(truth.Keys.ToList(), truth); + + if (audit.HasFatalIssues) + { + var diagnosticPath = await _tensorLearningDiagnosticWriter.WriteFailureAsync( + baselineName: quant.BaseQuant.Names[0], + schemeName: quant.BaseQuant.DefaultTensorScheme?.Names[0] ?? "external", + sourceKind: quant.BaseQuant.SourceKind.ToString(), + sourceRepository: quant.BaseQuant.SourceRepository, + sourceFileName: quant.BaseQuant.SourceFileName, + truthByTensor: truth, + audit: audit, + verification: verification, + ct: ct); + + AnsiConsole.MarkupLine( + $"[red]Tensor group learning failed.[/] See diagnostic log: [yellow]{Markup.Escape(diagnosticPath)}[/]"); + throw new InvalidOperationException( + $"Strict tensor-group learning validation failed for external baseline '{quant.BaseQuant.Names[0]}' " + + $"from '{quant.BaseQuant.SourceRepository}/{quant.BaseQuant.SourceFileName}'. " + + $"No normalized rebuilt baseline was produced and no learned tensor mappings were persisted. " + + $"Diagnostic log: {diagnosticPath}"); + } + + var normalizedOverrides = truth.ToDictionary( + x => x.Key, + x => NativePrecisionNormalization.NormalizeLearnedFinalQuantTypeForApplication(x.Value.FinalQuantType), + StringComparer.Ordinal); + + if (normalizedOverrides.Values.Any(string.IsNullOrWhiteSpace)) + throw new InvalidOperationException( + $"External baseline '{quant.BaseQuant.Names[0]}' produced one or more empty normalized tensor scheme names."); + + AnsiConsole.MarkupLine( + $"[cyan]Rebuilding normalized benchmark artifact for custom baseline:[/] {Markup.Escape(quant.BaseQuant.Names[0])}"); + await RunLlamaQuantizeAsync( + nativeBasePath, + rebuiltOutputPath, + quant, + normalizedOverrides, + logPath: logPath, + metadataWorkingDirectory: metadataWorkingDirectory, + ct: ct); + + return new PreparedExternalBaselineBuild + { + BenchmarkModelPath = rebuiltOutputPath, + DownloadedExternalModelPath = downloadedExternalBaselinePath, + TruthByTensor = truth, + GroupedByTensor = audit.GroupedByTensor, + AllTensorNamesInDownloadedArtifact = truth.Keys.ToList(), + AmbiguousGroupingRows = audit.Ambiguous, + UnresolvedTensorNames = audit.IllegalUnresolved.Select(x => x.TensorName).ToList(), + BaseQuantExceptionRows = audit.BaseQuantExceptions, + Verification = verification, + HasPreparedLearningTruth = true + }; + } + + private async Task PersistLearnedBaselineTensorMapFromPreparedAsync( + HybridQuant quant, + PreparedExternalBaselineBuild prepared, + CancellationToken ct) + { + if (!IsLearnableBaselineRun(quant) || !prepared.HasPreparedLearningTruth || prepared.TruthByTensor == null || + prepared.GroupedByTensor == null) + return; + + var tensorScheme = quant.BaseQuant.DefaultTensorScheme!; + var verification = prepared.Verification ?? new TensorTruthVerificationResult + { TruthByTensor = prepared.TruthByTensor }; + var audit = new TensorGroupingAuditResult + { + GroupedByTensor = prepared.GroupedByTensor, + Ambiguous = prepared.AmbiguousGroupingRows ?? [], + IllegalUnresolved = (prepared.UnresolvedTensorNames ?? []).Select(x => new TensorGroupingAuditIssue + { + TensorName = x, + IssueKind = "IllegalUnresolvedTensor" + }).ToList(), + BaseQuantExceptions = prepared.BaseQuantExceptionRows ?? [] + }; + + if (audit.HasFatalIssues || verification.HasFatalIssues) + { + var diagnosticPath = await _tensorLearningDiagnosticWriter.WriteFailureAsync( + baselineName: quant.BaseQuant.Names[0], + schemeName: tensorScheme.Names[0], + sourceKind: quant.BaseQuant.SourceKind.ToString(), + sourceRepository: quant.BaseQuant.SourceRepository, + sourceFileName: quant.BaseQuant.SourceFileName, + truthByTensor: prepared.TruthByTensor, + audit: audit, + verification: verification, + ct: ct); + + AnsiConsole.MarkupLine( + $"[red]Tensor group learning failed.[/] See diagnostic log: [yellow]{Markup.Escape(diagnosticPath)}[/]"); + throw new InvalidOperationException( + $"Strict tensor-group learning validation failed for external baseline '{quant.BaseQuant.Names[0]}' " + + $"from '{quant.BaseQuant.SourceRepository}/{quant.BaseQuant.SourceFileName}'. " + + $"Prepared external baseline learning truth was invalid. No learned tensor mappings were persisted. " + + $"Diagnostic log: {diagnosticPath}"); + } + + await using var db = new MagicQuantContext(); + + var scopedAiModelHashId = await ResolveCurrentScopedAiModelHashIdOrNullAsync(db, ct); + if (scopedAiModelHashId == null) + throw new InvalidOperationException( + "Unable to persist learned mappings because scoped AiModelHash row was not found."); + + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + var baselineDefinition = await BaselineDefinitionResolver.ResolveRequiredDefinitionAsync(db, quant.BaseQuant, ct); + + var combo = await db.TensorCombos + .AsNoTracking() + .FirstAsync(x => x.BaseQuant == quant.BaseQuant.UniqueId && + x.Embeddings == 0 && x.LmHead == 0 && x.AttnQ == 0 && x.AttnKV == 0 && + x.AttnOutput == 0 && x.FfnUpGate == 0 && x.FfnDown == 0 && x.MoeExperts == 0 && + x.MoeRouter == 0, ct); + + var benchmarkAiModelHashId = scopedAiModelHashId.Value; + var imatrixDefinitionId = + await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, benchmarkAiModelHashId, + createIfMissing: false, ct); + + var benchmarkId = await db.AiBenchmarks + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.AiModelHashId == benchmarkAiModelHashId && + x.ImatrixDefinitionId == imatrixDefinitionId && + x.TensorComboId == combo.Id) + .OrderByDescending(x => x.Id) + .Select(x => (Guid?)x.Id) + .FirstOrDefaultAsync(ct); + + if (!benchmarkId.HasValue) + throw new InvalidOperationException( + $"Unable to persist learned mappings because no AiBenchmark exists for rebuilt baseline '{quant.BaseQuant.Names[0]}'."); + + var rows = prepared.TruthByTensor + .OrderBy(x => x.Key, StringComparer.Ordinal) + .Select(kv => + { + var match = prepared.GroupedByTensor[kv.Key]; + + return new LearnedBaselineTensorQuant + { + Id = Guid.NewGuid(), + AiBenchmarkId = benchmarkId.Value, + ArchitectureFamilyId = architectureFamilyId, + TensorGroupProfileId = tensorGroupProfileId, + BaselineQuantDefinitionId = baselineDefinition.Id, + TensorComboId = combo.Id, + AiModelHashId = scopedAiModelHashId.Value, + BaselineQuantId = quant.BaseQuant.UniqueId, + TensorWeightSchemeId = tensorScheme.UniqueId, + TensorGroupId = match.PrimaryGroup?.UniqueId ?? UnknownTensorGroupId, + BaselineCanonicalKey = quant.BaseQuant.CanonicalKey, + BaselineSourceKind = quant.BaseQuant.SourceKind, + BaselineSourceRepository = quant.BaseQuant.SourceRepository, + BaselineSourceFileName = quant.BaseQuant.SourceFileName, + TensorName = kv.Key, + FinalQuantType = kv.Value.FinalQuantType + }; + }) + .ToList(); + + if (rows.Count == 0) + throw new InvalidOperationException( + $"Prepared learning truth for baseline '{quant.BaseQuant.Names[0]}' produced no persistable rows."); + + await db.LearnedBaselineTensorQuants + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.BaselineQuantDefinitionId == baselineDefinition.Id && + x.TensorWeightSchemeId == tensorScheme.UniqueId) + .ExecuteDeleteAsync(ct); + + db.LearnedBaselineTensorQuants.AddRange(rows); + await db.SaveChangesAsync(ct); + + await WriteLearningDiagnosticArtifactAsync( + baselineName: quant.BaseQuant.Names[0], + schemeName: tensorScheme.Names[0], + truthByTensor: prepared.TruthByTensor, + grouped: prepared.GroupedByTensor, + allTensorNamesInModel: prepared.AllTensorNamesInDownloadedArtifact ?? prepared.TruthByTensor.Keys.ToList(), + ambiguous: audit.Ambiguous, + unresolved: prepared.UnresolvedTensorNames ?? new List()); + + AnsiConsole.MarkupLine( + $"[green]Persisted rebuilt custom-baseline learning truth:[/] [cyan]{rows.Count:N0}[/] row(s) for [yellow]{Markup.Escape(quant.BaseQuant.Names[0])}[/]."); + } + + private async Task CleanupExternalBaselineDownloadArtifactsAsync(string? downloadedExternalBaselinePath) + { + if (string.IsNullOrWhiteSpace(downloadedExternalBaselinePath)) + return; + + string fullFile = Path.GetFullPath(downloadedExternalBaselinePath); + string? root = Cache.ExternalBaselineCacheDirectory; + + if (string.IsNullOrWhiteSpace(root)) + throw new InvalidOperationException("External baseline cache root is not configured."); + + string fullRoot = Path.GetFullPath(root); + if (!IsPathInside(fullFile, fullRoot)) + { + throw new InvalidOperationException( + $"Refusing to clean external baseline artifact outside configured cache root: {fullFile}"); + } + + string? stagingDir = Path.GetDirectoryName(fullFile); + + if (!string.IsNullOrWhiteSpace(stagingDir)) + { + string fullStagingDir = Path.GetFullPath(stagingDir); + + if (IsPathInside(fullStagingDir, fullRoot) && + !string.Equals(fullStagingDir, fullRoot, StringComparison.OrdinalIgnoreCase) && + Directory.Exists(fullStagingDir)) + { + await HardDeleteHelper.DeleteDirectoryIfExistsAsync(fullStagingDir); + return; + } + } + + await HardDeleteHelper.DeleteFileIfExistsAsync(fullFile); + await HardDeleteHelper.DeleteFileIfExistsAsync(fullFile + ".nativecheck"); + await HardDeleteHelper.DeleteFileIfExistsAsync(fullFile + ".externalcheck"); + } + + private async Task TryCleanupExternalBaselineDownloadArtifactsAsync(string? path, string reason) + { + if (string.IsNullOrWhiteSpace(path)) + return; + + try + { + await CleanupExternalBaselineDownloadArtifactsAsync(path); + AnsiConsole.MarkupLine( + $"[green]Cleaned disposable external baseline[/] [grey]({Markup.Escape(reason)}):[/] {Markup.Escape(path)}"); + } + catch (Exception cleanupEx) + { + AnsiConsole.MarkupLine( + $"[yellow]Warning:[/] failed to clean external baseline {Markup.Escape(reason)}: {Markup.Escape(cleanupEx.Message)}"); + } + } + + private bool IsPathInsideExternalBaselineCacheRoot(string path) + { + if (string.IsNullOrWhiteSpace(path) || string.IsNullOrWhiteSpace(Cache.ExternalBaselineCacheDirectory)) + return false; + + return IsPathInside(path, Cache.ExternalBaselineCacheDirectory); + } + + private static bool IsPathInside(string childPath, string parentPath) + { + var child = Path.GetFullPath(childPath).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + var parent = Path.GetFullPath(parentPath).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + return child.StartsWith(parent + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase); + } + + // ---------------------------------------------------------------- + // Benchmark/logit helpers + // ---------------------------------------------------------------- + + private string GetBaseLogitsDirectory() => _paths.GetBaseLogitsDirectory(); + + private async Task BenchmarkExistsAsync(HybridQuant quant, CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(Cache.CurrentModelId)) + throw new InvalidOperationException("Cache.CurrentModelId is not set."); + + var lookup = BuildTensorLookup(quant); + + await using var db = new MagicQuantContext(); + + var scopedAiModelHashId = await ResolveCurrentScopedAiModelHashIdOrNullAsync(db, ct); + + if (scopedAiModelHashId == null) + return false; + + var imatrixDefinitionId = + await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, scopedAiModelHashId.Value, + createIfMissing: false, ct); + + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + + var bench = await db.AiBenchmarks + .AsNoTracking() + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.AiModelHashId == scopedAiModelHashId.Value && + x.ImatrixDefinitionId == imatrixDefinitionId) + .Join( + db.TensorCombos.AsNoTracking(), + benchmark => benchmark.TensorComboId, + combo => combo.Id, + (benchmark, combo) => new { benchmark, combo }) + .Where(x => + x.combo.BaseQuant == lookup.BaseQuant && + x.combo.Embeddings == lookup.Embeddings && + x.combo.LmHead == lookup.LmHead && + x.combo.AttnQ == lookup.AttnQ && + x.combo.AttnKV == lookup.AttnKV && + x.combo.AttnOutput == lookup.AttnOutput && + x.combo.FfnUpGate == lookup.FfnUpGate && + x.combo.FfnDown == lookup.FfnDown && + x.combo.MoeExperts == lookup.MoeExperts && + x.combo.MoeRouter == lookup.MoeRouter) + .Select(x => x.benchmark.Id) + .FirstOrDefaultAsync(ct); + + if (bench == Guid.Empty) + return false; + + bool hasCategory = await db.Set() + .AsNoTracking() + .AnyAsync(x => x.AiBenchmarkId == bench, ct); + + return hasCategory; + } + + private static TensorConfig BuildTensorLookup(HybridQuant quant) + { + return (TensorConfig)quant; + } + + private static async Task ResolveCurrentScopedAiModelHashIdOrNullAsync(MagicQuantContext db, + CancellationToken ct) + { + return await ArchitectureFamilyService.ResolveScopedAiModelHashIdOrNullAsync(db, ct); + } + + private static async Task ResolveCurrentScopedAiModelHashIdAsync(MagicQuantContext db, CancellationToken ct) + { + return await ArchitectureFamilyService.ResolveScopedAiModelHashIdAsync(db, ct); + } + + private static async Task ResolveCurrentExactAiModelHashIdOrNullAsync(MagicQuantContext db, + CancellationToken ct) + { + return await ArchitectureFamilyService.ResolveExactCurrentAiModelHashIdOrNullAsync(db, ct); + } + + private static async Task ResolveCurrentExactAiModelHashIdAsync(MagicQuantContext db, CancellationToken ct) + { + return await ArchitectureFamilyService.ResolveExactCurrentAiModelHashIdAsync(db, ct); + } + + + private async Task PersistQuantizationRunAsync( + HybridQuant quant, + int? imatrixDefinitionId, + DateTime startedUtc, + DateTime completedUtc, + bool succeeded, + string? outputModelPath, + string? error, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (string.IsNullOrWhiteSpace(Cache.CurrentModelId)) + throw new InvalidOperationException("Cache.CurrentModelId is not set."); + + var lookup = BuildTensorLookup(quant); + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + + await using var db = new MagicQuantContext(); + + var aiModelHash = await db.AiModelHashes + .FirstOrDefaultAsync(x => x.UniqueHash == Cache.CurrentModelId, ct); + + if (aiModelHash == null) + { + aiModelHash = new AiModelHash + { + UniqueHash = Cache.CurrentModelId + }; + + db.AiModelHashes.Add(aiModelHash); + await db.SaveChangesAsync(ct); + } + + uint persistenceAiModelHashId = aiModelHash.Id; + + var tensorCombo = await db.TensorCombos.FirstOrDefaultAsync(x => + x.BaseQuant == lookup.BaseQuant && + x.Embeddings == lookup.Embeddings && + x.LmHead == lookup.LmHead && + x.AttnQ == lookup.AttnQ && + x.AttnKV == lookup.AttnKV && + x.AttnOutput == lookup.AttnOutput && + x.FfnUpGate == lookup.FfnUpGate && + x.FfnDown == lookup.FfnDown && + x.MoeExperts == lookup.MoeExperts && + x.MoeRouter == lookup.MoeRouter, ct); + + if (tensorCombo == null) + { + tensorCombo = new TensorCombo(lookup); + db.TensorCombos.Add(tensorCombo); + await db.SaveChangesAsync(ct); + } + + imatrixDefinitionId ??= + await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, persistenceAiModelHashId, + createIfMissing: true, ct); + await ImatrixIdentityService.ValidateOwnershipAsync(db, persistenceAiModelHashId, imatrixDefinitionId, ct); + + Guid? aiBenchmarkId = await db.AiBenchmarks + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.AiModelHashId == persistenceAiModelHashId && + x.ImatrixDefinitionId == imatrixDefinitionId && + x.TensorComboId == tensorCombo.Id) + .Select(x => (Guid?)x.Id) + .FirstOrDefaultAsync(ct); + + var row = new QuantizationRun + { + Id = Guid.NewGuid(), + ArchitectureFamilyId = architectureFamilyId, + TensorGroupProfileId = tensorGroupProfileId, + AiModelHashId = persistenceAiModelHashId, + ImatrixDefinitionId = imatrixDefinitionId, + TensorComboId = tensorCombo.Id, + AiBenchmarkId = aiBenchmarkId, + StartedUtc = startedUtc, + CompletedUtc = completedUtc, + DurationMs = Math.Max(0L, (long)(completedUtc - startedUtc).TotalMilliseconds), + Succeeded = succeeded, + Error = error, + OutputModelPath = outputModelPath + }; + + db.QuantizationRuns.Add(row); + await db.SaveChangesAsync(ct); + } + + // ---------------------------------------------------------------- + // Base/native model helpers + // ---------------------------------------------------------------- + + public async Task EnsureBaseModelAsync(bool deleteProcess = false) + { + string outputPath = await EnsureBaseModelFileAsync(deleteProcess); + + string typeStr = (Cache.TorchType ?? Cache.MainTorchType.BF16).ToString(); + string benchPath = Path.Combine(_paths.BenchDir, typeStr); + string logitsDir = Path.Combine(benchPath, "logits"); + + AnsiConsole.MarkupLine($"[bold yellow]Benchmarking Base {Markup.Escape(typeStr)} (Saving Logits)...[/]"); + + var baseModelQuant = new HybridQuant + { + BaseQuant = BaselineQuants.GetNativeQuant(), + Tensors = new List() + }; + + await _benchmarker.RunAllBenchmarksAsync( + quantConfig: baseModelQuant, + modelPath: outputPath, + benchDir: benchPath, + klLogitsDir: logitsDir, + saveLogits: true, + domainsOverride: new[] { "general", "code", "math" } + ); + + return outputPath; + } + + public Task EnsureBaseModelFileAsync(bool deleteProcess = false) + => new NativeModelConversionService(_paths, _python).EnsureAsync(deleteProcess); + + public async Task BuildExportArtifactFromExactTensorMapAsync( + IReadOnlyDictionary tensorTypes, + string outputPath, + string baseQuantName, + bool forceRebuild = false, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (tensorTypes == null || tensorTypes.Count == 0) + throw new ArgumentException("A clone tensor map must contain at least one tensor entry.", + nameof(tensorTypes)); + + if (string.IsNullOrWhiteSpace(outputPath)) + throw new InvalidOperationException("Export output path is required."); + + var baseQuant = BaselineQuants.ResolveBuiltInStandardBaseline(baseQuantName) + ?? BaselineQuants.Q8_0; + + Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!); + + if (!forceRebuild && File.Exists(outputPath) && new FileInfo(outputPath).Length > 0 && File.Exists(outputPath + ".success.json")) + return outputPath; + + if (forceRebuild && File.Exists(outputPath)) + await HardDeleteHelper.DeleteFileIfExistsAsync(outputPath); + + await _cpuQuantLock.WaitAsync(ct); + try + { + string nativeBasePath = await EnsureBaseModelFileAsync(); + await RunLlamaQuantizeWithExactTensorMapAsync( + inputFile: nativeBasePath, + outputFile: outputPath, + tensorTypes: tensorTypes, + baseQuant: baseQuant, + ct: ct); + + await File.WriteAllTextAsync(outputPath + ".success.json", "{\"status\":\"success\"}", ct); + return outputPath; + } + catch + { + await HardDeleteHelper.DeleteFileIfExistsAsync(outputPath); + await HardDeleteHelper.DeleteFileIfExistsAsync(outputPath + ".success.json"); + throw; + } + finally + { + _cpuQuantLock.Release(); + } + } + + public async Task BuildExportArtifactAsync( + HybridQuant quant, + string outputPath, + bool forceRebuild = false, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (quant == null) + throw new ArgumentNullException(nameof(quant)); + + if (string.IsNullOrWhiteSpace(outputPath)) + throw new InvalidOperationException("Export output path is required."); + + Directory.CreateDirectory(Path.GetDirectoryName(outputPath)!); + + if (!forceRebuild && File.Exists(outputPath) && new FileInfo(outputPath).Length > 0 && File.Exists(outputPath + ".success.json")) + return outputPath; + + if (forceRebuild && File.Exists(outputPath)) + await HardDeleteHelper.DeleteFileIfExistsAsync(outputPath); + + HybridQuant quantToExecute = quant.BaseQuant.IsExternalRepositoryBaseline + ? CreateEquivalentStandardCarrierQuantForExternalRebuild(quant) + : quant; + + IReadOnlyDictionary? temporaryCarrierOverrides = null; + + if (quant.BaseQuant.IsExternalRepositoryBaseline) + { + temporaryCarrierOverrides = TryLoadAllLearnedTensorMappings( + canonicalBaselineKey: quant.BaseQuant.CanonicalKey, + preferredSourceScheme: quant.BaseQuant.DefaultTensorScheme, + allowDominantFallback: true); + + if (temporaryCarrierOverrides.Count == 0) + { + throw new InvalidOperationException( + $"Missing blanket learned mapping for external/custom baseline '{quant.BaseQuant.Names[0]}'. " + + "MagicQuant cannot export a hybrid from an external baseline until that baseline has been learned."); + } + } + + await _cpuQuantLock.WaitAsync(ct); + try + { + string nativeBasePath = await EnsureBaseModelFileAsync(); + + await RunLlamaQuantizeAsync( + inputFile: nativeBasePath, + outputFile: outputPath, + quant: quantToExecute, + temporaryCarrierOverrides: temporaryCarrierOverrides, + ct: ct); + + await File.WriteAllTextAsync(outputPath + ".success.json", "{\"status\":\"success\"}", ct); + return outputPath; + } + catch + { + await HardDeleteHelper.DeleteFileIfExistsAsync(outputPath); + await HardDeleteHelper.DeleteFileIfExistsAsync(outputPath + ".success.json"); + throw; + } + finally + { + _cpuQuantLock.Release(); + } + } + + + public async Task BuildPureQ8ProbeLeaseAsync(CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + string basePath = await EnsureBaseModelFileAsync(); + + var pureQ8 = new HybridQuant + { + BaseQuant = BaselineQuants.Q8_0, + Tensors = new List() + }; + + string modelName = GenerateHybridName(pureQ8); + var lease = await _scratchStorage.AcquireAsync(ScratchArtifactKind.PureQ8Probe, modelName, ct: ct); + + try + { + await _cpuQuantLock.WaitAsync(ct); + try + { + AnsiConsole.MarkupLine($"[cyan]Building pure Q8 probe baseline:[/] {Markup.Escape(modelName)}"); + await RunLlamaQuantizeAsync( + basePath, + lease.GgufPath, + pureQ8, + logPath: lease.PrimaryLogPath, + metadataWorkingDirectory: lease.LeaseDirectory, + ct: ct); + } + finally + { + _cpuQuantLock.Release(); + } + + return lease; + } + catch + { + await lease.DisposeAsync(); + throw; + } + } + + // ---------------------------------------------------------------- + // Quantization + // ---------------------------------------------------------------- + // Quantization + // ---------------------------------------------------------------- + + + private async Task RunLlamaQuantizeWithExactTensorMapAsync( + string inputFile, + string outputFile, + IReadOnlyDictionary tensorTypes, + BaselineQuants baseQuant, + string? logPath = null, + string? metadataWorkingDirectory = null, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (string.IsNullOrWhiteSpace(inputFile) || !File.Exists(inputFile)) + throw new FileNotFoundException($"Input GGUF not found: {inputFile}"); + + Directory.CreateDirectory(Path.GetDirectoryName(outputFile)!); + + var inputTensorMetadata = await ReadTensorMetadataFromGgufAsync(inputFile, metadataWorkingDirectory ?? Path.GetDirectoryName(outputFile)!); + var requestedOverrides = tensorTypes + .OrderBy(x => x.Key, StringComparer.Ordinal) + .Select(x => new RequestedTensorOverride + { + GroupName = "clone_exact_tensor_map", + TensorName = x.Key, + SchemeName = NormalizeQuantName(x.Value) + }) + .ToList(); + + var concreteOverrides = ResolveConcreteTensorOverrides( + allTensorNames: inputTensorMetadata.TensorNames, + requestedOverrides: requestedOverrides); + + var missingInManifest = inputTensorMetadata.TensorNames + .Except(tensorTypes.Keys, StringComparer.Ordinal) + .Take(20) + .ToList(); + + var unexpectedInManifest = tensorTypes.Keys + .Except(inputTensorMetadata.TensorNames, StringComparer.Ordinal) + .Take(20) + .ToList(); + + if (missingInManifest.Count > 0 || unexpectedInManifest.Count > 0 || + inputTensorMetadata.TensorNames.Count != tensorTypes.Count) + { + throw new InvalidOperationException( + $"Clone tensor manifest does not exactly match this model architecture. " + + $"MissingInManifest=[{string.Join(", ", missingInManifest)}] UnexpectedInManifest=[{string.Join(", ", unexpectedInManifest)}] " + + $"ModelTensorCount={inputTensorMetadata.TensorNames.Count} ManifestTensorCount={tensorTypes.Count}."); + } + + var args = new List(capacity: concreteOverrides.Count + 8); + + foreach (var overrideItem in concreteOverrides) + args.AddRange(["--tensor-type", $"{overrideItem.TensorName}={overrideItem.SchemeName}"]); + + if (_imatrixService.ShouldUseImatrixForQuant(HybridQuant.CreatePureBaseline(baseQuant))) + { + string imatrixPath = _imatrixService.GetCanonicalImatrixPath(); + if (!File.Exists(imatrixPath)) + throw new InvalidOperationException( + $"Imatrix was marked active but canonical artifact is missing: {imatrixPath}"); + + args.AddRange(["--imatrix", imatrixPath]); + } + + args.Add(inputFile); + args.Add(outputFile); + args.Add(baseQuant.QuantizeBaseArgumentName); + args.Add(_quantThreadsPerProcess.ToString()); + + string bin = Path.Combine( + Cache.LlamaBin!, + RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "llama-quantize.exe" : "llama-quantize"); + + string quantizeLogPath = string.IsNullOrWhiteSpace(logPath) ? outputFile + ".quantize.log" : logPath; + Directory.CreateDirectory(Path.GetDirectoryName(quantizeLogPath)!); + AnsiConsole.MarkupLine( + $"[cyan]Quantizing clone artifact:[/] {Markup.Escape(Path.GetFileName(outputFile))} [grey](log: {Markup.Escape(quantizeLogPath)})[/]"); + + var result = await RunLoggedProcessAsync(new MagicQuant.Runtime.NativeCommand(bin, args).CreateStartInfo(), quantizeLogPath, ct); + + if (result.ExitCode != 0) + { + await HardDeleteHelper.DeleteFileIfExistsAsync(outputFile); + throw new InvalidOperationException( + $"Clone quantization failed for '{outputFile}'. ExitCode={result.ExitCode}. See '{quantizeLogPath}'."); + } + + if (!File.Exists(outputFile) || new FileInfo(outputFile).Length == 0) + { + await HardDeleteHelper.DeleteFileIfExistsAsync(outputFile); + throw new InvalidOperationException( + $"Clone quantization exited successfully but produced no valid GGUF output: {outputFile}"); + } + + AnsiConsole.MarkupLine($"[green]Clone quantized model ready:[/] {Markup.Escape(outputFile)}"); + + return new QuantizationExecutionReport + { + LogPath = quantizeLogPath, + ResolvedOverrides = concreteOverrides + }; + } + + private async Task RunLlamaQuantizeAsync( + string inputFile, + string outputFile, + HybridQuant quant, + IReadOnlyDictionary? temporaryCarrierOverrides = null, + string? logPath = null, + string? metadataWorkingDirectory = null, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (string.IsNullOrWhiteSpace(inputFile) || !File.Exists(inputFile)) + throw new FileNotFoundException($"Input GGUF not found: {inputFile}"); + + if (temporaryCarrierOverrides != null || quant.BaseQuant.IsExternalRepositoryBaseline) + { + string nativeBase = await EnsureBaseModelFileAsync(); + if (!string.Equals(Path.GetFullPath(inputFile), Path.GetFullPath(nativeBase), StringComparison.OrdinalIgnoreCase)) + { + throw new InvalidOperationException( + $"Quantization attempted to use non-native carrier input '{inputFile}' for external/override execution. " + + "External/custom baseline rebuilds must use the native base GGUF as input."); + } + } + + Directory.CreateDirectory(Path.GetDirectoryName(outputFile)!); + + var inputTensorMetadata = await ReadTensorMetadataFromGgufAsync(inputFile, metadataWorkingDirectory ?? Path.GetDirectoryName(outputFile)!); + var requestedOverrides = + BuildRequestedTensorOverrides(quant, inputTensorMetadata.TensorNames, temporaryCarrierOverrides); + var concreteOverrides = ResolveConcreteTensorOverrides( + allTensorNames: inputTensorMetadata.TensorNames, + requestedOverrides: requestedOverrides); + bool shouldRequireFullLearnedCoverage = ShouldApplyLearnedBaseCarrierBlanket(quant, temporaryCarrierOverrides); + + if (requestedOverrides.Count > 0 && concreteOverrides.Count == 0) + { + throw new InvalidOperationException( + $"No concrete tensors were resolved for requested overrides when quantizing '{outputFile}'. " + + "This means the requested tensor selectors did not match the input GGUF."); + } + + if (shouldRequireFullLearnedCoverage) + { + var concreteNames = concreteOverrides + .Select(x => x.TensorName) + .ToHashSet(StringComparer.Ordinal); + var missing = inputTensorMetadata.TensorNames + .Except(concreteNames, StringComparer.Ordinal) + .OrderBy(x => x, StringComparer.Ordinal) + .ToList(); + + if (missing.Count > 0) + { + throw new InvalidOperationException( + $"Full learned base-carrier coverage is incomplete for baseline '{quant.BaseQuant.Names[0]}'. " + + $"Missing={missing.Count}. Examples=[{string.Join(", ", missing.Take(15))}]. " + + "Use targeted YAML relearn configuration to regenerate only the affected baseline/profile truth."); + } + } + + var args = new List(capacity: 256); + + foreach (var overrideItem in concreteOverrides) + { + args.AddRange(["--tensor-type", $"{overrideItem.TensorName}={overrideItem.SchemeName}"]); + } + + if (ShouldApplyImatrix(quant)) + { + string imatrixPath = _imatrixService.GetCanonicalImatrixPath(); + if (!File.Exists(imatrixPath)) + throw new InvalidOperationException( + $"Imatrix was marked active but canonical artifact is missing: {imatrixPath}"); + + args.AddRange(["--imatrix", imatrixPath]); + } + + args.Add(inputFile); + args.Add(outputFile); + args.Add(ResolveQuantizeBaseArgument(quant, concreteOverrides)); + args.Add(_quantThreadsPerProcess.ToString()); + + string bin = Path.Combine( + Cache.LlamaBin!, + RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "llama-quantize.exe" : "llama-quantize"); + + string quantizeLogPath = string.IsNullOrWhiteSpace(logPath) ? outputFile + ".quantize.log" : logPath; + Directory.CreateDirectory(Path.GetDirectoryName(quantizeLogPath)!); + + var psi = new MagicQuant.Runtime.NativeCommand(bin, args).CreateStartInfo(); + + AnsiConsole.MarkupLine( + $"[cyan]Quantizing:[/] {Markup.Escape(Path.GetFileName(outputFile))} [grey](log: {Markup.Escape(quantizeLogPath)})[/]"); + var result = await RunLoggedProcessAsync(psi, quantizeLogPath, ct); + + if (result.ExitCode != 0) + { + await HardDeleteHelper.DeleteFileIfExistsAsync(outputFile); + + throw new InvalidOperationException( + $"Quantization failed for '{outputFile}'. ExitCode={result.ExitCode}. See '{quantizeLogPath}'."); + } + + if (!File.Exists(outputFile) || new FileInfo(outputFile).Length == 0) + { + await HardDeleteHelper.DeleteFileIfExistsAsync(outputFile); + + throw new InvalidOperationException( + $"Quantization process exited successfully but produced no valid GGUF output: {outputFile}"); + } + + AnsiConsole.MarkupLine($"[green]Quantized model ready:[/] {Markup.Escape(outputFile)}"); + + return new QuantizationExecutionReport + { + LogPath = quantizeLogPath, + ResolvedOverrides = concreteOverrides + }; + } + + private static string ResolveQuantizeBaseArgument( + HybridQuant quant, + List concreteOverrides) + { + if (quant.BaseQuant.UniqueId == BaselineQuants.NativeSourceUniqueId && + concreteOverrides.Count > 0) + { + throw new InvalidOperationException( + "Native BF16/F16/F32 + tensor overrides is disabled. " + + "In this build of llama-quantize it produced no-op outputs for isolation tests. " + + "Use a real carrier baseline (Q8_0 recommended) and apply only learned exact tensor overrides for the target configuration."); + } + + return quant.BaseQuant.QuantizeBaseArgumentName; + } + + private static HybridQuant CreateEquivalentStandardCarrierQuantForExternalRebuild(HybridQuant quant) + { + if (!quant.BaseQuant.IsExternalRepositoryBaseline) + return quant; + + var standardFamily = BaselineQuants.ResolveBuiltInStandardBaseline(quant.BaseQuant.QuantizeBaseArgumentName) + ?? BaselineQuants.ResolveBuiltInStandardBaseline(quant.BaseQuant.Names[0]) + ?? throw new InvalidOperationException( + $"Could not resolve a built-in carrier baseline for external baseline '{quant.BaseQuant.Names[0]}' using quantize base name '{quant.BaseQuant.QuantizeBaseArgumentName}'."); + + var clone = quant.Clone(); + clone.BaseQuant = standardFamily; + return clone; + } + + private bool ShouldApplyImatrix(HybridQuant quant) + { + return _imatrixService.ShouldUseImatrixForQuant(quant); + } + + public async Task> ReadExactTensorTypesAsync( + string ggufPath, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + var meta = await ReadTensorMetadataFromGgufAsync(ggufPath, Path.GetDirectoryName(ggufPath)!); + return meta.TensorTypes + .OrderBy(x => x.Key, StringComparer.Ordinal) + .ToDictionary(x => x.Key, x => NormalizeQuantName(x.Value), StringComparer.Ordinal); + } + + public Task ClearLearnedBaselineTensorMappingsAsync(CancellationToken ct = default) + { + throw new NotSupportedException( + "Global learned tensor mapping wipes were removed. Use targeted YAML relearn options so deletion is scoped, counted, and confirmed."); + } + + public Task InvalidateBaselineArtifactsAsync(CancellationToken ct = default) + { + throw new NotSupportedException( + "Global baseline artifact invalidation was removed. Use learning.force_relearn_architecture_family, learning.force_relearn_standard_baselines, or include.force_relearn."); + } + + public async Task HasNativeSourceLearnedTruthAsync(CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (string.IsNullOrWhiteSpace(Cache.CurrentModelId)) + return false; + + var nativeScheme = TensorWeightScheme.GetCurrentNativePrecisionScheme(); + + await using var db = new MagicQuantContext(); + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + var baselineDefinition = await BaselineDefinitionResolver.ResolveRequiredDefinitionAsync(db, BaselineQuants.GetNativeQuant(), ct); + + return await db.LearnedBaselineTensorQuants + .AsNoTracking() + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.BaselineQuantDefinitionId == baselineDefinition.Id && + x.TensorWeightSchemeId == nativeScheme.UniqueId) + .AnyAsync(ct); + } + + public async Task LearnNativeSourceTruthAsync( + string nativeGgufPath, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (string.IsNullOrWhiteSpace(nativeGgufPath) || !File.Exists(nativeGgufPath)) + throw new FileNotFoundException($"Native GGUF path not found for learning: {nativeGgufPath}"); + + var nativeScheme = TensorWeightScheme.GetCurrentNativePrecisionScheme(); + + if (await HasNativeSourceLearnedTruthAsync(ct)) + { + AnsiConsole.MarkupLine( + $"[grey]Native-source learned truth already exists:[/] for [yellow]{Markup.Escape(nativeScheme.Names[0])}[/]. Skipping. Use targeted YAML relearn to regenerate native source truth if needed."); + return; + } + + var metadata = await ReadTensorMetadataFromGgufAsync(nativeGgufPath, Path.GetDirectoryName(nativeGgufPath)!); + var ggufTruth = metadata.TensorTypes + .ToDictionary(x => x.Key, x => NormalizeQuantName(x.Value), StringComparer.Ordinal); + + var verification = BuildTruthMapWithVerification( + logTruth: new Dictionary(StringComparer.Ordinal), + ggufTruth: ggufTruth, + baselineName: "NATIVE"); + var truth = verification.TruthByTensor; + + var audit = _tensorGroupingAuditService.Audit(truth.Keys.ToList(), truth); + + if (audit.HasFatalIssues || verification.HasFatalIssues) + { + var diagnosticPath = await _tensorLearningDiagnosticWriter.WriteFailureAsync( + baselineName: $"NATIVE_{nativeScheme.Names[0]}", + schemeName: nativeScheme.Names[0], + sourceKind: "NativeSource", + sourceRepository: null, + sourceFileName: Path.GetFileName(nativeGgufPath), + truthByTensor: truth, + audit: audit, + verification: verification, + ct: ct); + + AnsiConsole.MarkupLine( + $"[red]Tensor group learning failed.[/] See diagnostic log: [yellow]{Markup.Escape(diagnosticPath)}[/]"); + throw new InvalidOperationException( + $"Strict tensor-group learning validation failed for baseline 'NATIVE_{nativeScheme.Names[0]}'. " + + $"Ambiguous={audit.Ambiguous.Count}, IllegalUnresolved={audit.IllegalUnresolved.Count}, " + + $"AllowedBaseQuantFallback={audit.BaseQuantExceptions.Count}. " + + $"No learned tensor mappings were persisted. Diagnostic log: {diagnosticPath}"); + } + + await using var db = new MagicQuantContext(); + var scopedAiModelHashId = await ResolveCurrentScopedAiModelHashIdOrNullAsync(db, ct) + ?? throw new InvalidOperationException( + "Could not persist native-source learning because scoped AiModelHash row was missing."); + + var combo = await db.TensorCombos + .AsNoTracking() + .FirstOrDefaultAsync(x => x.BaseQuant == BaselineQuants.NativeSourceUniqueId && + x.Embeddings == 0 && x.LmHead == 0 && x.AttnQ == 0 && x.AttnKV == 0 && + x.AttnOutput == 0 && x.FfnUpGate == 0 && x.FfnDown == 0 && + x.MoeExperts == 0 && x.MoeRouter == 0, ct); + + if (combo == null) + { + combo = new TensorCombo((TensorConfig)HybridQuant.CreatePureBaseline(BaselineQuants.GetNativeQuant())); + db.TensorCombos.Add(combo); + await db.SaveChangesAsync(ct); + } + + var benchmarkAiModelHashId = scopedAiModelHashId; + var imatrixDefinitionId = await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync( + db, + benchmarkAiModelHashId, + createIfMissing: false, + ct); + + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + var baselineDefinition = await BaselineDefinitionResolver.ResolveRequiredDefinitionAsync(db, BaselineQuants.GetNativeQuant(), ct); + + var benchmarkId = await db.AiBenchmarks + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.AiModelHashId == benchmarkAiModelHashId && + x.ImatrixDefinitionId == imatrixDefinitionId && + x.TensorComboId == combo.Id) + .OrderByDescending(x => x.Id) + .Select(x => (Guid?)x.Id) + .FirstOrDefaultAsync(ct); + + if (!benchmarkId.HasValue) + throw new InvalidOperationException( + "Native-source benchmark row is missing; benchmark base model before native-source learning."); + + var rows = truth + .OrderBy(x => x.Key, StringComparer.Ordinal) + .Select(x => + { + var primaryGroup = audit.GroupedByTensor[x.Key].PrimaryGroup; + + return new LearnedBaselineTensorQuant + { + Id = Guid.NewGuid(), + AiBenchmarkId = benchmarkId.Value, + ArchitectureFamilyId = architectureFamilyId, + TensorGroupProfileId = tensorGroupProfileId, + BaselineQuantDefinitionId = baselineDefinition.Id, + TensorComboId = combo.Id, + AiModelHashId = scopedAiModelHashId, + BaselineQuantId = BaselineQuants.NativeSourceUniqueId, + TensorWeightSchemeId = nativeScheme.UniqueId, + BaselineCanonicalKey = BaselineQuants.GetNativeQuant().CanonicalKey, + BaselineSourceKind = BaselineQuants.GetNativeQuant().SourceKind, + BaselineSourceRepository = BaselineQuants.GetNativeQuant().SourceRepository, + BaselineSourceFileName = BaselineQuants.GetNativeQuant().SourceFileName, + TensorGroupId = primaryGroup?.UniqueId ?? UnknownTensorGroupId, + TensorName = x.Key, + FinalQuantType = x.Value.FinalQuantType + }; + }) + .ToList(); + + if (rows.Count == 0) + throw new InvalidOperationException("Native-source learning produced no persistable rows."); + + await db.LearnedBaselineTensorQuants + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.BaselineQuantDefinitionId == baselineDefinition.Id && + x.TensorWeightSchemeId == nativeScheme.UniqueId) + .ExecuteDeleteAsync(ct); + + db.LearnedBaselineTensorQuants.AddRange(rows); + await db.SaveChangesAsync(ct); + + await WriteLearningDiagnosticArtifactAsync( + baselineName: $"NATIVE_{nativeScheme.Names[0]}", + schemeName: nativeScheme.Names[0], + truthByTensor: truth, + grouped: audit.GroupedByTensor, + allTensorNamesInModel: metadata.TensorNames, + ambiguous: audit.Ambiguous, + unresolved: audit.IllegalUnresolved.Select(x => x.TensorName).ToList()); + + var sourcePrecision = nativeScheme.Names[0]; + var distribution = rows.GroupBy(x => x.FinalQuantType) + .OrderByDescending(g => g.Count()) + .Select(g => $"{g.Key}:{g.Count()}") + .ToList(); + + AnsiConsole.MarkupLine( + $"[green]Native-source learned truth:[/] precision={Markup.Escape(sourcePrecision)}, tensors={rows.Count}, unresolved={audit.IllegalUnresolved.Count}, ambiguous={audit.Ambiguous.Count}, baseFallback={audit.BaseQuantExceptions.Count}, dist={Markup.Escape($"[{string.Join(", ", distribution)}]")}"); + } + + private static bool IsLearnableBaselineRun(HybridQuant quant) + { + return quant.Tensors.Count == 0 && + quant.BaseQuant.UniqueId != BaselineQuants.NativeSourceUniqueId && + quant.BaseQuant.DefaultTensorScheme != null; + } + + private async Task LearnAndPersistBaselineTensorMapAsync( + HybridQuant quant, + string quantizedModelPath, + QuantizationExecutionReport? report, + CancellationToken ct) + { + if (!IsLearnableBaselineRun(quant)) + return; + + if (quant.BaseQuant.IsExternalRepositoryBaseline) + { + string nativeBasePath = await EnsureBaseModelFileAsync(); + await ValidateExternalBaselineTensorParityOrThrow(nativeBasePath, quantizedModelPath); + } + + var tensorScheme = quant.BaseQuant.DefaultTensorScheme!; + string logPath = report?.LogPath ?? (quantizedModelPath + ".quantize.log"); + var parsed = ParseQuantizeLogForTensorTypes(logPath); + var ggufMetadata = await ReadTensorMetadataFromGgufAsync(quantizedModelPath, Path.GetDirectoryName(quantizedModelPath)!); + var ggufTruth = ggufMetadata.TensorTypes + .ToDictionary(x => x.Key, x => NormalizeQuantName(x.Value), StringComparer.Ordinal); + + if (parsed.Count == 0 && ggufTruth.Count == 0) + { + throw new InvalidOperationException( + $"Strict tensor learning failed for baseline '{quant.BaseQuant.Names[0]}': no tensor truth could be read from either the quantize log or GGUF metadata. " + + $"QuantizedModelPath={quantizedModelPath}; LogPath={logPath}"); + } + + var verification = BuildTruthMapWithVerification(parsed, ggufTruth, quant.BaseQuant.Names[0]); + var truth = verification.TruthByTensor; + if (truth.Count == 0) + throw new InvalidOperationException( + $"No verified tensor truth entries were available for baseline '{quant.BaseQuant.Names[0]}'."); + + var audit = _tensorGroupingAuditService.Audit(truth.Keys.ToList(), truth); + if (audit.HasFatalIssues || verification.HasFatalIssues) + { + var diagnosticPath = await _tensorLearningDiagnosticWriter.WriteFailureAsync( + baselineName: quant.BaseQuant.Names[0], + schemeName: tensorScheme.Names[0], + sourceKind: quant.BaseQuant.SourceKind.ToString(), + sourceRepository: quant.BaseQuant.SourceRepository, + sourceFileName: quant.BaseQuant.SourceFileName, + truthByTensor: truth, + audit: audit, + verification: verification, + ct: ct); + + AnsiConsole.MarkupLine( + $"[red]Tensor group learning failed.[/] See diagnostic log: [yellow]{Markup.Escape(diagnosticPath)}[/]"); + throw new InvalidOperationException( + $"Strict tensor-group learning validation failed for baseline '{quant.BaseQuant.Names[0]}'. " + + $"Ambiguous={audit.Ambiguous.Count}, " + + $"IllegalUnresolved={audit.IllegalUnresolved.Count}, " + + $"AllowedBaseQuantFallback={audit.BaseQuantExceptions.Count}. " + + $"No learned tensor mappings were persisted. " + + $"Diagnostic log: {diagnosticPath}"); + } + + await using var db = new MagicQuantContext(); + + var scopedAiModelHashId = await ResolveCurrentScopedAiModelHashIdOrNullAsync(db, ct); + if (scopedAiModelHashId == null) + throw new InvalidOperationException( + "Unable to persist learned mappings because scoped AiModelHash row was not found."); + + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + var baselineDefinition = await BaselineDefinitionResolver.ResolveRequiredDefinitionAsync(db, quant.BaseQuant, ct); + + var combo = await db.TensorCombos + .AsNoTracking() + .FirstAsync(x => x.BaseQuant == quant.BaseQuant.UniqueId && + x.Embeddings == 0 && x.LmHead == 0 && x.AttnQ == 0 && x.AttnKV == 0 && + x.AttnOutput == 0 && x.FfnUpGate == 0 && x.FfnDown == 0 && x.MoeExperts == 0 && + x.MoeRouter == 0, ct); + + var benchmarkAiModelHashId = scopedAiModelHashId.Value; + var imatrixDefinitionId = + await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, benchmarkAiModelHashId, + createIfMissing: false, ct); + + var benchmarkId = await db.AiBenchmarks + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.AiModelHashId == benchmarkAiModelHashId && + x.ImatrixDefinitionId == imatrixDefinitionId && + x.TensorComboId == combo.Id) + .OrderByDescending(x => x.Id) + .Select(x => (Guid?)x.Id) + .FirstOrDefaultAsync(ct); + + if (!benchmarkId.HasValue) + throw new InvalidOperationException( + $"Unable to persist learned mappings because no AiBenchmark exists for baseline '{quant.BaseQuant.Names[0]}'."); + + var rows = truth + .OrderBy(x => x.Key, StringComparer.Ordinal) + .Select(kv => + { + var match = audit.GroupedByTensor[kv.Key]; + + return new LearnedBaselineTensorQuant + { + Id = Guid.NewGuid(), + AiBenchmarkId = benchmarkId.Value, + ArchitectureFamilyId = architectureFamilyId, + TensorGroupProfileId = tensorGroupProfileId, + BaselineQuantDefinitionId = baselineDefinition.Id, + TensorComboId = combo.Id, + AiModelHashId = scopedAiModelHashId.Value, + BaselineQuantId = quant.BaseQuant.UniqueId, + TensorWeightSchemeId = tensorScheme.UniqueId, + TensorGroupId = match.PrimaryGroup?.UniqueId ?? UnknownTensorGroupId, + BaselineCanonicalKey = quant.BaseQuant.CanonicalKey, + BaselineSourceKind = quant.BaseQuant.SourceKind, + BaselineSourceRepository = quant.BaseQuant.SourceRepository, + BaselineSourceFileName = quant.BaseQuant.SourceFileName, + TensorName = kv.Key, + FinalQuantType = kv.Value.FinalQuantType + }; + }) + .ToList(); + + if (rows.Count == 0) + throw new InvalidOperationException( + $"Learning baseline '{quant.BaseQuant.Names[0]}' produced no persistable rows."); + + await db.LearnedBaselineTensorQuants + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.BaselineQuantDefinitionId == baselineDefinition.Id && + x.TensorWeightSchemeId == tensorScheme.UniqueId) + .ExecuteDeleteAsync(ct); + + db.LearnedBaselineTensorQuants.AddRange(rows); + await db.SaveChangesAsync(ct); + + await WriteLearningDiagnosticArtifactAsync( + baselineName: quant.BaseQuant.Names[0], + schemeName: tensorScheme.Names[0], + truthByTensor: truth, + grouped: audit.GroupedByTensor, + allTensorNamesInModel: ggufMetadata.TensorNames, + ambiguous: audit.Ambiguous, + unresolved: audit.IllegalUnresolved.Select(x => x.TensorName).ToList()); + + AnsiConsole.MarkupLine( + $"[green]Learned baseline tensor mapping persisted:[/] [cyan]{rows.Count:N0}[/] row(s) for [yellow]{Markup.Escape(quant.BaseQuant.Names[0])}[/]."); + } + + private Dictionary ParseQuantizeLogForTensorTypes(string logPath) + { + if (!File.Exists(logPath)) + { + AnsiConsole.MarkupLine( + $"[red]WARNING:[/] quantization log does not exist, cannot learn tensor mapping: {Markup.Escape(logPath)}"); + return new Dictionary(StringComparer.Ordinal); + } + + var byTensor = new Dictionary(StringComparer.Ordinal); + + foreach (var raw in File.ReadLines(logPath)) + { + var match = TensorLogLineRegex.Match(raw); + if (!match.Success) + continue; + + string tensorName = match.Groups["tensor"].Value.Trim(); + string declaredType = NormalizeQuantName(match.Groups["type"].Value); + + string final = declaredType; + var convert = match.Groups["convert"]; + if (convert.Success && !string.IsNullOrWhiteSpace(convert.Value)) + final = NormalizeQuantName(convert.Value); + + byTensor[tensorName] = final; + } + + return byTensor; + } + + private TensorTruthVerificationResult BuildTruthMapWithVerification( + IReadOnlyDictionary logTruth, + IReadOnlyDictionary ggufTruth, + string baselineName) + { + var allNames = logTruth.Keys + .Concat(ggufTruth.Keys) + .Distinct(StringComparer.Ordinal) + .OrderBy(x => x, StringComparer.Ordinal) + .ToList(); + + var result = new Dictionary(StringComparer.Ordinal); + var hardMismatches = new List(); + var softMismatches = new List(); + var logOnly = new List(); + + foreach (var name in allNames) + { + var inLog = logTruth.TryGetValue(name, out var logType); + var inGguf = ggufTruth.TryGetValue(name, out var ggufType); + + if (inLog && inGguf) + { + if (string.Equals(logType, ggufType, StringComparison.OrdinalIgnoreCase)) + { + result[name] = new LearnedTensorTruth(name, ggufType!, LearningSource.Both); + } + else + { + // GGUF is the final artifact truth. The log is secondary evidence only. + // A log/GGUF disagreement is useful diagnostic information, but it is + // not fatal as long as GGUF truth exists. + result[name] = new LearnedTensorTruth(name, ggufType!, LearningSource.BothWithMismatch); + + softMismatches.Add(new TensorTruthMismatch + { + TensorName = name, + LogQuantType = logType!, + GgufQuantType = ggufType!, + IsHighSeverity = IsHighSeverityMismatch(logType!, ggufType!) + }); + } + } + else if (inGguf) + { + result[name] = new LearnedTensorTruth(name, ggufType!, LearningSource.GgufOnly); + } + else if (inLog) + { + // Log-only entries are not reliable enough to learn from because there is no + // final GGUF artifact truth confirming them. + logOnly.Add($"{name}:{logType}"); + } + } + + if (softMismatches.Count > 0) + { + var highSeverityCount = softMismatches.Count(x => x.IsHighSeverity); + var lowSeverityCount = softMismatches.Count - highSeverityCount; + + var severitySummary = highSeverityCount > 0 && lowSeverityCount > 0 + ? $"{highSeverityCount} high-severity, {lowSeverityCount} low-severity" + : highSeverityCount > 0 + ? $"{highSeverityCount} high-severity" + : $"{lowSeverityCount} low-severity"; + + AnsiConsole.MarkupLine( + $"[yellow]GGUF/log mismatch:[/] Baseline [yellow]{Markup.Escape(baselineName)}[/] had {softMismatches.Count} tensor type disagreement(s) ({severitySummary}). GGUF artifact truth was used."); + + AnsiConsole.MarkupLine( + $"[grey]Examples: {Markup.Escape(string.Join(" | ", softMismatches.Take(6).Select(x => $"{x.TensorName}: log={x.LogQuantType} gguf={x.GgufQuantType}")))}[/]"); + } + + if (logOnly.Count > 0) + { + AnsiConsole.MarkupLine( + $"[yellow]GGUF/log mismatch:[/] Baseline [yellow]{Markup.Escape(baselineName)}[/] produced {logOnly.Count} log-only tensor mapping(s) with no GGUF artifact truth. They were ignored."); + + AnsiConsole.MarkupLine( + $"[grey]Examples: {Markup.Escape(string.Join(" | ", logOnly.Take(6)))}[/]"); + } + + return new TensorTruthVerificationResult + { + TruthByTensor = result, + + // Deliberately empty for log-vs-GGUF disagreements where GGUF truth exists. + // GGUF wins, so these are diagnostics, not fatal validation failures. + HardMismatches = hardMismatches, + + SoftMismatches = softMismatches, + LogOnly = logOnly + }; + } + + private static bool IsHighSeverityMismatch(string logType, string ggufType) + { + bool logHighPrecision = IsHighPrecisionType(logType); + bool ggufHighPrecision = IsHighPrecisionType(ggufType); + return logHighPrecision != ggufHighPrecision; + } + + private async Task WriteLearningDiagnosticArtifactAsync( + string baselineName, + string schemeName, + IReadOnlyDictionary truthByTensor, + IReadOnlyDictionary grouped, + IReadOnlyCollection allTensorNamesInModel, + IReadOnlyCollection ambiguous, + IReadOnlyCollection unresolved) + { + var summaries = new List(); + var severeCoverageIssues = new List(); + + foreach (var group in TReg.All.OrderBy(x => x.UniqueId)) + { + var expected = allTensorNamesInModel + .Where(x => group.Tensors.Any(p => Regex.IsMatch(x, $"^{p}$"))) + .OrderBy(x => x, StringComparer.Ordinal) + .ToList(); + + var learned = truthByTensor + .Where(x => grouped.TryGetValue(x.Key, out var g) && g.PrimaryGroup?.UniqueId == group.UniqueId) + .Select(x => x.Key) + .OrderBy(x => x, StringComparer.Ordinal) + .ToList(); + + var unmatched = expected.Except(learned, StringComparer.Ordinal).Take(20).ToList(); + var unexpected = learned.Except(expected, StringComparer.Ordinal).Take(20).ToList(); + + var distribution = truthByTensor + .Where(x => learned.Contains(x.Key, StringComparer.Ordinal)) + .GroupBy(x => x.Value.FinalQuantType) + .OrderByDescending(g => g.Count()) + .ToDictionary(g => g.Key, g => g.Count()); + + var sourceCounts = truthByTensor + .Where(x => learned.Contains(x.Key, StringComparer.Ordinal)) + .GroupBy(x => x.Value.Source.ToString()) + .ToDictionary(g => g.Key, g => g.Count()); + + summaries.Add(new + { + Group = group.Name, + ExpectedTensorCount = expected.Count, + LearnedTensorCount = learned.Count, + UnmatchedExpected = unmatched, + UnexpectedLearned = unexpected, + Ambiguous = ambiguous.Where(x => x.MatchedGroups.Contains(group.Name)).Select(x => x.TensorName) + .Take(20).ToList(), + QuantDistribution = distribution, + SourceDistribution = sourceCounts + }); + + var distShort = distribution.Count == 0 + ? "none" + : string.Join(", ", distribution.Select(kv => $"{kv.Key}:{kv.Value}")); + + var srcShort = sourceCounts.Count == 0 + ? "none" + : string.Join(", ", sourceCounts.Select(kv => $"{kv.Key}:{kv.Value}")); + + var label = $"[learn:{baselineName}:{group.Name}]"; + + AnsiConsole.MarkupLine( + $"[grey]{Markup.Escape(label)} " + + $"expected={expected.Count} " + + $"learned={learned.Count} " + + $"unmatched={unmatched.Count} " + + $"ambiguous={ambiguous.Count(x => x.MatchedGroups.Contains(group.Name))} " + + $"dist={Markup.Escape($"[{distShort}]")} " + + $"src={Markup.Escape($"[{srcShort}]")}[/]"); + + if (expected.Count > 0 && unmatched.Count > 0) + { + severeCoverageIssues.Add( + $"{group.Name}: expected={expected.Count} learned={learned.Count} unmatched={unmatched.Count}"); + } + } + + var artifact = new + { + Baseline = baselineName, + Scheme = schemeName, + TotalTruthTensors = truthByTensor.Count, + UnresolvedTensorCount = unresolved.Count, + AmbiguousTensorCount = ambiguous.Count, + GeneratedUtc = DateTime.UtcNow, + Groups = summaries + }; + + string debugDir = Path.Combine(_paths.BenchDir, "_learning_debug"); + Directory.CreateDirectory(debugDir); + string path = Path.Combine(debugDir, $"{baselineName}_{schemeName}_learned_map.json"); + await File.WriteAllTextAsync(path, + JsonSerializer.Serialize(artifact, new JsonSerializerOptions { WriteIndented = true })); + + AnsiConsole.MarkupLine( + $"[grey]Learned mapping diagnostic written:[/] {Markup.Escape(path)}"); + + if (severeCoverageIssues.Count > 0) + { + AnsiConsole.MarkupLine( + $"[yellow]WARNING:[/] Baseline learning coverage was incomplete for {severeCoverageIssues.Count} group(s): " + + $"{Markup.Escape(string.Join(" | ", severeCoverageIssues.Take(8)))}"); + } + } + + private static TensorWeightScheme? TryResolveBaseTensorScheme(BaselineQuants baseQuant) + { + if (baseQuant.UniqueId == BaselineQuants.NativeSourceUniqueId) + return TensorWeightScheme.GetCurrentNativePrecisionScheme(); + + return baseQuant.DefaultTensorScheme; + } + + private static HashSet GetExpectedTensorNamesForGroup( + TensorGroup group, + IReadOnlyCollection sourceTensorNames) + { + return sourceTensorNames + .Where(x => group.Tensors.Any(p => Regex.IsMatch(x, $"^{p}$"))) + .ToHashSet(StringComparer.Ordinal); + } + + + private List BuildRequestedTensorOverrides( + HybridQuant quant, + IReadOnlyCollection sourceTensorNames, + IReadOnlyDictionary? temporaryCarrierOverrides = null) + { + var result = new List(); + + bool hasTemporaryCarrierOverrides = temporaryCarrierOverrides != null && temporaryCarrierOverrides.Count > 0; + bool hasExplicitGroupOverrides = quant.Tensors != null && quant.Tensors.Count > 0; + bool shouldApplyBaseCarrierBlanket = ShouldApplyLearnedBaseCarrierBlanket(quant, temporaryCarrierOverrides); + + if (!shouldApplyBaseCarrierBlanket) + return result; + + var baseScheme = TryResolveBaseTensorScheme(quant.BaseQuant); + var blanket = LoadBaseCarrierTensorMappingsOrThrow( + quant: quant, + temporaryCarrierOverrides: temporaryCarrierOverrides, + requireFullCoverage: shouldApplyBaseCarrierBlanket); + + foreach (var kv in blanket.OrderBy(x => x.Key, StringComparer.Ordinal)) + { + result.Add(new RequestedTensorOverride + { + GroupName = "base_carrier", + TensorName = kv.Key, + SchemeName = kv.Value + }); + } + + if (quant.Tensors == null || !hasExplicitGroupOverrides) + return result; + + foreach (var hybrid in quant.Tensors) + { + if (hybrid?.TGroup == null) + continue; + + hybrid.ValidateOrThrow(); + + if (hybrid.MaterializedTensorScheme.UniqueId == TensorWeightScheme.NULL.UniqueId) + continue; + + var expectedForGroup = GetExpectedTensorNamesForGroup(hybrid.TGroup, sourceTensorNames); + if (expectedForGroup.Count == 0) + continue; + + switch (hybrid.OverrideMode) + { + case HybridTensorOverrideMode.ExactTensorScheme: + { + var exactScheme = hybrid.ExactTensorScheme!; + if (!quant.BaseQuant.IsExternalRepositoryBaseline && baseScheme != null && + exactScheme.UniqueId == baseScheme.UniqueId) + continue; + + string schemeName = ResolveSchemeName(exactScheme); + foreach (var tensorName in expectedForGroup.OrderBy(x => x, StringComparer.Ordinal)) + { + result.Add(new RequestedTensorOverride + { + GroupName = hybrid.TGroup.Name, + TensorName = tensorName, + SchemeName = schemeName + }); + } + + break; + } + + case HybridTensorOverrideMode.LearnedBaselineCandidate: + { + var sourceBaseline = hybrid.CandidateBaseline!; + var learned = TryLoadLearnedTensorMapping( + sourceBaseline: sourceBaseline, + targetGroup: hybrid.TGroup, + preferredSourceScheme: sourceBaseline.DefaultTensorScheme, + allowDominantFallback: false); + + if (learned.Count == 0) + throw new InvalidOperationException( + $"Missing required learned baseline mapping for group '{hybrid.TGroup.Name}' + baseline '{sourceBaseline.Names[0]}'. Use targeted YAML relearn configuration to regenerate only the affected baseline/profile truth."); + + var learnedNames = learned.Keys.ToHashSet(StringComparer.Ordinal); + var missingExpected = expectedForGroup.Except(learnedNames).OrderBy(x => x).ToList(); + var unexpectedLearned = learnedNames.Except(expectedForGroup).OrderBy(x => x).ToList(); + + if (missingExpected.Count > 0 || unexpectedLearned.Count > 0) + { + var missingText = missingExpected.Count == 0 + ? "none" + : string.Join(", ", missingExpected.Take(15)); + var unexpectedText = unexpectedLearned.Count == 0 + ? "none" + : string.Join(", ", unexpectedLearned.Take(15)); + throw new InvalidOperationException( + $"Learned mapping coverage mismatch for group '{hybrid.TGroup.Name}' + baseline '{sourceBaseline.Names[0]}'. Expected={expectedForGroup.Count}, Learned={learnedNames.Count}, Missing=[{missingText}], Unexpected=[{unexpectedText}]."); + } + + foreach (var kv in learned.OrderBy(x => x.Key, StringComparer.Ordinal)) + { + result.Add(new RequestedTensorOverride + { + GroupName = hybrid.TGroup.Name, + TensorName = kv.Key, + SchemeName = kv.Value + }); + } + + break; + } + + default: + throw new InvalidOperationException( + $"Hybrid tensor for group '{hybrid.TGroup.Name}' has unsupported override mode '{hybrid.OverrideMode}'."); + } + } + + return result; + } + + private bool ShouldApplyLearnedBaseCarrierBlanket( + HybridQuant quant, + IReadOnlyDictionary? temporaryCarrierOverrides = null) + { + bool hasTemporaryCarrierOverrides = temporaryCarrierOverrides != null && temporaryCarrierOverrides.Count > 0; + bool hasExplicitGroupOverrides = quant.Tensors != null && quant.Tensors.Count > 0; + + return hasTemporaryCarrierOverrides || + quant.BaseQuant.IsExternalRepositoryBaseline || + hasExplicitGroupOverrides; + } + + private Dictionary LoadBaseCarrierTensorMappingsOrThrow( + HybridQuant quant, + IReadOnlyDictionary? temporaryCarrierOverrides, + bool requireFullCoverage) + { + var blanket = temporaryCarrierOverrides != null && temporaryCarrierOverrides.Count > 0 + ? new Dictionary(temporaryCarrierOverrides, StringComparer.Ordinal) + : TryLoadAllLearnedTensorMappings( + canonicalBaselineKey: quant.BaseQuant.CanonicalKey, + preferredSourceScheme: quant.BaseQuant.DefaultTensorScheme, + allowDominantFallback: false); + + if (requireFullCoverage && blanket.Count == 0) + { + throw new InvalidOperationException( + $"Missing full learned base-carrier mapping for baseline '{quant.BaseQuant.Names[0]}'. " + + "Use targeted YAML relearn configuration before applying learned tensor configurations."); + } + + return blanket; + } + + private Dictionary TryLoadAllLearnedTensorMappings( + string canonicalBaselineKey, + TensorWeightScheme? preferredSourceScheme = null, + bool allowDominantFallback = false) + { + using var db = new MagicQuantContext(); + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + var normalizedCanonicalKey = BaselineDefinitionResolver.NormalizeCanonicalKey(canonicalBaselineKey); + var baselineDefinitionId = db.BaselineQuantDefinitions + .AsNoTracking() + .Where(x => (x.ArchitectureFamilyId == architectureFamilyId || x.ArchitectureFamilyId == null) && + x.NormalizedCanonicalKey == normalizedCanonicalKey) + .OrderByDescending(x => x.ArchitectureFamilyId.HasValue) + .Select(x => (int?)x.Id) + .FirstOrDefault(); + + if (!baselineDefinitionId.HasValue) + return new Dictionary(StringComparer.Ordinal); + + var allRows = db.LearnedBaselineTensorQuants + .AsNoTracking() + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.BaselineQuantDefinitionId == baselineDefinitionId.Value) + .OrderBy(x => x.TensorName) + .ToList(); + + if (allRows.Count == 0) + return new Dictionary(StringComparer.Ordinal); + + var rows = allRows; + if (preferredSourceScheme != null) + { + var preferred = allRows.Where(x => x.TensorWeightSchemeId == preferredSourceScheme.UniqueId).ToList(); + if (preferred.Count > 0) + rows = preferred; + else if (!allowDominantFallback) + return new Dictionary(StringComparer.Ordinal); + } + + if (rows.Select(x => x.TensorWeightSchemeId).Distinct().Count() > 1) + { + if (!allowDominantFallback) + return new Dictionary(StringComparer.Ordinal); + + var dominantSchemeId = rows.GroupBy(x => x.TensorWeightSchemeId) + .OrderByDescending(g => g.Count()) + .ThenBy(g => g.Key) + .Select(g => g.Key) + .First(); + rows = rows.Where(x => x.TensorWeightSchemeId == dominantSchemeId).ToList(); + } + + var result = new Dictionary(StringComparer.Ordinal); + + foreach (var row in rows) + { + var appliedSchemeName = + NativePrecisionNormalization.NormalizeLearnedFinalQuantTypeForApplication(row.FinalQuantType); + + if (string.IsNullOrWhiteSpace(appliedSchemeName)) + { + throw new InvalidOperationException( + $"Learned tensor mapping for tensor '{row.TensorName}' on baseline key '{canonicalBaselineKey}' " + + $"returned an empty normalized scheme name. Observed FinalQuantType='{row.FinalQuantType}'."); + } + + result[row.TensorName] = appliedSchemeName; + } + + return result; + } + + private Dictionary TryLoadLearnedTensorMapping( + BaselineQuants sourceBaseline, + TensorGroup targetGroup, + TensorWeightScheme? preferredSourceScheme = null, + bool allowDominantFallback = false) + { + using var db = new MagicQuantContext(); + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + var baselineDefinition = BaselineDefinitionResolver.ResolveRequiredDefinitionAsync(db, sourceBaseline) + .GetAwaiter() + .GetResult(); + + var allRows = db.LearnedBaselineTensorQuants + .AsNoTracking() + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.BaselineQuantDefinitionId == baselineDefinition.Id && + x.TensorGroupId == targetGroup.UniqueId) + .OrderBy(x => x.TensorName) + .ToList(); + + if (allRows.Count == 0) + return new Dictionary(StringComparer.Ordinal); + + List rows = allRows; + + if (preferredSourceScheme != null) + { + var preferred = allRows + .Where(x => x.TensorWeightSchemeId == preferredSourceScheme.UniqueId) + .ToList(); + + if (preferred.Count > 0) + { + rows = preferred; + } + else if (!allowDominantFallback) + { + return new Dictionary(StringComparer.Ordinal); + } + } + + if (rows.Count == 0) + return new Dictionary(StringComparer.Ordinal); + + if (rows.Select(x => x.TensorWeightSchemeId).Distinct().Count() > 1) + { + if (!allowDominantFallback) + return new Dictionary(StringComparer.Ordinal); + + var dominantSchemeId = rows + .GroupBy(x => x.TensorWeightSchemeId) + .OrderByDescending(g => g.Count()) + .ThenBy(g => g.Key) + .Select(g => g.Key) + .First(); + + rows = rows.Where(x => x.TensorWeightSchemeId == dominantSchemeId).ToList(); + } + + var result = new Dictionary(StringComparer.Ordinal); + + foreach (var row in rows) + { + var appliedSchemeName = + NativePrecisionNormalization.NormalizeLearnedFinalQuantTypeForApplication(row.FinalQuantType); + + if (string.IsNullOrWhiteSpace(appliedSchemeName)) + { + throw new InvalidOperationException( + $"Learned tensor mapping for tensor '{row.TensorName}' in group '{targetGroup.Name}' " + + $"returned an empty normalized scheme name. Observed FinalQuantType='{row.FinalQuantType}'."); + } + + result[row.TensorName] = appliedSchemeName; + } + + return result; + } + + + private List ResolveConcreteTensorOverrides( + IReadOnlyCollection allTensorNames, + List requestedOverrides) + { + if (requestedOverrides.Count == 0) + return new List(); + + var nameSet = allTensorNames.ToHashSet(StringComparer.Ordinal); + + var missing = requestedOverrides + .Where(x => !nameSet.Contains(x.TensorName)) + .ToList(); + + if (missing.Count > 0) + { + throw new InvalidOperationException( + $"Required learned tensor mappings were missing in source GGUF ({missing.Count} tensors). Examples: {string.Join(", ", missing.Take(10).Select(x => x.TensorName))}"); + } + + var lastWins = new Dictionary(StringComparer.Ordinal); + foreach (var item in requestedOverrides) + lastWins[item.TensorName] = item; + + return lastWins.Values + .Select(x => new ConcreteTensorOverride + { + GroupName = x.GroupName, + SchemeName = x.SchemeName, + TensorName = x.TensorName + }) + .OrderBy(x => x.TensorName, StringComparer.Ordinal) + .ToList(); + } + + private async Task ReadTensorMetadataFromGgufAsync(string ggufPath, string workingDirectory) + { + return await _ggufMetadataReader.ReadAsync(ggufPath, workingDirectory); + } + + + private async Task> BuildIsolationDeduplicationPlanAsync( + IReadOnlyCollection plans, + CancellationToken ct) + { + var result = new Dictionary(StringComparer.Ordinal); + var firstBySignature = new Dictionary(StringComparer.Ordinal); + + foreach (var plan in plans) + { + string? signature = await TryBuildIsolationEquivalenceKeyAsync(plan, ct); + if (string.IsNullOrWhiteSpace(signature)) + { + result[plan.Key] = plan.Key; + continue; + } + + if (!firstBySignature.TryGetValue(signature, out var firstKey)) + { + firstBySignature[signature] = plan.Key; + result[plan.Key] = plan.Key; + continue; + } + + result[plan.Key] = firstKey; + AnsiConsole.MarkupLine( + $"[grey]Isolation dedupe planned:[/] {Markup.Escape(plan.Key)} -> {Markup.Escape(firstKey)}"); + } + + return result; + } + + private async Task TryBuildIsolationEquivalenceKeyAsync( + RequiredSamplePlan plan, + CancellationToken ct) + { + if (plan.Kind != RequiredSampleKind.GroupIsolationProbe && + plan.Kind != RequiredSampleKind.GroupIsolationContinuation) + return null; + + if (plan.TargetGroupId == null || string.IsNullOrWhiteSpace(plan.TestedCandidateCanonicalKey)) + return null; + + await using var db = new MagicQuantContext(); + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + var normalizedCanonicalKey = BaselineDefinitionResolver.NormalizeCanonicalKey(plan.TestedCandidateCanonicalKey); + var baselineDefinitionId = await db.BaselineQuantDefinitions + .AsNoTracking() + .Where(x => (x.ArchitectureFamilyId == architectureFamilyId || x.ArchitectureFamilyId == null) && + x.NormalizedCanonicalKey == normalizedCanonicalKey) + .OrderByDescending(x => x.ArchitectureFamilyId.HasValue) + .Select(x => (int?)x.Id) + .FirstOrDefaultAsync(ct); + if (!baselineDefinitionId.HasValue) + return null; + + var rows = await db.LearnedBaselineTensorQuants + .AsNoTracking() + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.BaselineQuantDefinitionId == baselineDefinitionId.Value) + .Where(x => x.TensorGroupId == plan.TargetGroupId.Value) + .OrderBy(x => x.TensorName) + .Select(x => new { x.TensorName, x.FinalQuantType }) + .ToListAsync(ct); + + if (rows.Count == 0) + return null; + + var sb = new StringBuilder(); + sb.Append("group=").Append(plan.TargetGroupId.Value).Append('|'); + foreach (var row in rows) + { + sb.Append(row.TensorName).Append('=') + .Append(NormalizeLearnedIsolationQuantToken(row.FinalQuantType)) + .Append(';'); + } + + return sb.ToString(); + } + + private static string NormalizeLearnedIsolationQuantToken(string value) + { + if (string.IsNullOrWhiteSpace(value)) + return string.Empty; + + return value.Trim().Replace(" ", string.Empty).Replace("-", "_").ToUpperInvariant(); + } + + private async Task CloneEquivalentIsolationBenchmarkAsync( + RequiredSamplePlan sourcePlan, + RequiredSamplePlan duplicatePlan, + CancellationToken ct) + { + var sourceIdentity = await ResolveBenchmarkIdentityAsync(sourcePlan.Quant, ct); + if (sourceIdentity.BenchmarkId == null) + return false; + + await using var db = new MagicQuantContext(); + var scopedAiModelHashId = await ResolveCurrentScopedAiModelHashIdOrNullAsync(db, ct); + if (scopedAiModelHashId == null) + return false; + + var sourceBench = await db.AiBenchmarks + .Include(x => x.CategorBenchmarks) + .FirstOrDefaultAsync(x => x.Id == sourceIdentity.BenchmarkId.Value, ct); + + if (sourceBench == null) + return false; + + var duplicateLookup = BuildTensorLookup(duplicatePlan.Quant); + var duplicateCombo = await db.TensorCombos.FirstOrDefaultAsync(x => + x.BaseQuant == duplicateLookup.BaseQuant && + x.Embeddings == duplicateLookup.Embeddings && + x.LmHead == duplicateLookup.LmHead && + x.AttnQ == duplicateLookup.AttnQ && + x.AttnKV == duplicateLookup.AttnKV && + x.AttnOutput == duplicateLookup.AttnOutput && + x.FfnUpGate == duplicateLookup.FfnUpGate && + x.FfnDown == duplicateLookup.FfnDown && + x.MoeExperts == duplicateLookup.MoeExperts && + x.MoeRouter == duplicateLookup.MoeRouter, ct); + + if (duplicateCombo == null) + { + duplicateCombo = new TensorCombo(duplicateLookup); + db.TensorCombos.Add(duplicateCombo); + await db.SaveChangesAsync(ct); + } + + var benchmarkAiModelHashId = scopedAiModelHashId.Value; + var imatrixDefinitionId = await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync( + db, + benchmarkAiModelHashId, + createIfMissing: true, + ct); + + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + + var existing = await db.AiBenchmarks + .FirstOrDefaultAsync(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == tensorGroupProfileId && + x.AiModelHashId == benchmarkAiModelHashId && + x.ImatrixDefinitionId == imatrixDefinitionId && + x.TensorComboId == duplicateCombo.Id, ct); + + if (existing != null) + return true; + + var clonedBenchmark = new AiBenchmark + { + Id = Guid.NewGuid(), + ArchitectureFamilyId = architectureFamilyId, + TensorGroupProfileId = tensorGroupProfileId, + Ngl = sourceBench.Ngl, + SizeBytes = sourceBench.SizeBytes, + TokensPerSecond = sourceBench.TokensPerSecond, + TensorComboId = duplicateCombo.Id, + AiModelHashId = benchmarkAiModelHashId, + ImatrixDefinitionId = imatrixDefinitionId + }; + db.AiBenchmarks.Add(clonedBenchmark); + + var clonedCategories = sourceBench.CategorBenchmarks + .Select(x => new CategoryBenchmark + { + Id = Guid.NewGuid(), + AiBenchmarkId = clonedBenchmark.Id, + Category = x.Category, + Kld = x.Kld, + Ppl = x.Ppl, + PplError = x.PplError + }) + .ToList(); + db.AddRange(clonedCategories); + + db.QuantizationRuns.Add(new QuantizationRun + { + Id = Guid.NewGuid(), + ArchitectureFamilyId = architectureFamilyId, + TensorGroupProfileId = tensorGroupProfileId, + AiModelHashId = benchmarkAiModelHashId, + ImatrixDefinitionId = imatrixDefinitionId, + TensorComboId = duplicateCombo.Id, + AiBenchmarkId = clonedBenchmark.Id, + StartedUtc = DateTime.UtcNow, + CompletedUtc = DateTime.UtcNow, + DurationMs = 0, + Succeeded = true, + Error = $"Cloned from equivalent isolation benchmark '{sourcePlan.Key}'.", + OutputModelPath = null + }); + + foreach (var cat in clonedCategories) + { + db.BenchmarkRuns.Add(new BenchmarkRun + { + Id = Guid.NewGuid(), + ArchitectureFamilyId = architectureFamilyId, + TensorGroupProfileId = tensorGroupProfileId, + AiModelHashId = benchmarkAiModelHashId, + ImatrixDefinitionId = imatrixDefinitionId, + TensorComboId = duplicateCombo.Id, + AiBenchmarkId = clonedBenchmark.Id, + CategoryBenchmarkId = cat.Id, + Category = cat.Category, + StartedUtc = DateTime.UtcNow, + CompletedUtc = DateTime.UtcNow, + DurationMs = 0, + Succeeded = true, + Error = $"Cloned from equivalent isolation benchmark '{sourcePlan.Key}'." + }); + } + + await db.SaveChangesAsync(ct); + + AnsiConsole.MarkupLine( + $"[green]Isolation dedupe clone:[/] {Markup.Escape(duplicatePlan.Key)} reused benchmark data from {Markup.Escape(sourcePlan.Key)}"); + return true; + } + + // ---------------------------------------------------------------- + // Internal DTOs + // ---------------------------------------------------------------- + + private static readonly Regex TensorLogLineRegex = new( + @"\]\s+(?[^\s]+)\s+-\s+\[[^\]]+\],\s+type\s*=\s*(?[^\s,]+)(?:.*?converting to\s+(?[^\s,]+))?", + RegexOptions.Compiled | RegexOptions.CultureInvariant); + + private static string NormalizeQuantName(string value) + { + if (string.IsNullOrWhiteSpace(value)) + return "UNKNOWN"; + + string token = CanonicalizeQuantToken(value); + + if (QuantAliasLookup.Value.TryGetValue(token, out var canonical)) + return canonical; + + return token; + } + + private static Dictionary BuildQuantAliasLookup() + { + var map = new Dictionary(StringComparer.Ordinal); + + foreach (var scheme in TensorWeightScheme.All) + { + if (scheme.Names.IsDefaultOrEmpty) + continue; + + string canonical = scheme.Names[0]; + + foreach (var alias in scheme.Names) + { + string token = CanonicalizeQuantToken(alias); + + if (!map.TryAdd(token, canonical)) + { + if (!string.Equals(map[token], canonical, StringComparison.Ordinal)) + { + throw new InvalidOperationException( + $"Quant alias collision detected for token '{token}'. Existing='{map[token]}', New='{canonical}'."); + } + } + } + } + + return map; + } + + private static string CanonicalizeQuantToken(string value) + { + return value + .Trim() + .Replace("-", "_") + .Replace(" ", string.Empty) + .ToUpperInvariant(); + } + + private static bool IsHighPrecisionType(string value) + { + string normalized = NormalizeQuantName(value); + return normalized is "BF16" or "F16" or "F32"; + } + + private sealed class PreparedExternalBaselineBuild + { + public string BenchmarkModelPath { get; set; } = string.Empty; + public string? DownloadedExternalModelPath { get; set; } + public Dictionary? TruthByTensor { get; set; } + public IReadOnlyDictionary? GroupedByTensor { get; set; } + public IReadOnlyCollection? AllTensorNamesInDownloadedArtifact { get; set; } + public IReadOnlyList? AmbiguousGroupingRows { get; set; } + public IReadOnlyList? UnresolvedTensorNames { get; set; } + public IReadOnlyList? BaseQuantExceptionRows { get; set; } + public TensorTruthVerificationResult? Verification { get; set; } + public bool HasPreparedLearningTruth { get; set; } + } + + private sealed class QuantizationExecutionReport + { + public string LogPath { get; set; } = string.Empty; + public List ResolvedOverrides { get; set; } = new(); + } + + private sealed class RequestedTensorOverride + { + public string GroupName { get; set; } = string.Empty; + public string TensorName { get; set; } = string.Empty; + public string SchemeName { get; set; } = string.Empty; + } + + private sealed class ConcreteTensorOverride + { + public string TensorName { get; set; } = string.Empty; + public string SchemeName { get; set; } = string.Empty; + public string GroupName { get; set; } = string.Empty; + } + + // ---------------------------------------------------------------- + // Naming helpers + // ---------------------------------------------------------------- + + private static string ResolveBaseName(BaselineQuants b) + { + if (b.Names.IsDefaultOrEmpty) + throw new InvalidOperationException($"BaselineQuants '{b.UniqueId}' has no Names."); + + return b.Names[0]; + } + + private static string ResolveSchemeName(TensorWeightScheme s) + { + if (s.Names.IsDefaultOrEmpty) + throw new InvalidOperationException($"TensorWeightScheme '{s.UniqueId}' has no Names."); + + return s.Names[0]; + } + + public string GenerateHybridName(HybridQuant quant) + { + string modelName = new DirectoryInfo(Cache.ModelDirectory!).Name; + string baseName = ResolveBaseName(quant.BaseQuant); + + var effectiveTensors = quant.Tensors? + .Where(t => t?.TGroup != null) + .ToList(); + + if (effectiveTensors == null || effectiveTensors.Count == 0) + return $"{modelName}-{baseName}"; + + var grouped = effectiveTensors + .Select(t => + { + t.ValidateOrThrow(); + + string typeName = t.OverrideMode switch + { + HybridTensorOverrideMode.LearnedBaselineCandidate => t.CandidateBaseline!.Names[0], + HybridTensorOverrideMode.ExactTensorScheme => ResolveSchemeName(t.ExactTensorScheme!), + _ => throw new InvalidOperationException($"Unknown override mode '{t.OverrideMode}'.") + }; + + return new + { + Type = typeName, + Code = t.TGroup.ShortCode + }; + }) + .GroupBy(x => x.Type) + .Select(g => new + { + Type = g.Key, + Codes = g.Select(x => x.Code) + .OrderBy(c => GetOrder(c)) + .ToArray() + }) + .OrderBy(x => GetOrder(x.Codes.FirstOrDefault())) + .ToList(); + + var nameParts = new List(grouped.Count); + + foreach (var group in grouped) + { + string codeStr = new string(group.Codes); + string quantStr = SimplifyQuant(group.Type); + nameParts.Add($"{codeStr}-{quantStr}"); + } + + string suffix = string.Join("-", nameParts); + return $"{modelName}-{baseName}-{suffix}"; + } + + private int GetOrder(char c) + { + return "EHQKOUDXR".IndexOf(c); + } + + private string SimplifyQuant(string quant) + { + return quant.Replace("_", "") + .Replace("BF16", "B16") + .Replace("F16", "F16") + .Replace("F32", "F32"); + } + + private sealed class LoggedProcessResult + { + public int ExitCode { get; init; } + public string StdOut { get; init; } = string.Empty; + public string StdErr { get; init; } = string.Empty; + } + + private async Task RunLoggedProcessAsync( + ProcessStartInfo psi, string? logPath, CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + var result = await new MagicQuant.Runtime.ProcessRunner().RunAsync(psi, logPath, + (line, _) => { if (Cache.VerboseProcessOutput) AnsiConsole.WriteLine(line); }, ct); + return new LoggedProcessResult { ExitCode = result.ExitCode, StdOut = result.StdOut, StdErr = result.StdErr }; + } +} diff --git a/src/MagicQuant/Services/RankSafeKldPredictionService.cs b/src/MagicQuant/Services/RankSafeKldPredictionService.cs new file mode 100644 index 0000000..3aad442 --- /dev/null +++ b/src/MagicQuant/Services/RankSafeKldPredictionService.cs @@ -0,0 +1,1159 @@ +using MagicQuant.Models; +using MQ.DB; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; +using Spectre.Console; + +namespace MagicQuant.Services; + +/// +/// Central prediction authority for MagicQuant's isolation-truth KLD estimator. +/// +/// This intentionally replaces the old MDA / bucket-survival prediction path. +/// It combines: +/// - Q8-carrier, native-exact blanket, single-group isolation measurements +/// - additive isolation KLD +/// - bit-stress interaction correction +/// - rank-safe isotonic projection over the additive backbone +/// +public sealed class RankSafeKldPredictionService +{ + private readonly HybridBenchmarkRepository _repository; + private readonly EffectiveCandidateStateResolverService _effectiveResolver; + + public RankSafeKldPredictionService( + HybridBenchmarkRepository repository, + EffectiveCandidateStateResolverService effectiveResolver) + { + _repository = repository; + _effectiveResolver = effectiveResolver; + } + + internal async Task BuildModelAsync(CancellationToken ct = default) + { + var context = await BuildContextAsync(ct); + var fitRows = await LoadFitRowsAsync(context, Array.Empty(), ct); + var fit = FitInteractionModel(fitRows, context); + + var notes = new List(context.Notes) + { + $"Prediction fit rows: {fit.FitRowCount:N0}; alpha={fit.Alpha:G6}; beta={fit.Beta:G6}; bit-stress-threshold={fit.BitStressThreshold:G4}; fallback={fit.UsedFallback}." + }; + + context.Fit = fit; + context.Notes = notes; + return context; + } + + public async Task PredictAsync( + IReadOnlyCollection configs, + CancellationToken ct = default) + { + if (configs == null) + throw new ArgumentNullException(nameof(configs)); + + var context = await BuildModelAsync(ct); + var uniqueConfigs = configs + .DistinctBy(TensorConfigIdentity.ToKey) + .ToList(); + + var rows = new List(uniqueConfigs.Count); + + foreach (var config in uniqueConfigs) + { + ct.ThrowIfCancellationRequested(); + var row = await PredictSingleAsync(config, context, ct); + rows.Add(row); + } + + foreach (var row in rows.Where(x => x.IsPredictable)) + { + row.CrossTerm = ComputeCrossTerm(row.Config, context, context.Fit.BitStressThreshold); + row.InteractionKld = Math.Max(0d, (context.Fit.Alpha * row.AdditiveKld) + (context.Fit.Beta * row.CrossTerm)); + } + + ApplyRankSafeProjection(rows); + + PrintPredictionDiagnostics(rows, context.Fit); + return new RankSafePredictionSet + { + Rows = rows + .OrderBy(x => x.PredictedKld) + .ThenBy(x => x.IsSizePredictable ? 0 : 1) + .ThenBy(x => x.PredictedSizeBytes) + .ToList(), + Fit = context.Fit, + Notes = context.Notes + }; + } + + private async Task PredictSingleAsync( + TensorConfig config, + RankSafePredictionModel context, + CancellationToken ct) + { + var quant = (HybridQuant)config; + var effective = await _effectiveResolver.ResolveAsync(config, ct); + + var row = new RankSafePredictionRow + { + Config = config, + Quant = quant, + IsPureBaseline = TensorConfigIdentity.IsPureBaseline(config), + EffectiveStateKey = effective.EffectiveStateKey, + HasUnknownMappings = effective.HasUnknownMappings, + Notes = effective.Warnings.ToList() + }; + + if (row.IsPureBaseline) + { + if (context.PureSnapshotsByBaselineId.TryGetValue(config.BaseQuant, out var pureDirect)) + { + row.PredictedSizeBytes = pureDirect.SizeBytes; + row.AdditiveKld = pureDirect.Kld; + row.InteractionKld = pureDirect.Kld; + row.PredictedKld = pureDirect.Kld; + row.PredictedPpl = pureDirect.Ppl; + return row; + } + + GuardAgainstDisabledPureBaselineSurrogateFallback(config.BaseQuant, context, row.Notes); + } + + row.PredictedSizeBytes = PredictSize(config, context, row.Notes, out bool canPredictSize); + row.IsSizePredictable = canPredictSize; + row.AdditiveKld = PredictAdditiveKld(config, context, row.Notes, out bool canPredict); + row.PredictedPpl = PredictPpl(config, context, row.Notes); + + if (!canPredict) + { + row.IsPredictable = false; + row.InteractionKld = double.PositiveInfinity; + row.PredictedKld = double.PositiveInfinity; + return row; + } + + row.InteractionKld = row.AdditiveKld; + row.PredictedKld = row.AdditiveKld; + return row; + } + + private async Task BuildContextAsync(CancellationToken ct) + { + var activeGroups = TReg.All + .Where(x => !Cache.UnusedTensorGroups.Any(u => u.UniqueId == x.UniqueId)) + .OrderBy(x => x.UniqueId) + .ToList(); + + if (activeGroups.Count == 0) + throw new InvalidOperationException("No active tensor groups were available for prediction."); + + var notes = new List(); + var pureSnapshots = await _repository.LoadPureBaselineSnapshotsAsync(ct); + var pureByBaselineId = pureSnapshots + .GroupBy(x => x.Quant.BaseQuant.UniqueId) + .ToDictionary( + g => g.Key, + g => g.OrderBy(x => x.Kld).ThenBy(x => x.SizeBytes).First()); + + if (!pureByBaselineId.TryGetValue(BaselineQuants.Q8_0.UniqueId, out var pureQ8)) + throw new InvalidOperationException("Rank-safe prediction requires a pure Q8_0 benchmark snapshot."); + + var nativeExactScheme = TensorWeightScheme.GetCurrentNativePrecisionScheme(); + var q8BaseOnlyQuant = HybridQuant.CreateExactBlanket( + baseQuant: BaselineQuants.Q8_0, + groups: activeGroups, + exactScheme: nativeExactScheme); + + var q8BaseOnly = await _repository.LoadBenchmarkSnapshotAsync((TensorConfig)q8BaseOnlyQuant, ct); + if (q8BaseOnly == null) + { + /* + * Deprecated fallback, intentionally disabled: + * + * notes.Add("Q8 native-exact base-only anchor was missing. Size fallback will use pure Q8..."); + * q8BaseOnly = pureQ8; + * + * Base-only anchors define the additive size coordinate system. Falling back to a pure + * Q8 model hides missing isolation truth and can flatten external/custom size geometry. + */ + throw new InvalidOperationException( + "Rank-safe prediction requires the Q8_0 native-exact base-only anchor. " + + "The old pure-Q8 fallback is intentionally disabled; generate the missing base-only isolation sample instead."); + } + + var baseOnlyByBaselineId = new Dictionary + { + [BaselineQuants.Q8_0.UniqueId] = q8BaseOnly + }; + + foreach (var baseline in BaselineQuants.GetAllRecognizedBaselines() + .Where(x => !BaselineQuants.IsNativeExactAlias(x.UniqueId)) + .OrderBy(x => x.UniqueId)) + { + if (baseOnlyByBaselineId.ContainsKey(baseline.UniqueId)) + continue; + + var directBaseOnlyQuant = HybridQuant.CreateExactBlanket( + baseQuant: baseline, + groups: activeGroups, + exactScheme: nativeExactScheme); + + var directBaseOnlySnapshot = await _repository.LoadBenchmarkSnapshotAsync((TensorConfig)directBaseOnlyQuant, ct); + if (directBaseOnlySnapshot != null) + { + baseOnlyByBaselineId[baseline.UniqueId] = directBaseOnlySnapshot; + continue; + } + + if (TryGetDisabledSurrogateBaselineId(baseline.UniqueId, out var disabledSurrogateId) && + baseOnlyByBaselineId.ContainsKey(disabledSurrogateId)) + { + notes.Add( + $"Missing exact base-only anchor for external baseline {FormatBaselineForNote(baseline.UniqueId)} (id '{baseline.UniqueId}'). " + + $"A normalized surrogate {FormatBaselineForNote(disabledSurrogateId)} (id '{disabledSurrogateId}') exists, but surrogate base-size fallback is intentionally disabled."); + } + } + + var isolationByGroupAndBaseline = new Dictionary<(byte GroupId, byte BaselineId), BenchmarkSnapshotRecord>(); + + foreach (var group in activeGroups) + { + foreach (var baseline in BaselineQuants.GetAllRecognizedBaselines() + .Where(x => !BaselineQuants.IsNativeExactAlias(x.UniqueId)) + .OrderBy(x => x.UniqueId)) + { + if (isolationByGroupAndBaseline.ContainsKey((group.UniqueId, baseline.UniqueId))) + continue; + + var isolationQuant = HybridQuant.CreateExactBlanket( + baseQuant: BaselineQuants.Q8_0, + groups: activeGroups, + exactScheme: nativeExactScheme); + + isolationQuant.SetLearnedCandidateOverride(group, baseline); + var snapshot = await _repository.LoadBenchmarkSnapshotAsync((TensorConfig)isolationQuant, ct); + + if (snapshot != null) + { + isolationByGroupAndBaseline[(group.UniqueId, baseline.UniqueId)] = snapshot; + continue; + } + + if (TryGetDisabledSurrogateBaselineId(baseline.UniqueId, out var disabledSurrogateId) && + isolationByGroupAndBaseline.ContainsKey((group.UniqueId, disabledSurrogateId))) + { + notes.Add( + $"Missing exact isolation snapshot for group '{group.Name}' and external baseline {FormatBaselineForNote(baseline.UniqueId)} (id '{baseline.UniqueId}'). " + + $"A normalized surrogate {FormatBaselineForNote(disabledSurrogateId)} (id '{disabledSurrogateId}') exists, but surrogate isolation fallback is intentionally disabled."); + } + } + } + + var missingQ8IsolationGroups = activeGroups + .Where(group => !isolationByGroupAndBaseline.ContainsKey((group.UniqueId, BaselineQuants.Q8_0.UniqueId))) + .Select(group => group.Name) + .ToList(); + + if (missingQ8IsolationGroups.Count > 0) + { + foreach (var groupName in missingQ8IsolationGroups) + { + notes.Add($"Missing KLD isolation snapshot for group '{groupName}' and baseline Q8_0. Q8_0 is a quantized state, not native truth; prediction will not silently fall back to zero for this group."); + } + } + else + { + notes.Add($"Q8_0 isolation snapshots loaded for {activeGroups.Count:N0} active tensor groups. Q8_0 will contribute measured prediction-space KLD, not zero/native damage."); + } + + var isolationDominanceBitTruthByGroupAndBaseline = BuildIsolationDominanceBitTruthOverrides( + activeGroups, + isolationByGroupAndBaseline, + notes); + + AppendExternalCoverageDiagnostics(notes, activeGroups, pureByBaselineId, baseOnlyByBaselineId, isolationByGroupAndBaseline); + + return new RankSafePredictionModel( + activeGroups: activeGroups, + pureQ8: pureQ8, + q8BaseOnly: q8BaseOnly, + pureSnapshotsByBaselineId: pureByBaselineId, + baseOnlySnapshotsByBaselineId: baseOnlyByBaselineId, + isolationByGroupAndBaseline: isolationByGroupAndBaseline, + isolationDominanceBitTruthByGroupAndBaseline: isolationDominanceBitTruthByGroupAndBaseline, + notes: notes); + } + + private async Task> LoadFitRowsAsync( + RankSafePredictionModel context, + IReadOnlyList alreadyPredicted, + CancellationToken ct) + { + var allBenchmarkRows = await _repository.LoadAllBenchmarkSnapshotsForCurrentContextAsync( + category: (byte)BenchmarkCategory.General, + strictImatrixContext: true, + ct: ct); + + var alreadyByKey = alreadyPredicted.ToDictionary(x => TensorConfigIdentity.ToKey(x.Config), StringComparer.Ordinal); + var fitRows = new List(); + var skippedFitReasons = new HashSet(StringComparer.Ordinal); + + foreach (var snapshot in allBenchmarkRows) + { + ct.ThrowIfCancellationRequested(); + + /* + * Keep the rank-safe predictor entirely in prediction space. + * + * Real pure baselines such as UD-Q6_K_XL can appear in the benchmark table + * as BaseQuant=UD-Q6_K_XL with NULL group slots. That shape is a real artifact + * identity, not a prediction-space base-only anchor. The prediction coordinate + * system is still the Q8_0 carrier plus exact isolated group overrides, so pure + * baselines are canonicalized to the virtual all-groups row before prediction: + * + * real pure UD-Q6_K_XL -> Q8_0 carrier with every active group = UD-Q6_K_XL + * real pure Q6_K -> Q8_0 carrier with every active group = Q6_K + * + * The real snapshot.Kld remains the fit target. Only the config used to produce + * the additive/cross-term prediction is canonicalized. This preserves the hard + * separation between real benchmark truth and synthetic prediction geometry. + */ + if (!TryCanonicalizeBenchmarkSnapshotConfigForPrediction( + snapshot.Config, + context, + out var predictionConfig, + out var skipReason)) + { + /* + * This row is real benchmark truth, but it is not representable in the + * rank-safe prediction coordinate system. Do not throw here: old runs and + * helper paths can leave real/external-base synthetic artifacts in SQLite + * even though DuckDB prediction-space candidates always use the Q8_0 + * carrier. Those rows are simply not fit observations for the synthetic + * model. + */ + if (!string.IsNullOrWhiteSpace(skipReason)) + skippedFitReasons.Add(skipReason); + + continue; + } + + var predictionKey = TensorConfigIdentity.ToKey(predictionConfig); + + RankSafePredictionRow predicted; + if (!alreadyByKey.TryGetValue(predictionKey, out predicted!)) + { + predicted = await PredictSingleAsync(predictionConfig, context, ct); + } + + if (!predicted.IsPredictable || double.IsInfinity(predicted.AdditiveKld) || double.IsNaN(predicted.AdditiveKld)) + continue; + + fitRows.Add(new FitObservation + { + Config = predictionConfig, + ActualKld = Math.Max(0d, snapshot.Kld), + AdditiveKld = predicted.AdditiveKld + }); + } + + if (skippedFitReasons.Count > 0) + context.Notes = context.Notes.Concat(skippedFitReasons.OrderBy(x => x, StringComparer.Ordinal)).ToList(); + + return fitRows; + } + + private static bool TryCanonicalizeBenchmarkSnapshotConfigForPrediction( + TensorConfig config, + RankSafePredictionModel context, + out TensorConfig predictionConfig, + out string? skipReason) + { + predictionConfig = config; + skipReason = null; + + if (config.BaseQuant == BaselineQuants.Q8_0.UniqueId) + return true; + + var baseline = BaselineQuants.FromId(config.BaseQuant); + if (BaselineQuants.IsNativeExactAlias(baseline.UniqueId)) + return true; + + if (TensorConfigIdentity.IsPureBaseline(config) && + !context.PureSnapshotsByBaselineId.ContainsKey(baseline.UniqueId)) + { + throw new InvalidOperationException( + $"Rank-safe prediction fit encountered pure baseline {baseline.Names[0]} (id '{baseline.UniqueId}'), but the pure benchmark snapshot was not loaded into context. " + + "This is a critical truth-loading error, not a soft warning."); + } + + byte inheritedBaseSlot = BaselineQuants.EncodeTensorConfigGroupSlot(baseline); + + predictionConfig = new TensorConfig( + baseQuant: BaselineQuants.Q8_0.UniqueId, + embeddings: CanonicalizePredictionSlot(TReg.Embeddings, config.Embeddings, inheritedBaseSlot, context), + lmHead: CanonicalizePredictionSlot(TReg.LmHead, config.LmHead, inheritedBaseSlot, context), + attnQ: CanonicalizePredictionSlot(TReg.AttnQ, config.AttnQ, inheritedBaseSlot, context), + attnKV: CanonicalizePredictionSlot(TReg.AttnKV, config.AttnKV, inheritedBaseSlot, context), + attnOutput: CanonicalizePredictionSlot(TReg.AttnOutput, config.AttnOutput, inheritedBaseSlot, context), + ffnUpGate: CanonicalizePredictionSlot(TReg.FfnUpGate, config.FfnUpGate, inheritedBaseSlot, context), + ffnDown: CanonicalizePredictionSlot(TReg.FfnDown, config.FfnDown, inheritedBaseSlot, context), + moeExperts: CanonicalizePredictionSlot(TReg.MoeExperts, config.MoeExperts, inheritedBaseSlot, context), + moeRouter: CanonicalizePredictionSlot(TReg.MoeRouter, config.MoeRouter, inheritedBaseSlot, context)); + + return true; + } + + private static byte CanonicalizePredictionSlot( + TensorGroup group, + byte storedValue, + byte inheritedBaseSlot, + RankSafePredictionModel context) + { + if (!context.ActiveGroups.Any(x => x.UniqueId == group.UniqueId)) + return BaselineQuants.TensorConfigNullSlotValue; + + return BaselineQuants.IsNullTensorConfigGroupSlot(storedValue) + ? inheritedBaseSlot + : storedValue; + } + + private RankSafePredictionFit FitInteractionModel( + IReadOnlyList observations, + RankSafePredictionModel context) + { + var usable = observations + .Where(x => x.ActualKld >= 0d) + .Where(x => !double.IsNaN(x.AdditiveKld) && !double.IsInfinity(x.AdditiveKld)) + .ToList(); + + if (usable.Count < Math.Max(3, Config.PredictionMinimumFitRows)) + { + return new RankSafePredictionFit + { + Alpha = 1.0d, + Beta = 0.0d, + BitStressThreshold = Config.PredictionDefaultBitStressThreshold, + FitRowCount = usable.Count, + UsedFallback = true + }; + } + + RankSafePredictionFit? best = null; + + foreach (double threshold in Config.PredictionBitStressThresholdCandidates) + { + double s11 = 0d; + double s12 = 0d; + double s22 = 0d; + double y1 = 0d; + double y2 = 0d; + + var crossTerms = new Dictionary(StringComparer.Ordinal); + foreach (var row in usable) + { + double x1 = row.AdditiveKld; + double x2 = ComputeCrossTerm(row.Config, context, threshold); + double y = row.ActualKld; + + s11 += x1 * x1; + s12 += x1 * x2; + s22 += x2 * x2; + y1 += x1 * y; + y2 += x2 * y; + crossTerms[TensorConfigIdentity.ToKey(row.Config)] = x2; + } + + double det = (s11 * s22) - (s12 * s12); + double alpha; + double beta; + + if (Math.Abs(det) <= 1e-18d) + { + alpha = s11 <= 1e-18d ? 1.0d : y1 / s11; + beta = 0.0d; + } + else + { + alpha = ((y1 * s22) - (y2 * s12)) / det; + beta = ((s11 * y2) - (s12 * y1)) / det; + } + + if (double.IsNaN(alpha) || double.IsInfinity(alpha)) + alpha = 1.0d; + + if (double.IsNaN(beta) || double.IsInfinity(beta)) + beta = 0.0d; + + // Keep the correction sane. The fit can be noisy when the only benchmarked + // rows are pure baselines and isolation probes. + alpha = Math.Clamp(alpha, 0.05d, 10.0d); + beta = Math.Clamp(beta, -1_000_000d, 1_000_000d); + + double mae = usable + .Select(x => + { + double cross = crossTerms[TensorConfigIdentity.ToKey(x.Config)]; + double pred = Math.Max(0d, (alpha * x.AdditiveKld) + (beta * cross)); + return Math.Abs(pred - x.ActualKld); + }) + .Average(); + + var candidate = new RankSafePredictionFit + { + Alpha = alpha, + Beta = beta, + BitStressThreshold = threshold, + FitRowCount = usable.Count, + FitMae = mae, + UsedFallback = false + }; + + if (best == null || candidate.FitMae < best.FitMae) + best = candidate; + } + + return best ?? new RankSafePredictionFit + { + Alpha = 1.0d, + Beta = 0.0d, + BitStressThreshold = Config.PredictionDefaultBitStressThreshold, + FitRowCount = usable.Count, + UsedFallback = true + }; + } + + private static void ApplyRankSafeProjection(IReadOnlyList rows) + { + var predictable = rows + .Where(x => x.IsPredictable) + .OrderBy(x => x.AdditiveKld) + .ThenBy(x => x.InteractionKld) + .ThenBy(x => x.PredictedSizeBytes) + .ToList(); + + if (predictable.Count == 0) + return; + + double[] projected = Pava(predictable.Select(x => x.InteractionKld).ToArray()); + + for (int i = 0; i < predictable.Count; i++) + predictable[i].PredictedKld = Math.Max(0d, projected[i]); + + ulong rank = 1; + foreach (var row in rows + .Where(x => x.IsPredictable) + .OrderBy(x => x.PredictedKld) + .ThenBy(x => x.PredictedSizeBytes)) + { + row.PredictedRank = rank++; + } + } + + private static double[] Pava(double[] values) + { + var blocks = new List(); + + foreach (double value in values) + { + blocks.Add(new PavaBlock { Sum = value, Weight = 1d, Count = 1 }); + + while (blocks.Count >= 2) + { + var right = blocks[^1]; + var left = blocks[^2]; + + if (left.Mean <= right.Mean) + break; + + left.Sum += right.Sum; + left.Weight += right.Weight; + left.Count += right.Count; + blocks[^2] = left; + blocks.RemoveAt(blocks.Count - 1); + } + } + + var result = new double[values.Length]; + int index = 0; + foreach (var block in blocks) + { + double mean = block.Mean; + for (int i = 0; i < block.Count; i++) + result[index++] = mean; + } + + return result; + } + + private double PredictAdditiveKld( + TensorConfig config, + RankSafePredictionModel context, + List notes, + out bool canPredict) + { + canPredict = true; + double total = 0d; + + foreach (var (group, effectiveBaselineId) in EnumerateEffectiveBaselines(config, context.ActiveGroups)) + { + if (IsZeroDamageAlias(effectiveBaselineId)) + continue; + + if (!TryResolveIsolationBaselineForPrediction(group, effectiveBaselineId, context, notes, out var resolved)) + { + notes.Add(BuildMissingIsolationNote(group, effectiveBaselineId)); + canPredict = false; + continue; + } + + total += Math.Max(0d, resolved.Snapshot.Kld); + } + + return Math.Max(0d, total); + } + + private double PredictPpl( + TensorConfig config, + RankSafePredictionModel context, + List notes) + { + double total = 0d; + + foreach (var (group, effectiveBaselineId) in EnumerateEffectiveBaselines(config, context.ActiveGroups)) + { + if (IsZeroDamageAlias(effectiveBaselineId)) + continue; + + if (TryResolveIsolationBaselineForPrediction(group, effectiveBaselineId, context, notes, out var resolved)) + total += resolved.Snapshot.Ppl; + else + notes.Add(BuildMissingIsolationNote(group, effectiveBaselineId)); + } + + return total; + } + + private ulong PredictSize( + TensorConfig config, + RankSafePredictionModel context, + List notes, + out bool canPredictSize) + { + canPredictSize = true; + + if (!TryResolveBaseOnlySnapshotForPrediction(config.BaseQuant, context, notes, out var baseOnlyAnchor)) + { + notes.Add($"Missing base-only size anchor for base baseline {FormatBaselineForNote(config.BaseQuant)} (id '{config.BaseQuant}'). Size prediction is not safe for selection."); + canPredictSize = false; + return 0; + } + + long total = (long)baseOnlyAnchor.SizeBytes; + long q8ExactBlanketSize = (long)context.Q8BaseOnly.SizeBytes; + + foreach (var (group, effectiveBaselineId) in EnumerateEffectiveBaselines(config, context.ActiveGroups)) + { + // Base-only anchors already hold every active group at native exact precision. + // Exact aliases therefore contribute no size delta. + if (BaselineQuants.IsNativeExactAlias(effectiveBaselineId)) + continue; + + if (!TryResolveIsolationBaselineForPrediction(group, effectiveBaselineId, context, notes, out var resolved)) + { + notes.Add($"Missing group size-isolation snapshot for group '{group.Name}' and effective baseline {FormatBaselineForNote(effectiveBaselineId)} (id '{effectiveBaselineId}'). Size prediction is not safe for selection."); + canPredictSize = false; + continue; + } + + total += (long)resolved.Snapshot.SizeBytes - q8ExactBlanketSize; + } + + if (total <= 0) + { + notes.Add($"Predicted size collapsed to {total:N0} bytes. Size prediction is not safe for selection."); + canPredictSize = false; + return 0; + } + + return (ulong)total; + } + + private static Dictionary<(byte GroupId, byte BaselineId), double> BuildIsolationDominanceBitTruthOverrides( + IReadOnlyList activeGroups, + Dictionary<(byte GroupId, byte BaselineId), BenchmarkSnapshotRecord> isolationByGroupAndBaseline, + List notes) + { + const double kldEpsilon = 1e-12; + + var result = new Dictionary<(byte GroupId, byte BaselineId), double>(); + var detailNotes = new List(); + var baselinesById = BaselineQuants.GetAllRecognizedBaselines() + .Where(x => !BaselineQuants.IsNativeExactAlias(x.UniqueId)) + .GroupBy(x => x.UniqueId) + .ToDictionary(g => g.Key, g => g.First()); + + foreach (var group in activeGroups.OrderBy(x => x.UniqueId)) + { + var entries = isolationByGroupAndBaseline + .Where(x => x.Key.GroupId == group.UniqueId && baselinesById.ContainsKey(x.Key.BaselineId)) + .Select(x => new IsolationBitTruthEntry( + Baseline: baselinesById[x.Key.BaselineId], + Snapshot: x.Value, + DeclaredBitRange: (double)baselinesById[x.Key.BaselineId].BitRange)) + .OrderByDescending(x => x.DeclaredBitRange) + .ThenBy(x => x.Snapshot.Kld) + .ThenBy(x => x.Snapshot.SizeBytes) + .ToList(); + + foreach (var candidate in entries) + { + double inheritedBitTruth = candidate.DeclaredBitRange; + IsolationBitTruthEntry? strongestVictim = null; + + foreach (var victim in entries) + { + if (victim.DeclaredBitRange <= candidate.DeclaredBitRange) + continue; + + bool sameSizeOrSmaller = candidate.Snapshot.SizeBytes <= victim.Snapshot.SizeBytes; + bool lowerKld = candidate.Snapshot.Kld < victim.Snapshot.Kld - kldEpsilon; + if (!sameSizeOrSmaller || !lowerKld) + continue; + + if (victim.DeclaredBitRange > inheritedBitTruth) + { + inheritedBitTruth = victim.DeclaredBitRange; + strongestVictim = victim; + } + } + + if (inheritedBitTruth <= candidate.DeclaredBitRange) + continue; + + result[(group.UniqueId, candidate.Baseline.UniqueId)] = inheritedBitTruth; + + if (strongestVictim != null && detailNotes.Count < 32) + { + detailNotes.Add( + $"Isolation bit-truth override: group '{group.Name}' treats {candidate.Baseline.Names[0]} as {inheritedBitTruth:G4}b stress truth instead of {candidate.DeclaredBitRange:G4}b because it isolated-dominated higher-fidelity {strongestVictim.Baseline.Names[0]} (candidate size={candidate.Snapshot.SizeBytes:N0}, kld={candidate.Snapshot.Kld:0.######}; victim size={strongestVictim.Snapshot.SizeBytes:N0}, kld={strongestVictim.Snapshot.Kld:0.######})."); + } + } + } + + if (result.Count == 0) + { + notes.Add("Isolation bit-truth overrides: none. Declared quant bit ranges will drive bit-stress interaction correction."); + return result; + } + + notes.Add( + $"Isolation bit-truth overrides active: {result.Count:N0} group/baseline state(s) inherit higher-fidelity stress truth because isolated sampling showed same-size-or-smaller lower-KLD dominance."); + + foreach (var detail in detailNotes) + notes.Add(detail); + + if (result.Count > detailNotes.Count) + notes.Add($"Isolation bit-truth overrides: {result.Count - detailNotes.Count:N0} additional override(s) omitted from diagnostics."); + + return result; + } + + internal static double GetStressBitRangeForPrediction( + TensorGroup group, + byte baselineId, + RankSafePredictionModel context) + { + if (IsZeroDamageAlias(baselineId)) + return 99d; + + double declared = BaselineQuants.FromId(baselineId).BitRange; + return context.IsolationDominanceBitTruthByGroupAndBaseline.TryGetValue((group.UniqueId, baselineId), out var inherited) + ? Math.Max(declared, inherited) + : declared; + } + + private double ComputeCrossTerm(TensorConfig config, RankSafePredictionModel context, double threshold) + { + var contributions = new List<(double Kld, double Bits)>(); + + foreach (var (group, effectiveBaselineId) in EnumerateEffectiveBaselines(config, context.ActiveGroups)) + { + if (IsZeroDamageAlias(effectiveBaselineId)) + continue; + + if (!TryResolveIsolationBaselineForPrediction(group, effectiveBaselineId, context, notes: null, out var resolved)) + continue; + + double stressBitRange = GetStressBitRangeForPrediction(group, resolved.BaselineId, context); + contributions.Add((Math.Max(0d, resolved.Snapshot.Kld), stressBitRange)); + } + + double cross = 0d; + for (int i = 0; i < contributions.Count; i++) + { + for (int j = i + 1; j < contributions.Count; j++) + { + double stressI = Math.Max(0d, threshold - contributions[i].Bits); + double stressJ = Math.Max(0d, threshold - contributions[j].Bits); + if (stressI <= 0d || stressJ <= 0d) + continue; + + cross += contributions[i].Kld * contributions[j].Kld * stressI * stressJ; + } + } + + return cross; + } + + public static IReadOnlyList<(TensorGroup Group, byte EffectiveBaselineId)> EnumerateEffectiveBaselines( + TensorConfig config, + IReadOnlyList? activeGroups = null) + { + activeGroups ??= TReg.All + .Where(x => !Cache.UnusedTensorGroups.Any(u => u.UniqueId == x.UniqueId)) + .OrderBy(x => x.UniqueId) + .ToList(); + + var result = new List<(TensorGroup Group, byte EffectiveBaselineId)>(activeGroups.Count); + + foreach (var (group, storedValue) in TensorConfigIdentity.EnumerateGroupSlots(config)) + { + if (!activeGroups.Any(x => x.UniqueId == group.UniqueId)) + continue; + + byte effective = BaselineQuants.IsNullTensorConfigGroupSlot(storedValue) + ? config.BaseQuant + : BaselineQuants.DecodeTensorConfigGroupSlotToBaselineId(storedValue); + + result.Add((group, effective)); + } + + return result; + } + + public static byte NormalizeBaselineIdForIsolation(byte baselineId) + { + if (BaselineQuants.IsNativeExactAlias(baselineId)) + return baselineId; + + var baseline = BaselineQuants.FromId(baselineId); + if (!baseline.IsExternalRepositoryBaseline) + return baselineId; + + var builtIn = BaselineQuants.ResolveBuiltInStandardBaseline(baseline.QuantizeBaseArgumentName) + ?? BaselineQuants.ResolveBuiltInStandardBaseline(baseline.Names[0]); + + return builtIn?.UniqueId ?? baselineId; + } + + internal static bool TryResolveIsolationBaselineForPrediction( + TensorGroup group, + byte effectiveBaselineId, + RankSafePredictionModel context, + List? notes, + out IsolationBaselineResolution resolution) + { + if (context.IsolationByGroupAndBaseline.TryGetValue((group.UniqueId, effectiveBaselineId), out var exact)) + { + resolution = new IsolationBaselineResolution(effectiveBaselineId, exact, false, null); + return true; + } + + if (TryGetDisabledSurrogateBaselineId(effectiveBaselineId, out var disabledSurrogateId) && + context.IsolationByGroupAndBaseline.TryGetValue((group.UniqueId, disabledSurrogateId), out var disabledSurrogate)) + { + /* + * Deprecated surrogate fallback, intentionally disabled: + * + * resolution = new IsolationBaselineResolution(disabledSurrogateId, disabledSurrogate, true, disabledSurrogateId); + * notes?.Add($"External baseline {FormatBaselineForNote(effectiveBaselineId)} used surrogate isolation {FormatBaselineForNote(disabledSurrogateId)} for group '{group.Name}'."); + * return true; + * + * This used to collapse external/custom repositories such as Unsloth Dynamic into + * their built-in llama.cpp family before prediction. MagicQuant should already have + * exact isolated samples for every registered external candidate, so using this path + * would hide a truth-coverage bug. Keep the old shape here only as a breadcrumb if a + * future emergency compatibility mode is deliberately reintroduced. + */ + _ = disabledSurrogate; + ThrowExternalIsolationSurrogateFallbackDisabled(group, effectiveBaselineId, disabledSurrogateId); + } + + if (IsExternalRepositoryBaseline(effectiveBaselineId)) + ThrowMissingExactExternalIsolation(group, effectiveBaselineId); + + resolution = default; + return false; + } + + internal static bool TryResolveBaseOnlySnapshotForPrediction( + byte baselineId, + RankSafePredictionModel context, + List? notes, + out BenchmarkSnapshotRecord snapshot) + { + if (context.BaseOnlySnapshotsByBaselineId.TryGetValue(baselineId, out var exactSnapshot)) + { + snapshot = exactSnapshot; + return true; + } + + if (TryGetDisabledSurrogateBaselineId(baselineId, out var disabledSurrogateId) && + context.BaseOnlySnapshotsByBaselineId.ContainsKey(disabledSurrogateId)) + { + /* + * Deprecated surrogate fallback, intentionally disabled: + * + * snapshot = context.BaseOnlySnapshotsByBaselineId[disabledSurrogateId]; + * notes?.Add($"External baseline {FormatBaselineForNote(baselineId)} used surrogate base-only size {FormatBaselineForNote(disabledSurrogateId)}."); + * return true; + * + * Base-only anchors must preserve the exact runtime baseline id. Falling back + * here makes UD-Q4_K_XL and Q4_K_M look byte-identical before selection even starts. + */ + ThrowExternalBaseOnlySurrogateFallbackDisabled(baselineId, disabledSurrogateId); + } + + if (IsExternalRepositoryBaseline(baselineId)) + throw new InvalidOperationException( + $"Missing exact synthetic base-only anchor for external baseline {FormatBaselineForNote(baselineId)} (id '{baselineId}'). " + + "The rank-safe predictor must not use real pure baseline snapshots as base-only prediction anchors. " + + "Pure baselines are canonicalized to Q8_0-carrier virtual blankets before prediction; reaching size prediction with an external/custom BaseQuant means a non-canonical config escaped normalization."); + + snapshot = default!; + return false; + } + + internal static bool TryGetDisabledSurrogateBaselineId(byte baselineId, out byte surrogateBaselineId) + { + surrogateBaselineId = baselineId; + + if (BaselineQuants.IsNativeExactAlias(baselineId)) + return false; + + var baseline = BaselineQuants.FromId(baselineId); + if (!baseline.IsExternalRepositoryBaseline) + return false; + + var normalized = NormalizeBaselineIdForIsolation(baselineId); + if (normalized == baselineId) + return false; + + surrogateBaselineId = normalized; + return true; + } + + internal static bool IsExternalRepositoryBaseline(byte baselineId) + { + if (BaselineQuants.IsNativeExactAlias(baselineId)) + return false; + + return BaselineQuants.FromId(baselineId).IsExternalRepositoryBaseline; + } + + private static void GuardAgainstDisabledPureBaselineSurrogateFallback( + byte baselineId, + RankSafePredictionModel context, + List notes) + { + if (!TryGetDisabledSurrogateBaselineId(baselineId, out var disabledSurrogateId) || + !context.PureSnapshotsByBaselineId.ContainsKey(disabledSurrogateId)) + { + return; + } + + /* + * Deprecated surrogate fallback, intentionally disabled: + * + * var pureSurrogate = context.PureSnapshotsByBaselineId[disabledSurrogateId]; + * notes.Add($"Pure baseline {FormatBaselineForNote(baselineId)} used surrogate pure snapshot {FormatBaselineForNote(disabledSurrogateId)}."); + * + * Pure external baselines must not inherit standard-family prediction identity. + */ + throw new InvalidOperationException( + $"Missing exact pure snapshot for external baseline {FormatBaselineForNote(baselineId)} (id '{baselineId}'), " + + $"but surrogate pure snapshot {FormatBaselineForNote(disabledSurrogateId)} (id '{disabledSurrogateId}') exists. " + + "Surrogate pure-baseline fallback is disabled to prevent external/custom collapse."); + } + + private static void ThrowExternalIsolationSurrogateFallbackDisabled( + TensorGroup group, + byte externalBaselineId, + byte surrogateBaselineId) + { + throw new InvalidOperationException( + $"Missing exact isolation snapshot for group '{group.Name}' and external baseline {FormatBaselineForNote(externalBaselineId)} (id '{externalBaselineId}'). " + + $"Surrogate isolation {FormatBaselineForNote(surrogateBaselineId)} (id '{surrogateBaselineId}') exists, but fallback is disabled. " + + "External/custom baselines must be scored from exact isolated prediction truth; regenerate/relearn the missing isolated sample instead of silently collapsing it."); + } + + private static void ThrowExternalBaseOnlySurrogateFallbackDisabled(byte externalBaselineId, byte surrogateBaselineId) + { + throw new InvalidOperationException( + $"Missing exact base-only anchor for external baseline {FormatBaselineForNote(externalBaselineId)} (id '{externalBaselineId}'). " + + $"Surrogate base-only anchor {FormatBaselineForNote(surrogateBaselineId)} (id '{surrogateBaselineId}') exists, but fallback is disabled. " + + "External/custom baselines must preserve exact runtime identity for size prediction."); + } + + private static void ThrowMissingExactExternalIsolation(TensorGroup group, byte externalBaselineId) + { + throw new InvalidOperationException( + $"Missing exact isolation snapshot for group '{group.Name}' and external baseline {FormatBaselineForNote(externalBaselineId)} (id '{externalBaselineId}'). " + + "No surrogate fallback was used. MagicQuant expects external/custom isolated samples to exist before prediction materialization."); + } + + private static void AppendExternalCoverageDiagnostics( + List notes, + IReadOnlyList activeGroups, + Dictionary pureByBaselineId, + Dictionary baseOnlyByBaselineId, + Dictionary<(byte GroupId, byte BaselineId), BenchmarkSnapshotRecord> isolationByGroupAndBaseline) + { + var externalBaselines = BaselineQuants.GetAllRecognizedBaselines() + .Where(x => x.IsExternalRepositoryBaseline) + .OrderBy(x => x.UniqueId) + .ToList(); + + if (externalBaselines.Count == 0) + return; + + notes.Add("External/custom surrogate fallback is disabled; missing exact external prediction truth will throw instead of collapsing to a standard family."); + + foreach (var baseline in externalBaselines) + { + int exactIsolation = activeGroups.Count(group => isolationByGroupAndBaseline.ContainsKey((group.UniqueId, baseline.UniqueId))); + bool exactBaseOnly = baseOnlyByBaselineId.ContainsKey(baseline.UniqueId); + bool exactPure = pureByBaselineId.ContainsKey(baseline.UniqueId); + string baseAnchorText = exactBaseOnly + ? "base-only=exact" + : exactPure + ? "base-only=missing; pure-anchor=exact" + : "base-only=missing; pure-anchor=missing"; + string fallbackText = TryGetDisabledSurrogateBaselineId(baseline.UniqueId, out var fallbackId) + ? $"; disabled fallback target would have been {FormatBaselineForNote(fallbackId)}:{fallbackId}" + : string.Empty; + + notes.Add( + $"External isolation exact coverage: {baseline.Names[0]}:{baseline.UniqueId} exact={exactIsolation}/{activeGroups.Count} groups; {baseAnchorText}{fallbackText}."); + } + } + + private sealed record IsolationBitTruthEntry( + BaselineQuants Baseline, + BenchmarkSnapshotRecord Snapshot, + double DeclaredBitRange); + + internal readonly record struct IsolationBaselineResolution( + byte BaselineId, + BenchmarkSnapshotRecord Snapshot, + bool IsSurrogate, + byte? FallbackBaselineId); + + private static bool IsZeroDamageAlias(byte baselineId) => IsNativeExactZeroReferenceAlias(baselineId); + + private static bool IsNativeExactZeroReferenceAlias(byte baselineId) + { + // MagicQuant's zero-damage reference is native exact precision (BF16/F16/F32), + // not Q8_0. Q8_0 is a real quantized state with measured per-group isolation + // KLD and must flow through the same lookup path as Q6_K/Q5_K/Q4/etc. + return BaselineQuants.IsNativeExactAlias(baselineId); + } + + private static string BuildMissingIsolationNote(TensorGroup group, byte baselineId) + { + var baselineName = FormatBaselineForNote(baselineId); + if (baselineId == BaselineQuants.Q8_0.UniqueId) + { + return $"Missing KLD isolation snapshot for group '{group.Name}' and baseline Q8_0. Q8_0 is quantized damage, not native truth; this row is marked incomplete instead of silently receiving zero KLD."; + } + + return $"Missing KLD isolation snapshot for group '{group.Name}' and baseline {baselineName} (id '{baselineId}')."; + } + + private static string FormatBaselineForNote(byte baselineId) + { + try + { + return BaselineQuants.FromId(baselineId).Names[0]; + } + catch + { + return $"id {baselineId}"; + } + } + + private static void PrintPredictionDiagnostics(IReadOnlyCollection rows, RankSafePredictionFit fit) + { + int predictable = rows.Count(x => x.IsPredictable); + int sizePredictable = rows.Count(x => x.IsPredictable && x.IsSizePredictable); + int skipped = rows.Count - predictable; + int unsafeSize = predictable - sizePredictable; + + AnsiConsole.MarkupLine($"[grey]Rank-safe prediction rows:[/] [cyan]{predictable:N0}[/] KLD-predictable / [cyan]{sizePredictable:N0}[/] size-safe / [yellow]{skipped:N0}[/] KLD-incomplete / [yellow]{unsafeSize:N0}[/] unsafe-size"); + AnsiConsole.MarkupLine($"[grey]Interaction fit:[/] alpha=[cyan]{fit.Alpha:G6}[/] beta=[cyan]{fit.Beta:G6}[/] bit-stress=[cyan]{fit.BitStressThreshold:G4}[/] fit-rows=[cyan]{fit.FitRowCount:N0}[/] fallback=[cyan]{fit.UsedFallback}[/]"); + + if (predictable == 0) + return; + + var sizeSafeRows = rows.Where(x => x.IsPredictable && x.IsSizePredictable).ToList(); + ulong minSize = sizeSafeRows.Count == 0 ? 0UL : sizeSafeRows.Min(x => x.PredictedSizeBytes); + ulong maxSize = sizeSafeRows.Count == 0 ? 0UL : sizeSafeRows.Max(x => x.PredictedSizeBytes); + double minKld = rows.Where(x => x.IsPredictable).Min(x => x.PredictedKld); + double maxKld = rows.Where(x => x.IsPredictable).Max(x => x.PredictedKld); + + AnsiConsole.MarkupLine($"[grey]Predicted size spread, size-safe rows only:[/] [cyan]{ToGb(minSize):0.00}[/] GB .. [cyan]{ToGb(maxSize):0.00}[/] GB"); + AnsiConsole.MarkupLine($"[grey]Predicted KLD spread:[/] [cyan]{minKld:0.000000}[/] .. [cyan]{maxKld:0.000000}[/]"); + } + + private static double ToGb(ulong bytes) => bytes / 1024d / 1024d / 1024d; + + internal sealed class RankSafePredictionModel + { + public RankSafePredictionModel( + IReadOnlyList activeGroups, + BenchmarkSnapshotRecord pureQ8, + BenchmarkSnapshotRecord q8BaseOnly, + Dictionary pureSnapshotsByBaselineId, + Dictionary baseOnlySnapshotsByBaselineId, + Dictionary<(byte GroupId, byte BaselineId), BenchmarkSnapshotRecord> isolationByGroupAndBaseline, + Dictionary<(byte GroupId, byte BaselineId), double> isolationDominanceBitTruthByGroupAndBaseline, + IReadOnlyList notes) + { + ActiveGroups = activeGroups; + PureQ8 = pureQ8; + Q8BaseOnly = q8BaseOnly; + PureSnapshotsByBaselineId = pureSnapshotsByBaselineId; + BaseOnlySnapshotsByBaselineId = baseOnlySnapshotsByBaselineId; + IsolationByGroupAndBaseline = isolationByGroupAndBaseline; + IsolationDominanceBitTruthByGroupAndBaseline = isolationDominanceBitTruthByGroupAndBaseline; + Notes = notes; + } + + public IReadOnlyList ActiveGroups { get; } + public BenchmarkSnapshotRecord PureQ8 { get; } + public BenchmarkSnapshotRecord Q8BaseOnly { get; } + public Dictionary PureSnapshotsByBaselineId { get; } + public Dictionary BaseOnlySnapshotsByBaselineId { get; } + public Dictionary<(byte GroupId, byte BaselineId), BenchmarkSnapshotRecord> IsolationByGroupAndBaseline { get; } + public Dictionary<(byte GroupId, byte BaselineId), double> IsolationDominanceBitTruthByGroupAndBaseline { get; } + public IReadOnlyList Notes { get; set; } + public RankSafePredictionFit Fit { get; set; } = new(); + } + + private sealed class FitObservation + { + public TensorConfig Config { get; init; } + public double ActualKld { get; init; } + public double AdditiveKld { get; init; } + } + + private struct PavaBlock + { + public double Sum; + public double Weight; + public int Count; + public double Mean => Weight <= 0d ? 0d : Sum / Weight; + } +} diff --git a/src/MagicQuant/Services/ReadmeGenerationService.cs b/src/MagicQuant/Services/ReadmeGenerationService.cs new file mode 100644 index 0000000..5f2874b --- /dev/null +++ b/src/MagicQuant/Services/ReadmeGenerationService.cs @@ -0,0 +1,728 @@ +using System.Globalization; +using System.Text.Json; +using System.Text; +using MagicQuant.Models; +using MQ.DB; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class ReadmeGenerationService +{ + private readonly FinalArtifactNamingService _namingService = new(); + + public async Task GenerateAsync( + string outputDirectory, + string modelName, + IReadOnlyCollection exportedArtifacts, + IReadOnlyCollection pureBaselineSnapshots, + IReadOnlyCollection? eliminatedBaselines = null, + BenchmarkSnapshotRecord? pplReference = null, + CancellationToken ct = default) + { + var replacementMap = + FinalReleaseMetadataService.BuildReplacementMap(eliminatedBaselines ?? + Array.Empty()); + var namingContext = _namingService.CreateContext(pureBaselineSnapshots); + var exportedByKey = exportedArtifacts + .GroupBy(x => TensorConfigIdentity.ToKey(x.Snapshot.Config), StringComparer.Ordinal) + .ToDictionary(g => g.Key, g => g.First(), StringComparer.Ordinal); + + double? referencePpl = + ResolveReferencePpl(pplReference, pureBaselineSnapshots, exportedArtifacts.Select(x => x.Snapshot)); + + var rows = exportedArtifacts + .OrderBy(x => x.Snapshot.Kld) + .ThenBy(x => x.Snapshot.SizeBytes) + .Select(artifact => + { + string key = TensorConfigIdentity.ToKey(artifact.Snapshot.Config); + string shortName = _namingService.ToPublicArtifactShortName( + artifact.DisplayName, + artifact.FileName, + artifact.ProviderName, + artifact.BaselineFamily, + artifact.Snapshot, + namingContext); + + var replacements = FinalReleaseMetadataService.ResolveTransitiveReplacements(key, replacementMap); + string nameCell = BuildNameCell(shortName, replacements, exportedByKey, namingContext); + string download = artifact.IsExternalReference + ? artifact.DownloadTarget + : MagicQuantManifestPathService.HuggingFaceGgufResolvePath(artifact.FileName ?? string.Empty); + + return new ReadmeArtifactRow + { + NameCell = nameCell, + Provider = artifact.ProviderName, + QuantFamily = artifact.BaselineFamily, + Kld = artifact.Snapshot.Kld, + Ppl = artifact.Snapshot.Ppl, + PplDeltaPercent = + FinalReleaseMetadataService.CalculatePplDeltaPercent(artifact.Snapshot.Ppl, referencePpl), + SizeBytes = artifact.Snapshot.SizeBytes, + DownloadTarget = download + }; + }) + .ToList(); + + return await GenerateCoreAsync( + outputDirectory, + modelName, + rows, + hasReplacementDetails: (eliminatedBaselines?.Count ?? 0) > 0, + cloneContext: null, + exportedArtifacts: exportedArtifacts, + ct: ct); + } + + public async Task GenerateCloneAsync( + string outputDirectory, + string modelName, + string sourceDescription, + bool sourceWasHuggingFaceRepo, + IReadOnlyCollection records, + IReadOnlyCollection? archivedManifestFileNames = null, + CancellationToken ct = default) + { + var cloneReplacementHints = LoadCloneReplacementHints(outputDirectory); + + var rows = records + .OrderBy(x => x.Kld ?? double.MaxValue) + .ThenBy(x => x.ActualSizeBytes) + .Select(record => + { + var artifact = record.ManifestArtifact; + string rawName = string.IsNullOrWhiteSpace(artifact.ShortName) + ? Path.GetFileNameWithoutExtension(artifact.FileName) + : artifact.ShortName; + string name = BuildCloneNameCell(rawName, artifact.FileName, cloneReplacementHints); + + return new ReadmeArtifactRow + { + NameCell = name, + Provider = string.IsNullOrWhiteSpace(artifact.Provider) ? "Cloned config" : artifact.Provider, + QuantFamily = string.IsNullOrWhiteSpace(artifact.QuantFamily) + ? artifact.BaseQuant + : artifact.QuantFamily, + Kld = record.Kld, + Ppl = record.Ppl, + PplDeltaPercent = record.PplDeltaPercent, + SizeBytes = record.ActualSizeBytes, + DownloadTarget = MagicQuantManifestPathService.HuggingFaceGgufResolvePath(artifact.FileName) + }; + }) + .ToList(); + + var cloneContext = new ReadmeCloneContext + { + SourceDescription = sourceDescription, + SourceWasHuggingFaceRepo = sourceWasHuggingFaceRepo, + ArchivedManifestFileNames = archivedManifestFileNames?.Where(x => !string.IsNullOrWhiteSpace(x)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .OrderBy(x => x, StringComparer.OrdinalIgnoreCase).ToList() + ?? new List() + }; + + return await GenerateCoreAsync( + outputDirectory, + modelName, + rows, + hasReplacementDetails: cloneContext.ArchivedManifestFileNames.Contains( + MagicQuantManifestPathService.ReplacementsFileName, StringComparer.OrdinalIgnoreCase), + cloneContext: cloneContext, + exportedArtifacts: Array.Empty(), + ct: ct); + } + + private async Task GenerateCoreAsync( + string outputDirectory, + string modelName, + IReadOnlyCollection rows, + bool hasReplacementDetails, + ReadmeCloneContext? cloneContext, + IReadOnlyCollection exportedArtifacts, + CancellationToken ct) + { + Directory.CreateDirectory(outputDirectory); + MagicQuantManifestPathService.EnsureManifestDirectory(outputDirectory); + + string readmePath = Path.Combine(outputDirectory, "README.md"); + var sb = new StringBuilder(); + + AppendHuggingFaceFrontmatter(sb); + + string resolvedModelName = ResolveReadmeTitleModelName(modelName); + sb.AppendLine($"# MagicQuant Hybrids - {resolvedModelName}"); + sb.AppendLine(); + sb.AppendLine( + "[MagicQuant](https://github.com/magiccodingman/MagicQuant) is a benchmark driven GGUF hybrid discovery and validation system focused on finding real, practical GGUF quants specific to each architecture."); + sb.AppendLine(); + sb.AppendLine( + "Whether it's a pure baseline model built by llama.cpp, learned tensor configurations from Unsloth, or a custom built MagicQuant hybrid, the model table below shows quants that have won dominance checks, survived collapse spaces, and/or were found to be nonlinearly better. Instead of dumping every quant type possible, MagicQuant tests, validates, and brutally murders anything deemed unworthy."); + sb.AppendLine(); + sb.AppendLine("
"); + sb.AppendLine("Support MagicQuant"); + sb.AppendLine(); + sb.AppendLine( + "I’m a solo developer working full time for myself to achieve my dream. I build open source code on the side. If you like any of my work, buying me a coffee is always appreciated. Otherwise, I hope you enjoy, maybe give me a star or something. Or just send me good vibes. Either way, thank you!"); + sb.AppendLine(); + sb.AppendLine("[Click here to see ways to support](https://sayou.biz/support) - BTC, Paypal, GitHub sponsors."); + sb.AppendLine(); + sb.AppendLine("
"); + sb.AppendLine(); + + if (cloneContext != null) + AppendCloneNotice(sb, cloneContext); + + sb.AppendLine("---"); + sb.AppendLine(); + sb.AppendLine("## Final survivors"); + sb.AppendLine(); + AppendDownloadTable(sb, rows); + sb.AppendLine(); + + //if (cloneContext == null) + //{ + AppendProviderCredits(sb, exportedArtifacts); + sb.AppendLine(); + + sb.AppendLine("
"); + sb.AppendLine("Warning - Is MagicQuant Better? (hint: how you frame the question matters)"); + sb.AppendLine(); + sb.AppendLine("External/custom baselines are normalized into MagicQuant's controlled comparison flow. MagicQuant rebuilds a learned baseline under native-source / MagicQuant-controlled conditions, including its own imatrix handling, so hybrids or external baselines (like Unsloth) can be judged on a more equal footing. That does **not** mean MagicQuant proved the original upstream artifact or upstream imatrix was worse. These comparisons exist for internal hybrid-search consistency and equal playing field comparisons, not as a universal judgment of the original creator's exact release artifact."); + sb.AppendLine(); + sb.AppendLine("MagicQuant learns tensor quantization assignments and rebuilds from local source weights. It does not automatically reproduce a provider's additional weight transformations, calibration recipes, custom processing, or other techniques unless explicitly supported. These results are not a byte-for-byte reproduction or a test of the provider's original GGUF."); + sb.AppendLine(); + sb.AppendLine("**Easier to digest explanation:**"); + sb.AppendLine(); + sb.AppendLine("MagicQuant compares and benchmarks the models quant to tensor configurations, but not the original artifact. And there's different reasons MagicQuant chooses to lift up a winning quant, not all winners are purely \"better\". It depends heavily on a variety of factors. Though choices are always documented in the repo under the manifest folder. You can always view what and why decisions were made by the automated system."); + sb.AppendLine(); + sb.AppendLine("So, MagicQuant can confidently tell you, \"under the same quantization to tensor configurations and identical imatrix, with this benchmark, I deemed this a winner\"."); + sb.AppendLine(); + sb.AppendLine("
"); + sb.AppendLine(); + //} + + sb.AppendLine("
"); + sb.AppendLine("Re-Uploading External Provider Baselines"); + sb.AppendLine(); + sb.AppendLine("By default, if an external provider like Unsloth is deemed the winner, the repo should generally link directly to the original provider instead of re-hosting the quant. External GGUFs are normally only re-uploaded when a specific winning variant does not already exist (e.g. Heretic models or similar)."); + sb.AppendLine(); + sb.AppendLine("
"); + + sb.AppendLine(); + + sb.AppendLine("---"); + sb.AppendLine(); + + AppendReleaseMetadata(sb, cloneContext); + sb.AppendLine(); + sb.AppendLine("---"); + sb.AppendLine(); + + await File.WriteAllTextAsync(readmePath, sb.ToString(), ct); + AnsiConsole.MarkupLine($"[green]README generated:[/] {Markup.Escape(readmePath)}"); + return readmePath; + } + + private static void AppendCloneNotice(StringBuilder sb, ReadmeCloneContext clone) + { + string source = clone.SourceWasHuggingFaceRepo + ? BuildHuggingFaceRepoLink(clone.SourceDescription) + : $"`{EscapePipe(clone.SourceDescription)}`"; + + sb.AppendLine("
"); + sb.AppendLine("Clone Notice"); + sb.AppendLine(); + sb.AppendLine( + $"This repository did not run through the full MagicQuant discovery pipeline. It is a clone of the final survivor tensor configurations from {source}, rebuilt and benchmarked locally for this model."); + sb.AppendLine(); + sb.AppendLine( + "The archived MagicQuant JSON files in `magicquant-manifest/` are copied from the source release for durability. The clone benchmark JSON and the table below are from this clone run, so those metrics reflect the rebuilt outputs in this repository."); + sb.AppendLine(); + sb.AppendLine("
"); + sb.AppendLine(); + } + + private static string BuildCloneNameCell(string rawName, string fileName, + IReadOnlyDictionary> replacementHints) + { + string safeName = EscapePipe(rawName); + + if (!TryGetReplacementHint(replacementHints, fileName, out var replaced) && + !TryGetReplacementHint(replacementHints, rawName, out replaced)) + { + return safeName; + } + + var shown = replaced.Take(5).Where(x => !string.IsNullOrWhiteSpace(x)).Select(EscapePipe).ToList(); + string tooltip = shown.Count == 0 + ? $"Replaced one or more source artifacts. See {MagicQuantManifestPathService.RelativeManifestPath(MagicQuantManifestPathService.ReplacementsFileName)}." + : $"Replaced: {string.Join(", ", shown)}"; + + if (replaced.Count > shown.Count) + tooltip += $" + {replaced.Count - shown.Count} more"; + + return $"[{safeName}](#winner-notes \"{EscapeTooltip(tooltip)}\")"; + } + + private static bool TryGetReplacementHint(IReadOnlyDictionary> replacementHints, + string? key, out IReadOnlyList replaced) + { + replaced = Array.Empty(); + if (string.IsNullOrWhiteSpace(key)) + return false; + + if (!replacementHints.TryGetValue(key.Trim(), out var found) || found.Count == 0) return false; + replaced = found; + return true; + } + + private static Dictionary> LoadCloneReplacementHints(string outputDirectory) + { + var output = new Dictionary>(StringComparer.OrdinalIgnoreCase); + string path = MagicQuantManifestPathService.GetManifestFilePath(outputDirectory, + MagicQuantManifestPathService.FinalSurvivorsFileName); + if (!File.Exists(path)) + return output; + + try + { + using var doc = JsonDocument.Parse(File.ReadAllText(path)); + if (doc.RootElement.ValueKind != JsonValueKind.Array) + return output; + + foreach (var row in doc.RootElement.EnumerateArray()) + { + var replaced = ReadReplacedShortNames(row); + if (replaced.Count == 0) + continue; + + AddReplacementHint(output, TryGetString(row, "fileName"), replaced); + AddReplacementHint(output, TryGetString(row, "shortName"), replaced); + AddReplacementHint(output, TryGetString(row, "displayName"), replaced); + } + } + catch (Exception ex) + { + AnsiConsole.MarkupLine( + $"[yellow]Could not read clone replacement hints from archived final-survivors JSON:[/] {Markup.Escape(ex.Message)}"); + } + + return output; + } + + private static List ReadReplacedShortNames(JsonElement survivorRow) + { + if (!survivorRow.TryGetProperty("replacedArtifacts", out var replacedArtifacts) || + replacedArtifacts.ValueKind != JsonValueKind.Array) + { + return new List(); + } + + return replacedArtifacts.EnumerateArray() + .Select(x => TryGetString(x, "shortName") ?? TryGetString(x, "displayName") ?? TryGetString(x, "fileName")) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Select(x => x!) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); + } + + private static void AddReplacementHint(Dictionary> output, string? key, + IReadOnlyList replaced) + { + if (string.IsNullOrWhiteSpace(key) || replaced.Count == 0) + return; + + output[key.Trim()] = replaced; + } + + private static string? TryGetString(JsonElement row, string propertyName) + { + return row.TryGetProperty(propertyName, out var value) && value.ValueKind == JsonValueKind.String + ? value.GetString() + : null; + } + + + private static string BuildHuggingFaceRepoLink(string repoId) + { + string clean = (repoId ?? string.Empty).Trim().Trim('/'); + if (string.IsNullOrWhiteSpace(clean)) + return "the source Hugging Face repository"; + + return $"[{EscapePipe(clean)}](https://huggingface.co/{clean})"; + } + + private static void AppendReleaseMetadata(StringBuilder sb, ReadmeCloneContext? cloneContext) + { + sb.AppendLine("## Release metadata"); + sb.AppendLine(); + if (ShouldLinkManifestFile(cloneContext, MagicQuantManifestPathService.FinalSurvivorsFileName)) + sb.AppendLine( + $"- [Final survivor metrics]({MagicQuantManifestPathService.HuggingFaceResolvePath(MagicQuantManifestPathService.FinalSurvivorsFileName)}) — full file names, KLD, PPL, PPL delta %, byte sizes, download targets, and replacement lineage. PPL delta % is measured against the native/reference PPL when available; negative is better and larger positive values are worse."); + + if (ShouldLinkManifestFile(cloneContext, MagicQuantManifestPathService.HybridMapFileName)) + sb.AppendLine( + $"- [Hybrid tensor map]({MagicQuantManifestPathService.HuggingFaceResolvePath(MagicQuantManifestPathService.HybridMapFileName)}) — tensor-group assignments and effective-state details for MagicQuant hybrid GGUFs."); + + sb.AppendLine( + $"- [Clone tensor configs]({MagicQuantManifestPathService.HuggingFaceResolvePath(MagicQuantManifestPathService.CloneConfigsFileName)}) — exact per-GGUF tensor quantization maps for reproducing this final output list in repository clone mode."); + + if (ShouldLinkManifestFile(cloneContext, MagicQuantManifestPathService.IsolationSamplesFileName)) + sb.AppendLine( + $"- [Isolation samples]({MagicQuantManifestPathService.HuggingFaceResolvePath(MagicQuantManifestPathService.IsolationSamplesFileName)}) — isolated base/group probe samples with KLD, PPL, PPL delta %, and size truth."); + + if (ShouldLinkManifestFile(cloneContext, MagicQuantManifestPathService.BadTradesFileName)) + sb.AppendLine( + $"- [Bad trade details]({MagicQuantManifestPathService.HuggingFaceResolvePath(MagicQuantManifestPathService.BadTradesFileName)}) — structured bad-trade pruning decisions from the isolation optimizer."); + + if (cloneContext != null) + sb.AppendLine( + $"- [Clone benchmark summary]({MagicQuantManifestPathService.HuggingFaceResolvePath(MagicQuantManifestPathService.CloneBenchmarksFileName)}) — fresh benchmark results from this clone run."); + + + if (ShouldLinkManifestFile(cloneContext, MagicQuantManifestPathService.ReplacementsFileName)) + { + sb.AppendLine( + $"- [Replacement details]({MagicQuantManifestPathService.HuggingFaceResolvePath(MagicQuantManifestPathService.ReplacementsFileName)}) — structured details for baselines or anchors removed from the final download table, including reason codes, KLD deltas, PPL delta %, and size deltas."); + + sb.AppendLine(); + AppendReasonCodeDetails(sb); + sb.AppendLine(); + } + } + + private static bool ShouldLinkManifestFile(ReadmeCloneContext? cloneContext, string fileName) + { + return cloneContext == null || + cloneContext.ArchivedManifestFileNames.Contains(fileName, StringComparer.OrdinalIgnoreCase) || + string.Equals(fileName, MagicQuantManifestPathService.CloneConfigsFileName, + StringComparison.OrdinalIgnoreCase) || + string.Equals(fileName, MagicQuantManifestPathService.CloneBenchmarksFileName, + StringComparison.OrdinalIgnoreCase); + } + + + private static void AppendHuggingFaceFrontmatter(StringBuilder sb) + { + var entries = OrderedFrontmatterEntries().ToList(); + if (entries.Count == 0) + return; + + sb.AppendLine("---"); + foreach (var (key, value) in entries) + { + if (TryGetSequence(value, out var values)) + { + var rendered = values + .Where(x => !IsEmptyFrontmatterValue(x)) + .Select(FormatYamlScalar) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .ToList(); + + if (rendered.Count == 0) + continue; + + sb.AppendLine($"{key}:"); + foreach (var item in rendered) + sb.AppendLine($"- {item}"); + } + else + { + if (IsEmptyFrontmatterValue(value)) + continue; + + sb.AppendLine($"{key}: {FormatYamlScalar(value)}"); + } + } + + sb.AppendLine("---"); + sb.AppendLine(); + } + + private static IEnumerable> OrderedFrontmatterEntries() + { + var frontmatter = Config.Current.Readme.Frontmatter; + if (frontmatter == null || frontmatter.Count == 0) + yield break; + + if (frontmatter.TryGetValue("license", out var license) && !IsEmptyFrontmatterValue(license)) + yield return new KeyValuePair("license", license); + + foreach (var entry in frontmatter) + { + if (string.IsNullOrWhiteSpace(entry.Key) || + string.Equals(entry.Key, "license", StringComparison.OrdinalIgnoreCase) || + IsEmptyFrontmatterValue(entry.Value)) + { + continue; + } + + yield return new KeyValuePair(entry.Key.Trim(), entry.Value); + } + } + + private static string ResolveReadmeTitleModelName(string fallbackModelName) + { + if (!string.IsNullOrWhiteSpace(Config.Current.Readme.TitleModelNameOverride)) + return Config.Current.Readme.TitleModelNameOverride.Trim(); + + if (!string.IsNullOrWhiteSpace(Config.Current.Identity.ArchitectureFamilyName)) + return Config.Current.Identity.ArchitectureFamilyName.Trim(); + + if (!string.IsNullOrWhiteSpace(Cache.CurrentArchitectureFamilyName)) + return Cache.CurrentArchitectureFamilyName.Trim(); + + return string.IsNullOrWhiteSpace(fallbackModelName) ? "model" : fallbackModelName.Trim(); + } + + private static bool TryGetSequence(object? value, out IReadOnlyList values) + { + values = Array.Empty(); + + if (value is string || value == null) + return false; + + if (value is System.Collections.IEnumerable sequence) + { + values = sequence.Cast().ToList(); + return true; + } + + return false; + } + + private static bool IsEmptyFrontmatterValue(object? value) + { + if (value == null) + return true; + + if (value is string text) + return string.IsNullOrWhiteSpace(text); + + if (TryGetSequence(value, out var values)) + return values.All(IsEmptyFrontmatterValue); + + return false; + } + + private static string FormatYamlScalar(object? value) + { + if (value == null) + return string.Empty; + + if (value is bool boolean) + return boolean ? "true" : "false"; + + if (value is IFormattable formattable && value is not string) + return formattable.ToString(null, CultureInfo.InvariantCulture) ?? string.Empty; + + string text = value.ToString() ?? string.Empty; + if (!NeedsYamlQuotes(text)) + return text; + + return "\"" + text + .Replace("\\", "\\\\", StringComparison.Ordinal) + .Replace("\"", "\\\"", StringComparison.Ordinal) + .Replace("\r", "\\r", StringComparison.Ordinal) + .Replace("\n", "\\n", StringComparison.Ordinal) + "\""; + } + + private static bool NeedsYamlQuotes(string text) + { + if (text.Length == 0) + return true; + + if (!string.Equals(text, text.Trim(), StringComparison.Ordinal)) + return true; + + if (text.Contains(": ", StringComparison.Ordinal) || + text.Contains("#", StringComparison.Ordinal) || + text.Contains("\n", StringComparison.Ordinal) || + text.Contains("\r", StringComparison.Ordinal)) + { + return true; + } + + char first = text[0]; + return first is '-' or '?' or ':' or '@' or '!' or '&' or '*' or '[' or ']' or '{' or '}' or '|' or '>' or '%' + or '`' or ','; + } + + private static void AppendDownloadTable(StringBuilder sb, IReadOnlyCollection rows) + { + sb.AppendLine("| Name | Provider | KLD | Size (GB) | Download |"); + sb.AppendLine("|---|---|---:|---:|---|"); + + foreach (var row in rows.OrderBy(x => x.Kld ?? double.MaxValue).ThenBy(x => x.SizeBytes)) + { + string kld = row.Kld.HasValue ? row.Kld.Value.ToString("0.000000", CultureInfo.InvariantCulture) : "n/a"; + //string ppl = row.Ppl.HasValue ? row.Ppl.Value.ToString("0.000000", CultureInfo.InvariantCulture) : "n/a"; + /*string pplDelta = row.PplDeltaPercent.HasValue + ? row.PplDeltaPercent.Value.ToString("0.000", CultureInfo.InvariantCulture) + "%" + : "n/a";*/ + string sizeGb = ToGB(row.SizeBytes); + string download = string.IsNullOrWhiteSpace(row.DownloadTarget) ? "n/a" : $"[Link]({row.DownloadTarget})"; + + sb.AppendLine( + $"| {row.NameCell} | {EscapePipe(row.Provider)} | {kld} | {sizeGb} | {download} |"); + } + } + + private string BuildNameCell( + string shortName, + IReadOnlyList replacements, + IReadOnlyDictionary exportedByKey, + FinalArtifactNamingContext namingContext) + { + if (replacements.Count == 0) + return EscapePipe(shortName); + + var replacedNames = replacements + .Select(x => GetPublicShortName(x.Eliminated, exportedByKey, namingContext)) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Take(5) + .ToList(); + + string tooltip = replacedNames.Count == 0 + ? $"Replaced one or more dominated artifacts. See {MagicQuantManifestPathService.RelativeManifestPath(MagicQuantManifestPathService.ReplacementsFileName)}." + : $"Replaced: {string.Join(", ", replacedNames)}"; + + if (replacements.Count > replacedNames.Count) + tooltip += $" + {replacements.Count - replacedNames.Count} more"; + + return $"[{EscapePipe(shortName)}](#winner-notes \"{EscapeTooltip(tooltip)}\")"; + } + + private string GetPublicShortName( + BenchmarkSnapshotRecord snapshot, + IReadOnlyDictionary exportedByKey, + FinalArtifactNamingContext namingContext) + { + string key = TensorConfigIdentity.ToKey(snapshot.Config); + if (exportedByKey.TryGetValue(key, out var artifact)) + return _namingService.ToPublicArtifactShortName( + artifact.DisplayName, + artifact.FileName, + artifact.ProviderName, + artifact.BaselineFamily, + artifact.Snapshot, + namingContext); + + return _namingService.ToPublicArtifactShortName( + _namingService.BuildDisplayLabel(snapshot, namingContext), + null, + snapshot.IsHybrid + ? "MagicQuant" + : HybridBenchmarkRepository.ResolveProviderName(snapshot.Quant, exportNaming: false), + snapshot.BaselineFamily, + snapshot, + namingContext); + } + + private static void AppendReasonCodeDetails(StringBuilder sb) + { + sb.AppendLine("
"); + sb.AppendLine("Replacement reason codes"); + sb.AppendLine(); + sb.AppendLine( + "- `STRICT_DOMINANCE` — the winner was no larger and had lower real KLD than the removed anchor."); + sb.AppendLine( + "- `NEAR_BASELINE_PREMIUM` — the winner used only the configured near-baseline size premium and beat the real linear KLD trade line."); + sb.AppendLine( + "- `INTERIOR_DISCOVERY` — the winner was selected as a useful interior point inside a size/KLD gap between anchors."); + sb.AppendLine( + "- `SPACING_COLLAPSE` — two candidates were too close in practical output space, so the stronger one was kept."); + sb.AppendLine( + "- `FINAL_DOMINANCE` — a later validated survivor dominated this artifact in final real benchmark comparison."); + sb.AppendLine(); + sb.AppendLine(""); + sb.AppendLine( + $"Underlined names in the table replaced or ultimately inherited the replacement of another artifact. Hover the name for the short replacement summary, or inspect `{MagicQuantManifestPathService.RelativeManifestPath(MagicQuantManifestPathService.ReplacementsFileName)}` for exact KLD/PPL/size deltas."); + sb.AppendLine(); + sb.AppendLine("
"); + } + + private void AppendProviderCredits(StringBuilder sb, IReadOnlyCollection artifacts) + { + var credits = _namingService.BuildProviderCredits(artifacts); + if (credits.Count == 0) + return; + + sb.AppendLine("
"); + sb.AppendLine("Provider credits"); + sb.AppendLine(); + + foreach (var credit in credits) + { + string name = EscapePipe(credit.Name); + string note = EscapePipe(credit.Note); + + if (!string.IsNullOrWhiteSpace(credit.Url)) + sb.AppendLine($"- [{name}]({credit.Url}) — {note}"); + else + sb.AppendLine($"- {name} — {note}"); + } + + sb.AppendLine(); + sb.AppendLine("
"); + } + + + private static double? ResolveReferencePpl( + BenchmarkSnapshotRecord? pplReference, + IReadOnlyCollection pureBaselineSnapshots, + IEnumerable snapshots) + { + if (pplReference is { Ppl: > 0d }) + return pplReference.Ppl; + + var bestPure = pureBaselineSnapshots + .Where(x => x.Ppl > 0d) + .OrderBy(x => x.Kld) + .ThenByDescending(x => x.SizeBytes) + .FirstOrDefault(); + + if (bestPure != null) + return bestPure.Ppl; + + return snapshots + .Where(x => x.Ppl > 0d) + .OrderBy(x => x.Kld) + .FirstOrDefault() + ?.Ppl; + } + + private static string ToGB(ulong bytes) => + (bytes / 1000d / 1000d / 1000d).ToString("0.00", CultureInfo.InvariantCulture); + + private static string EscapePipe(string value) => (value ?? string.Empty).Replace("|", "\\|"); + + private static string EscapeTooltip(string value) => + (value ?? string.Empty).Replace("\"", """).Replace("|", " "); + + private static string EscapeHtml(string value) => + (value ?? string.Empty).Replace("&", "&").Replace("<", "<").Replace(">", ">"); + + private sealed class ReadmeArtifactRow + { + public string NameCell { get; init; } = string.Empty; + public string Provider { get; init; } = string.Empty; + public string QuantFamily { get; init; } = string.Empty; + public double? Kld { get; init; } + public double? Ppl { get; init; } + public double? PplDeltaPercent { get; init; } + public ulong SizeBytes { get; init; } + public string DownloadTarget { get; init; } = string.Empty; + } + + private sealed class ReadmeCloneContext + { + public string SourceDescription { get; init; } = string.Empty; + public bool SourceWasHuggingFaceRepo { get; init; } + public IReadOnlyList ArchivedManifestFileNames { get; init; } = Array.Empty(); + } +} diff --git a/src/MagicQuant/Services/RemainingCombinationStore.cs b/src/MagicQuant/Services/RemainingCombinationStore.cs new file mode 100644 index 0000000..6d753c9 --- /dev/null +++ b/src/MagicQuant/Services/RemainingCombinationStore.cs @@ -0,0 +1,818 @@ +using DuckDB.NET.Data; +using MagicQuant.Helpers; +using MagicQuant.Models; +using MQ.DB; +using MQ.DB.Models; +using System.Numerics; +using System.Runtime.CompilerServices; + +namespace MagicQuant.Services; + +public sealed class RemainingCombinationStore +{ + private const string TableName = CombinationDuckDbSchema.TableName; + + private static string ConnectionString => $"Data Source={CombinationDatabasePathService.GetPath()}"; + + public string GetDatabaseFilePath() => CombinationDatabasePathService.GetPath(); + + public async Task CountAsync(CancellationToken ct = default) + { + using var connection = new DuckDBConnection(ConnectionString); + await connection.OpenAsync(ct); + await ConfigureFastLoadSessionAsync(connection, ct); + await EnsureTensorConfigsTableExistsAsync(connection, ct); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = $"SELECT COUNT(*) FROM {TableName};"; + return ToInt64(await cmd.ExecuteScalarAsync(ct)); + } + + public async Task> LoadAllAsync(CancellationToken ct = default) + { + long count = await CountAsync(ct); + if (count > Config.MaxInMemoryCombinationLoadRows) + throw new InvalidOperationException($"Refusing to load {count:N0} DuckDB tensor configs into memory. Use SQL-native filtering/streaming instead."); + + using var connection = new DuckDBConnection(ConnectionString); + await connection.OpenAsync(ct); + await ConfigureFastLoadSessionAsync(connection, ct); + + var results = new List(); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@" +SELECT {CombinationDuckDbSchema.SlotColumnList} +FROM {TableName} +ORDER BY {CombinationDuckDbSchema.SlotColumnList};"; + + using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + results.Add(ReadTensorConfig(reader)); + + return results; + } + + public async IAsyncEnumerable StreamAsync( + string? whereSql = null, + string? orderBySql = null, + long? limit = null, + [EnumeratorCancellation] CancellationToken ct = default) + { + using var connection = new DuckDBConnection(ConnectionString); + await connection.OpenAsync(ct); + await ConfigureFastLoadSessionAsync(connection, ct); + await EnsureTensorConfigsTableExistsAsync(connection, ct); + + string sql = $@"SELECT {CombinationDuckDbSchema.SlotColumnList} FROM {TableName}"; + if (!string.IsNullOrWhiteSpace(whereSql)) + sql += $" WHERE {whereSql}"; + if (!string.IsNullOrWhiteSpace(orderBySql)) + sql += $" ORDER BY {orderBySql}"; + if (limit.HasValue) + sql += $" LIMIT {limit.Value}"; + + using var cmd = connection.CreateCommand(); + cmd.CommandText = sql; + + using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + yield return ReadTensorConfig(reader); + } + + public async Task ReplaceAllAsync( + IReadOnlyCollection configs, + string reason, + CancellationToken ct = default) + { + using var connection = new DuckDBConnection(ConnectionString); + await connection.OpenAsync(ct); + await ConfigureFastLoadSessionAsync(connection, ct); + await RecreateTableAsync(connection, ct); + + using var tx = connection.BeginTransaction(); + using var insert = connection.CreateCommand(); + insert.CommandText = $@" +INSERT INTO {TableName} +({CombinationDuckDbSchema.SlotColumnList}) +VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"; + + foreach (var config in configs) + { + insert.Parameters.Clear(); + AddSlotParameters(insert, config); + await insert.ExecuteNonQueryAsync(ct); + } + + tx.Commit(); + } + + public async Task GetPredictionStatusAsync(CancellationToken ct = default) + { + using var connection = new DuckDBConnection(ConnectionString); + await connection.OpenAsync(ct); + await ConfigureFastLoadSessionAsync(connection, ct); + await EnsureTensorConfigsTableExistsAsync(connection, ct); + + using var cmd = connection.CreateCommand(); + cmd.CommandText = $@" +SELECT + COUNT(*) AS TotalRows, + COUNT(PredictedKld) AS PredictedRows, + COUNT(PredictionRank) AS RankedRows, + MIN(PredictedKld) AS MinPredictedKld, + MAX(PredictedKld) AS MaxPredictedKld, + MIN(PredictedSizeBytes) AS MinPredictedSizeBytes, + MAX(PredictedSizeBytes) AS MaxPredictedSizeBytes +FROM {TableName};"; + + using var r = await cmd.ExecuteReaderAsync(ct); + await r.ReadAsync(ct); + + long total = ToInt64(r.GetValue(0)); + long predicted = ToInt64(r.GetValue(1)); + long ranked = ToInt64(r.GetValue(2)); + + return new PredictionMaterializationStatus + { + TotalRows = total, + PredictedRows = predicted, + MissingPredictionRows = Math.Max(0, total - predicted), + RankedRows = ranked, + MinPredictedKld = r.IsDBNull(3) ? null : ToDouble(r.GetValue(3)), + MaxPredictedKld = r.IsDBNull(4) ? null : ToDouble(r.GetValue(4)), + MinPredictedSizeBytes = r.IsDBNull(5) ? null : ToUInt64(r.GetValue(5)), + MaxPredictedSizeBytes = r.IsDBNull(6) ? null : ToUInt64(r.GetValue(6)) + }; + } + + public async Task> GetPredictedAnchorRowsAsync(CancellationToken ct = default) + { + string sql = $@" +SELECT {CombinationDuckDbSchema.SlotColumnList}, + COALESCE(AnchorDisplayName, AnchorBaselineCanonicalKey, '') AS AnchorDisplayName, + COALESCE(AnchorBaselineCanonicalKey, '') AS AnchorBaselineCanonicalKey, + COALESCE(AnchorBaselineRuntimeId, 0) AS AnchorBaselineRuntimeId, + {CombinationDuckDbSchema.EffectivePredictedKldSql} AS PredictedKld, + PredictedSizeBytes, + PredictionConfidence, + PredictionRank, + COALESCE(IsVirtualPredictionAnchor, FALSE) AS IsVirtualPredictionAnchor +FROM {TableName} +WHERE {CombinationDuckDbSchema.VirtualPredictionAnchorPredicateSql} + AND COALESCE(FinalPredictedKld, PredictedKld) IS NOT NULL + AND PredictedSizeBytes IS NOT NULL + AND PredictionRank IS NOT NULL +ORDER BY PredictedKld ASC, + PredictedSizeBytes ASC, + PredictionRank ASC;"; + + using var c = new DuckDBConnection(ConnectionString); + await c.OpenAsync(ct); + await ConfigureFastLoadSessionAsync(c, ct); + await EnsureTensorConfigsTableExistsAsync(c, ct); + + using var cmd = c.CreateCommand(); + cmd.CommandText = sql; + + var list = new List(); + using var r = await cmd.ExecuteReaderAsync(ct); + while (await r.ReadAsync(ct)) + list.Add(MapPredictedAnchorRow(r)); + + return list; + } + + public async Task FindPredictedAnchorForRealAnchorAsync( + BenchmarkSnapshotRecord realAnchor, + IReadOnlyList predictedAnchors, + CancellationToken ct = default) + { + ct.ThrowIfCancellationRequested(); + + var sourceBaseline = HybridBenchmarkRepository.ResolveSourceBaselineForProvider(realAnchor.Quant); + var predictionSpaceConfig = CanonicalizeConfigForPredictionSpace(realAnchor.Config); + string predictionSpaceKey = TensorConfigIdentity.ToKey(predictionSpaceConfig); + + var byConfig = predictedAnchors.FirstOrDefault(x => + string.Equals(x.ConfigKey, predictionSpaceKey, StringComparison.Ordinal)); + if (byConfig != null) + return byConfig; + + if (HybridBenchmarkRepository.IsTrueMagicQuantHybrid(realAnchor.Quant)) + return await QueryPredictedAnchorForConfigAsync(realAnchor, sourceBaseline, ct); + + string canonicalKey = NormalizeAnchorKey(sourceBaseline.CanonicalKey); + + var byCanonical = predictedAnchors + .Where(x => !string.IsNullOrWhiteSpace(x.BaselineCanonicalKey)) + .FirstOrDefault(x => string.Equals(NormalizeAnchorKey(x.BaselineCanonicalKey), canonicalKey, StringComparison.Ordinal)); + + if (byCanonical != null) + return byCanonical; + + var byRuntimeId = predictedAnchors.FirstOrDefault(x => x.RuntimeBaselineId == sourceBaseline.UniqueId); + if (byRuntimeId != null) + return byRuntimeId; + + var displayNames = sourceBaseline.Names + .Concat([realAnchor.DisplayName, realAnchor.BaselineFamily]) + .Where(x => !string.IsNullOrWhiteSpace(x)) + .Select(NormalizeAnchorKey) + .ToHashSet(StringComparer.Ordinal); + + var byDisplay = predictedAnchors.FirstOrDefault(x => displayNames.Contains(NormalizeAnchorKey(x.DisplayName))); + if (byDisplay != null) + return byDisplay; + + // Accepted MagicQuant hybrids can become real anchors in later phases. They will + // not have a virtual baseline-anchor row, but their own tensor config should still + // be present and scored in DuckDB. Use that predicted row as the phase-local + // prediction-space anchor instead of falling back to real KLD/size. + return await QueryPredictedAnchorForConfigAsync(realAnchor, sourceBaseline, ct); + } + + private async Task QueryPredictedAnchorForConfigAsync( + BenchmarkSnapshotRecord realAnchor, + BaselineQuants sourceBaseline, + CancellationToken ct) + { + var predictionSpaceConfig = CanonicalizeConfigForPredictionSpace(realAnchor.Config); + + string sql = $@" +SELECT {CombinationDuckDbSchema.SlotColumnList}, + {CombinationDuckDbSchema.EffectivePredictedKldSql} AS PredictedKld, + PredictedSizeBytes, + PredictionConfidence, + PredictionRank, + COALESCE(IsVirtualPredictionAnchor, FALSE) AS IsVirtualPredictionAnchor +FROM {TableName} +WHERE COALESCE(FinalPredictedKld, PredictedKld) IS NOT NULL + AND PredictedSizeBytes IS NOT NULL + AND PredictionRank IS NOT NULL + AND {BuildSlotPredicateSql(predictionSpaceConfig)} +LIMIT 1;"; + + using var c = new DuckDBConnection(ConnectionString); + await c.OpenAsync(ct); + await ConfigureFastLoadSessionAsync(c, ct); + await EnsureTensorConfigsTableExistsAsync(c, ct); + + using var cmd = c.CreateCommand(); + cmd.CommandText = sql; + + using var r = await cmd.ExecuteReaderAsync(ct); + if (!await r.ReadAsync(ct)) + return null; + + var config = ReadTensorConfig(r); + return new PredictedAnchorRow + { + Config = config, + ConfigKey = TensorConfigIdentity.ToKey(config), + DisplayName = realAnchor.DisplayName, + BaselineCanonicalKey = sourceBaseline.CanonicalKey, + RuntimeBaselineId = sourceBaseline.UniqueId, + PredictedKld = ToDouble(r.GetValue(10)), + PredictedSizeBytes = ToUInt64(r.GetValue(11)), + PredictionConfidence = ToDouble(r.GetValue(12)), + PredictionRank = ToUInt64(r.GetValue(13)), + IsVirtualPredictionAnchor = ToBoolean(r.GetValue(14)) + }; + } + + public Task CountStrictDominanceCandidatesAsync( + PredictedAnchorRow anchor, + CancellationToken ct = default) + { + return CountStrictDominanceCandidatesAsync(anchor, anchor.PredictedSizeBytes, ct); + } + + public async Task CountStrictDominanceCandidatesAsync( + PredictedAnchorRow anchor, + ulong maxSizeBytes, + CancellationToken ct = default) + { + string sql = $@" +SELECT COUNT(*) +FROM {TableName} +WHERE COALESCE(FinalPredictedKld, PredictedKld) IS NOT NULL + AND PredictedSizeBytes IS NOT NULL + AND PredictionRank IS NOT NULL + AND {CombinationDuckDbSchema.ActiveCandidatePredicateSql} + AND {CombinationDuckDbSchema.HybridPredicateSql} + AND PredictedSizeBytes <= ? + AND {CombinationDuckDbSchema.EffectivePredictedKldSql} + ? < ?;"; + + return await ExecuteCountAsync( + sql, + new object[] { maxSizeBytes, Config.SelectionMinimumKldImprovementEpsilon, anchor.PredictedKld }, + ct); + } + + public async Task CountPredictedHybridCandidatesInSizeWindowAsync( + ulong minSize, + ulong maxSize, + CancellationToken ct = default) + { + string sql = $@" +SELECT COUNT(*) +FROM {TableName} +WHERE COALESCE(FinalPredictedKld, PredictedKld) IS NOT NULL + AND PredictedSizeBytes IS NOT NULL + AND PredictionRank IS NOT NULL + AND {CombinationDuckDbSchema.ActiveCandidatePredicateSql} + AND {CombinationDuckDbSchema.HybridPredicateSql} + AND PredictedSizeBytes BETWEEN ? AND ?;"; + + return await ExecuteCountAsync(sql, new object[] { minSize, maxSize }, ct); + } + + public Task CountBetterThanLinearCandidatesAsync( + PredictedAnchorRow higherDamageSmaller, + PredictedAnchorRow lowerDamageLarger, + ulong minSize, + ulong maxSize, + CancellationToken ct = default) + { + return CountBetterThanLinearCandidatesAsync( + higherDamageSmaller, + lowerDamageLarger, + predictionWindowMinSize: minSize, + predictionWindowMaxSize: maxSize, + deterministicWindowMinSize: minSize, + deterministicWindowMaxSize: maxSize, + ct: ct); + } + + public async Task CountBetterThanLinearCandidatesAsync( + PredictedAnchorRow higherDamageSmaller, + PredictedAnchorRow lowerDamageLarger, + ulong predictionWindowMinSize, + ulong predictionWindowMaxSize, + ulong deterministicWindowMinSize, + ulong deterministicWindowMaxSize, + CancellationToken ct = default) + { + var effectiveWindow = IntersectSizeWindows( + predictionWindowMinSize, + predictionWindowMaxSize, + deterministicWindowMinSize, + deterministicWindowMaxSize); + + if (effectiveWindow == null) + return 0; + + string sql = $@" +WITH scored AS ( + SELECT {CombinationDuckDbSchema.EffectivePredictedKldSql} AS PredictedKld, + PredictedSizeBytes, + (CAST(? AS DOUBLE) + + ((CAST(PredictedSizeBytes AS DOUBLE) - CAST(? AS DOUBLE)) / GREATEST(CAST(? AS DOUBLE), 1.0)) + * (CAST(? AS DOUBLE) - CAST(? AS DOUBLE))) AS LinearExpectedKld + FROM {TableName} + WHERE COALESCE(FinalPredictedKld, PredictedKld) IS NOT NULL + AND PredictedSizeBytes IS NOT NULL + AND PredictionRank IS NOT NULL + AND {CombinationDuckDbSchema.ActiveCandidatePredicateSql} + AND {CombinationDuckDbSchema.HybridPredicateSql} + AND PredictedSizeBytes BETWEEN ? AND ? +) +SELECT COUNT(*) +FROM scored +WHERE LinearExpectedKld - PredictedKld > ?;"; + + double denominator = Math.Max( + (double)lowerDamageLarger.PredictedSizeBytes - higherDamageSmaller.PredictedSizeBytes, + 1d); + + return await ExecuteCountAsync( + sql, + new object[] + { + higherDamageSmaller.PredictedKld, + (double)higherDamageSmaller.PredictedSizeBytes, + denominator, + lowerDamageLarger.PredictedKld, + higherDamageSmaller.PredictedKld, + effectiveWindow.Value.Min, + effectiveWindow.Value.Max, + Config.SelectionMinimumKldImprovementEpsilon + }, + ct); + } + + public Task> QueryStrictDominanceCandidatesAsync( + PredictedAnchorRow anchor, + int limit, + CancellationToken ct = default) + { + return QueryStrictDominanceCandidatesAsync(anchor, anchor.PredictedSizeBytes, limit, ct); + } + + public async Task> QueryStrictDominanceCandidatesAsync( + PredictedAnchorRow anchor, + ulong maxSizeBytes, + int limit, + CancellationToken ct = default) + { + string sql = $@" +SELECT {CombinationDuckDbSchema.SlotColumnList}, + {CombinationDuckDbSchema.EffectivePredictedKldSql} AS PredictedKld, + PredictedSizeBytes, + PredictionConfidence, + PredictionRank, + COALESCE(AnomalyAdjustmentKld, 0.0) AS AnomalyAdjustmentKld +FROM {TableName} +WHERE COALESCE(FinalPredictedKld, PredictedKld) IS NOT NULL + AND PredictedSizeBytes IS NOT NULL + AND PredictionRank IS NOT NULL + AND {CombinationDuckDbSchema.ActiveCandidatePredicateSql} + AND {CombinationDuckDbSchema.HybridPredicateSql} + AND PredictedSizeBytes <= ? + AND {CombinationDuckDbSchema.EffectivePredictedKldSql} + ? < ? +ORDER BY PredictedSizeBytes ASC, + PredictedKld ASC, + PredictionRank ASC, + PredictionConfidence DESC +LIMIT ?;"; + + return await QueryPredictedRowsAsync( + sql, + new object[] { maxSizeBytes, Config.SelectionMinimumKldImprovementEpsilon, anchor.PredictedKld, limit }, + ct); + } + + public async Task> QueryBetterThanLinearCandidatesAsync( + BenchmarkSnapshotRecord higherDamageSmaller, + BenchmarkSnapshotRecord lowerDamageLarger, + PredictedAnchorRow higherDamagePredictionAnchor, + PredictedAnchorRow lowerDamagePredictionAnchor, + ulong predictionWindowMinSize, + ulong predictionWindowMaxSize, + ulong realValidationWindowMinSize, + ulong realValidationWindowMaxSize, + HybridSelectionReason reason, + string windowLabel, + int limit, + CancellationToken ct = default) + { + var effectiveWindow = IntersectSizeWindows( + predictionWindowMinSize, + predictionWindowMaxSize, + realValidationWindowMinSize, + realValidationWindowMaxSize); + + if (effectiveWindow == null) + return Array.Empty(); + + string sql = $@" +WITH scored AS ( + SELECT {CombinationDuckDbSchema.SlotColumnList}, + {CombinationDuckDbSchema.EffectivePredictedKldSql} AS PredictedKld, + PredictedSizeBytes, + PredictionConfidence, + PredictionRank, + (CAST(? AS DOUBLE) + + ((CAST(PredictedSizeBytes AS DOUBLE) - CAST(? AS DOUBLE)) / GREATEST(CAST(? AS DOUBLE), 1.0)) + * (CAST(? AS DOUBLE) - CAST(? AS DOUBLE))) AS LinearExpectedKld + FROM {TableName} + WHERE COALESCE(FinalPredictedKld, PredictedKld) IS NOT NULL + AND PredictedSizeBytes IS NOT NULL + AND PredictionRank IS NOT NULL + AND {CombinationDuckDbSchema.ActiveCandidatePredicateSql} + AND {CombinationDuckDbSchema.HybridPredicateSql} + AND PredictedSizeBytes BETWEEN ? AND ? +), +ranked AS ( + SELECT *, + LinearExpectedKld - PredictedKld AS Gain + FROM scored +) +SELECT {CombinationDuckDbSchema.SlotColumnList}, + PredictedKld, + PredictedSizeBytes, + PredictionConfidence, + PredictionRank, + LinearExpectedKld, + Gain +FROM ranked +WHERE Gain > ? +ORDER BY Gain DESC, + PredictionConfidence DESC, + PredictedSizeBytes ASC, + PredictedKld ASC, + PredictionRank ASC +LIMIT ?;"; + + double denominator = Math.Max( + (double)lowerDamagePredictionAnchor.PredictedSizeBytes - higherDamagePredictionAnchor.PredictedSizeBytes, + 1d); + + using var c = new DuckDBConnection(ConnectionString); + await c.OpenAsync(ct); + await ConfigureFastLoadSessionAsync(c, ct); + await EnsureTensorConfigsTableExistsAsync(c, ct); + + using var cmd = c.CreateCommand(); + cmd.CommandText = sql; + + foreach (var value in new object[] + { + higherDamagePredictionAnchor.PredictedKld, + (double)higherDamagePredictionAnchor.PredictedSizeBytes, + denominator, + lowerDamagePredictionAnchor.PredictedKld, + higherDamagePredictionAnchor.PredictedKld, + effectiveWindow.Value.Min, + effectiveWindow.Value.Max, + Config.SelectionMinimumKldImprovementEpsilon, + limit + }) + { + cmd.Parameters.Add(new DuckDBParameter { Value = value }); + } + + var list = new List(); + using var r = await cmd.ExecuteReaderAsync(ct); + int attempt = 0; + while (await r.ReadAsync(ct)) + { + var prediction = MapPredictedRow(r); + double line = ToDouble(r.GetValue(14)); + double gain = ToDouble(r.GetValue(15)); + + list.Add(new HybridSelectionCandidate + { + Prediction = prediction, + Reason = reason, + HigherDamageAnchor = higherDamageSmaller, + LowerDamageAnchor = lowerDamageLarger, + HigherDamagePredictionAnchor = higherDamagePredictionAnchor, + LowerDamagePredictionAnchor = lowerDamagePredictionAnchor, + PredictionWindowMinSizeBytes = predictionWindowMinSize, + PredictionWindowMaxSizeBytes = predictionWindowMaxSize, + WindowMinSizeBytes = realValidationWindowMinSize, + WindowMaxSizeBytes = realValidationWindowMaxSize, + LinearExpectedKld = line, + PredictedGainOverLine = gain, + WindowLabel = windowLabel, + AttemptOrder = ++attempt + }); + } + + return list; + } + + private static (ulong Min, ulong Max)? IntersectSizeWindows( + ulong firstMin, + ulong firstMax, + ulong secondMin, + ulong secondMax) + { + ulong min = Math.Max(firstMin, secondMin); + ulong max = Math.Min(firstMax, secondMax); + return max < min ? null : (min, max); + } + + private async Task> QueryPredictedRowsAsync( + string sql, + object[] args, + CancellationToken ct) + { + using var c = new DuckDBConnection(ConnectionString); + await c.OpenAsync(ct); + await ConfigureFastLoadSessionAsync(c, ct); + await EnsureTensorConfigsTableExistsAsync(c, ct); + + using var cmd = c.CreateCommand(); + cmd.CommandText = sql; + foreach (var arg in args) + cmd.Parameters.Add(new DuckDBParameter { Value = arg }); + + using var r = await cmd.ExecuteReaderAsync(ct); + var list = new List(); + while (await r.ReadAsync(ct)) + list.Add(MapPredictedRow(r, anomalyAdjustmentColumnIndex: r.FieldCount > 14 ? 14 : null)); + + return list; + } + + private static PredictedAnchorRow MapPredictedAnchorRow(System.Data.Common.DbDataReader r) + { + var config = ReadTensorConfig(r); + string canonicalKey = Convert.ToString(r.GetValue(11)) ?? string.Empty; + byte runtimeBaselineId = ToByte(r.GetValue(12)); + string displayName = Convert.ToString(r.GetValue(10)) ?? canonicalKey; + + return new PredictedAnchorRow + { + Config = config, + ConfigKey = TensorConfigIdentity.ToKey(config), + DisplayName = string.IsNullOrWhiteSpace(displayName) ? canonicalKey : displayName, + BaselineCanonicalKey = canonicalKey, + RuntimeBaselineId = runtimeBaselineId, + PredictedKld = ToDouble(r.GetValue(13)), + PredictedSizeBytes = ToUInt64(r.GetValue(14)), + PredictionConfidence = ToDouble(r.GetValue(15)), + PredictionRank = ToUInt64(r.GetValue(16)), + IsVirtualPredictionAnchor = ToBoolean(r.GetValue(17)) + }; + } + + private static RankSafePredictionRow MapPredictedRow(System.Data.Common.DbDataReader r, int? anomalyAdjustmentColumnIndex = null) + { + var config = ReadTensorConfig(r); + + return new RankSafePredictionRow + { + Config = config, + Quant = (HybridQuant)config, + PredictedKld = ToDouble(r.GetValue(10)), + PredictedSizeBytes = ToUInt64(r.GetValue(11)), + PredictionConfidence = ToDouble(r.GetValue(12)), + PredictedRank = ToUInt64(r.GetValue(13)), + AnomalyAdjustmentKld = anomalyAdjustmentColumnIndex.HasValue ? ToDouble(r.GetValue(anomalyAdjustmentColumnIndex.Value)) : 0d, + IsPredictable = true, + IsSizePredictable = true + }; + } + + private static TensorConfig ReadTensorConfig(System.Data.Common.DbDataReader r) + { + return new TensorConfig( + ToByte(r.GetValue(0)), + ToByte(r.GetValue(1)), + ToByte(r.GetValue(2)), + ToByte(r.GetValue(3)), + ToByte(r.GetValue(4)), + ToByte(r.GetValue(5)), + ToByte(r.GetValue(6)), + ToByte(r.GetValue(7)), + ToByte(r.GetValue(8)), + ToByte(r.GetValue(9))); + } + + private static void AddSlotParameters(DuckDBCommand command, TensorConfig config) + { + command.Parameters.Add(new DuckDBParameter { Value = config.BaseQuant }); + command.Parameters.Add(new DuckDBParameter { Value = config.Embeddings }); + command.Parameters.Add(new DuckDBParameter { Value = config.LmHead }); + command.Parameters.Add(new DuckDBParameter { Value = config.AttnQ }); + command.Parameters.Add(new DuckDBParameter { Value = config.AttnKV }); + command.Parameters.Add(new DuckDBParameter { Value = config.AttnOutput }); + command.Parameters.Add(new DuckDBParameter { Value = config.FfnUpGate }); + command.Parameters.Add(new DuckDBParameter { Value = config.FfnDown }); + command.Parameters.Add(new DuckDBParameter { Value = config.MoeExperts }); + command.Parameters.Add(new DuckDBParameter { Value = config.MoeRouter }); + } + + private async Task ExecuteCountAsync(string sql, object[] args, CancellationToken ct) + { + using var c = new DuckDBConnection(ConnectionString); + await c.OpenAsync(ct); + await ConfigureFastLoadSessionAsync(c, ct); + await EnsureTensorConfigsTableExistsAsync(c, ct); + + using var cmd = c.CreateCommand(); + cmd.CommandText = sql; + foreach (var arg in args) + cmd.Parameters.Add(new DuckDBParameter { Value = arg }); + + return ToInt64(await cmd.ExecuteScalarAsync(ct)); + } + + private static TensorConfig CanonicalizeConfigForPredictionSpace(TensorConfig config) + { + if (config.BaseQuant == BaselineQuants.Q8_0.UniqueId) + return config; + + var baseBaseline = BaselineQuants.FromId(config.BaseQuant); + byte inheritedBaseSlot = BaselineQuants.EncodeTensorConfigGroupSlot(baseBaseline); + + return new TensorConfig( + baseQuant: BaselineQuants.Q8_0.UniqueId, + embeddings: CanonicalizePredictionSlot(TReg.Embeddings, config.Embeddings, inheritedBaseSlot), + lmHead: CanonicalizePredictionSlot(TReg.LmHead, config.LmHead, inheritedBaseSlot), + attnQ: CanonicalizePredictionSlot(TReg.AttnQ, config.AttnQ, inheritedBaseSlot), + attnKV: CanonicalizePredictionSlot(TReg.AttnKV, config.AttnKV, inheritedBaseSlot), + attnOutput: CanonicalizePredictionSlot(TReg.AttnOutput, config.AttnOutput, inheritedBaseSlot), + ffnUpGate: CanonicalizePredictionSlot(TReg.FfnUpGate, config.FfnUpGate, inheritedBaseSlot), + ffnDown: CanonicalizePredictionSlot(TReg.FfnDown, config.FfnDown, inheritedBaseSlot), + moeExperts: CanonicalizePredictionSlot(TReg.MoeExperts, config.MoeExperts, inheritedBaseSlot), + moeRouter: CanonicalizePredictionSlot(TReg.MoeRouter, config.MoeRouter, inheritedBaseSlot)); + } + + private static byte CanonicalizePredictionSlot(TensorGroup group, byte storedValue, byte inheritedBaseSlot) + { + if (Cache.UnusedTensorGroups.Any(x => x.UniqueId == group.UniqueId)) + return BaselineQuants.TensorConfigNullSlotValue; + + return BaselineQuants.IsNullTensorConfigGroupSlot(storedValue) + ? inheritedBaseSlot + : storedValue; + } + + private static string BuildSlotPredicateSql(TensorConfig config) + { + return $"BaseQuant = {config.BaseQuant} AND Embeddings = {config.Embeddings} AND LmHead = {config.LmHead} AND AttnQ = {config.AttnQ} AND AttnKV = {config.AttnKV} AND AttnOutput = {config.AttnOutput} AND FfnUpGate = {config.FfnUpGate} AND FfnDown = {config.FfnDown} AND MoeExperts = {config.MoeExperts} AND MoeRouter = {config.MoeRouter}"; + } + + private static string NormalizeAnchorKey(string? value) => (value ?? string.Empty).Trim().ToLowerInvariant(); + + private static long ToInt64(object? value) + { + if (value is null || value is DBNull) + return 0L; + + if (value is BigInteger big) + return (long)big; + + return Convert.ToInt64(value); + } + + private static ulong ToUInt64(object? value) + { + if (value is null || value is DBNull) + return 0UL; + + if (value is BigInteger big) + return (ulong)big; + + return Convert.ToUInt64(value); + } + + private static byte ToByte(object? value) + { + if (value is null || value is DBNull) + return 0; + + if (value is BigInteger big) + return (byte)big; + + return Convert.ToByte(value); + } + + private static double ToDouble(object? value) + { + if (value is null || value is DBNull) + return 0d; + + if (value is BigInteger big) + return (double)big; + + return Convert.ToDouble(value); + } + + private static bool ToBoolean(object? value) + { + if (value is null || value is DBNull) + return false; + + if (value is bool b) + return b; + + if (value is BigInteger big) + return big != BigInteger.Zero; + + return Convert.ToBoolean(value); + } + + private static async Task ConfigureFastLoadSessionAsync(DuckDBConnection connection, CancellationToken ct) + { + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = "SET preserve_insertion_order = false;"; + await cmd.ExecuteNonQueryAsync(ct); + } + + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = $"SET threads = {Math.Max(1, Environment.ProcessorCount)};"; + await cmd.ExecuteNonQueryAsync(ct); + } + } + + private static async Task RecreateTableAsync(DuckDBConnection connection, CancellationToken ct) + { + using var createCmd = connection.CreateCommand(); + createCmd.CommandText = CombinationDuckDbSchema.CreateTableSql; + await createCmd.ExecuteNonQueryAsync(ct); + } + + private static async Task EnsureTensorConfigsTableExistsAsync(DuckDBConnection connection, CancellationToken ct) + { + using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM information_schema.tables WHERE table_name = ?;"; + cmd.Parameters.Add(new DuckDBParameter { Value = TableName }); + + long matches = ToInt64(await cmd.ExecuteScalarAsync(ct)); + if (matches > 0) + return; + + throw new InvalidOperationException( + $"DuckDB search-space table '{TableName}' does not exist in '{CombinationDatabasePathService.GetPath()}'. " + + "This almost always means the generator and prediction reader are using different DuckDB filenames, " + + "or prediction started before QuantDatabaseService initialized/rebuilt the search-space table."); + } +} \ No newline at end of file diff --git a/src/MagicQuant/Services/RepositoryCloneManifestService.cs b/src/MagicQuant/Services/RepositoryCloneManifestService.cs new file mode 100644 index 0000000..7153bd8 --- /dev/null +++ b/src/MagicQuant/Services/RepositoryCloneManifestService.cs @@ -0,0 +1,155 @@ +using System.Text.Json; +using MagicQuant.Models; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class RepositoryCloneManifestService +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true, + ReadCommentHandling = JsonCommentHandling.Skip, + AllowTrailingCommas = true + }; + + private readonly HuggingFaceBaselineService _huggingFace; + + public RepositoryCloneManifestService(HuggingFaceBaselineService huggingFace) + { + _huggingFace = huggingFace; + } + + public async Task<(MagicQuantCloneManifest Manifest, string LocalPath, string SourceDescription)> ResolveAsync( + string? sourceRepo, + string? sourceJson, + string modelMagicQuantDirectory, + CancellationToken ct = default) + { + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(ct, MagicQuant.Runtime.RunCancellation.Token); + ct = runCancellation.Token; + ct.ThrowIfCancellationRequested(); + if (string.IsNullOrWhiteSpace(sourceRepo) && string.IsNullOrWhiteSpace(sourceJson)) + throw new InvalidOperationException("Clone mode requires --source-repo or --source-json ."); + + string cloneDir = Path.Combine(modelMagicQuantDirectory, "CloneSource"); + Directory.CreateDirectory(cloneDir); + + string localPath; + string sourceDescription; + + if (!string.IsNullOrWhiteSpace(sourceRepo)) + { + string repo = sourceRepo.Trim(); + sourceDescription = repo; + localPath = await DownloadRequiredCloneManifestFromRepoAsync(repo, cloneDir, ct); + } + else + { + string raw = sourceJson!.Trim(); + + if (raw.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || + raw.StartsWith("https://", StringComparison.OrdinalIgnoreCase)) + { + localPath = Path.Combine(MagicQuantManifestPathService.EnsureManifestDirectory(cloneDir), MagicQuantManifestPathService.CloneConfigsFileName); + sourceDescription = raw; + + using var http = new HttpClient(); + var json = await http.GetStringAsync(raw, ct); + await File.WriteAllTextAsync(localPath, json, ct); + } + else + { + string resolved = Path.GetFullPath(raw); + sourceDescription = resolved; + + if (!File.Exists(resolved)) + throw new FileNotFoundException($"Clone JSON file does not exist: {resolved}"); + + string copied = Path.Combine(MagicQuantManifestPathService.EnsureManifestDirectory(cloneDir), MagicQuantManifestPathService.CloneConfigsFileName); + File.Copy(resolved, copied, overwrite: true); + localPath = copied; + } + } + + var manifest = JsonSerializer.Deserialize( + await File.ReadAllTextAsync(localPath, ct), + JsonOptions); + + if (manifest == null) + throw new InvalidOperationException($"Clone manifest could not be parsed: {localPath}"); + + ValidateManifest(manifest, localPath); + + AnsiConsole.MarkupLine($"[green]Clone manifest loaded:[/] {Markup.Escape(localPath)} artifacts={manifest.Artifacts.Count:N0}"); + return (manifest, localPath, sourceDescription); + } + + private async Task DownloadRequiredCloneManifestFromRepoAsync(string repoId, string cloneDir, CancellationToken ct) + { + string manifestDir = MagicQuantManifestPathService.EnsureManifestDirectory(cloneDir); + string localPath = Path.Combine(manifestDir, MagicQuantManifestPathService.CloneConfigsFileName); + + var candidates = new[] + { + MagicQuantManifestPathService.RelativeManifestPath(MagicQuantManifestPathService.CloneConfigsFileName), + MagicQuantManifestPathService.CloneConfigsFileName + }; + + var errors = new List(); + foreach (var candidate in candidates) + { + try + { + await _huggingFace.DownloadRepositoryFileAsync( + repoId: repoId, + fileName: candidate, + destinationPath: localPath, + forceRedownload: true, + ct: ct); + + AnsiConsole.MarkupLine($"[green]Downloaded clone manifest:[/] {Markup.Escape(repoId)}/{Markup.Escape(candidate)}"); + return localPath; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + errors.Add($"{candidate}: {ex.Message}"); + } + } + + throw new InvalidOperationException( + $"Could not download clone manifest from Hugging Face repo '{repoId}'. Tried new manifest folder path and legacy root path." + + Environment.NewLine + string.Join(Environment.NewLine, errors.Select(x => "- " + x))); + } + + private static void ValidateManifest(MagicQuantCloneManifest manifest, string localPath) + { + if (manifest.SchemaVersion <= 0) + throw new InvalidOperationException($"Clone manifest has invalid schemaVersion in {localPath}."); + + if (manifest.Artifacts.Count == 0) + throw new InvalidOperationException($"Clone manifest contains zero artifacts: {localPath}"); + + var duplicateFiles = manifest.Artifacts + .Where(x => !string.IsNullOrWhiteSpace(x.FileName)) + .GroupBy(x => x.FileName, StringComparer.OrdinalIgnoreCase) + .Where(g => g.Count() > 1) + .Select(g => g.Key) + .ToList(); + + if (duplicateFiles.Count > 0) + throw new InvalidOperationException($"Clone manifest contains duplicate file names: {string.Join(", ", duplicateFiles)}"); + + foreach (var artifact in manifest.Artifacts) + { + if (string.IsNullOrWhiteSpace(artifact.FileName)) + throw new InvalidOperationException("Clone manifest artifact is missing fileName."); + + if (!artifact.FileName.EndsWith(".gguf", StringComparison.OrdinalIgnoreCase)) + throw new InvalidOperationException($"Clone manifest artifact fileName must end with .gguf: {artifact.FileName}"); + + if (artifact.TensorTypes.Count == 0) + throw new InvalidOperationException($"Clone manifest artifact '{artifact.FileName}' has no tensorTypes map."); + } + } +} diff --git a/src/MagicQuant/Services/RunProvenanceService.cs b/src/MagicQuant/Services/RunProvenanceService.cs new file mode 100644 index 0000000..7f7ba85 --- /dev/null +++ b/src/MagicQuant/Services/RunProvenanceService.cs @@ -0,0 +1,98 @@ +using System.Reflection; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text.Json; +using MagicQuant.Configuration; +using MagicQuant.Helpers; +using MagicQuant.Runtime; +using MQ.DB; + +namespace MagicQuant.Services; + +/// +/// Records local campaign inputs and completion independently of export cleanup. +/// It is not published into model cards: config/argv can contain private paths or URLs. +/// +public sealed class RunProvenanceService +{ + private readonly string _path; + private readonly Dictionary _record; + private static readonly JsonSerializerOptions JsonOptions = new() { WriteIndented = true }; + + public RunProvenanceService(string command, string[] args, MagicQuantYamlLoader.LoadedConfiguration loaded) + { + string root = string.IsNullOrWhiteSpace(loaded.Settings.Paths.ModelDir) + ? loaded.Settings.Paths.MagicQuantRoot! + : Path.Combine(Path.GetFullPath(loaded.Settings.Paths.ModelDir), "MagicQuant"); + string runId = $"{DateTime.UtcNow:yyyyMMddTHHmmssfffZ}-{Guid.NewGuid():N}"; + _path = Path.Combine(root, "Runs", runId, "run.json"); + _record = new() + { + ["schemaVersion"] = 1, + ["runId"] = runId, + ["command"] = command, + ["arguments"] = args, + ["startedUtc"] = DateTimeOffset.UtcNow, + ["status"] = "running", + ["programVersion"] = typeof(RunProvenanceService).Assembly.GetCustomAttribute()?.InformationalVersion, + ["dotnetVersion"] = Environment.Version.ToString(), + ["operatingSystem"] = RuntimeInformation.OSDescription, + ["configPath"] = loaded.Path, + ["configSha256"] = Convert.ToHexString(SHA256.HashData(File.ReadAllBytes(loaded.Path))).ToLowerInvariant(), + // Serialize now: dynamic custom-baseline registration must not rewrite the input snapshot. + ["configuration"] = JsonSerializer.SerializeToElement(loaded.Settings, JsonOptions) + }; + Write(); + } + + public string ManifestPath => _path; + + public async Task CaptureToolchainAsync() + { + _record["llamaRoot"] = Cache.LlamaRoot; + _record["llamaBin"] = Cache.LlamaBin; + _record["llamaRevision"] = await TryReadToolAsync("git", ["-C", Cache.LlamaRoot ?? "", "rev-parse", "HEAD"]); + string python = new PythonManager(Cache.MagicQuantDirectory!).GetPythonExecutable(); + _record["pythonExecutable"] = python; + _record["pythonVersion"] = await TryReadToolAsync(python, ["--version"]); + _record["pythonPackages"] = await TryReadToolAsync(python, ["-m", "pip", "freeze"]); + Write(); + } + + public void Complete(string status, string? error = null) + { + _record["status"] = status; + _record["completedUtc"] = DateTimeOffset.UtcNow; + _record["error"] = error; + _record["modelId"] = Cache.CurrentModelId; + _record["architectureFamily"] = Cache.CurrentArchitectureFamilyName; + _record["tensorGroupProfile"] = Cache.CurrentTensorGroupProfileFingerprintHash; + _record["imatrixIdentity"] = Cache.ActiveImatrixIdentityHash; + _record["outputDirectory"] = Cache.OutputDirectory; + Write(); + } + + private void Write() + { + Directory.CreateDirectory(Path.GetDirectoryName(_path)!); + string temporary = _path + ".tmp"; + try + { + File.WriteAllText(temporary, JsonSerializer.Serialize(_record, JsonOptions)); + File.Move(temporary, _path, overwrite: true); + } + finally { if (File.Exists(temporary)) File.Delete(temporary); } + } + + private static async Task TryReadToolAsync(string executable, string[] args) + { + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + try + { + var result = await new ProcessRunner().RunAsync(new NativeCommand(executable, args).CreateStartInfo(), ct: timeout.Token); + return result.Success ? result.CombinedOutput.Trim() : null; + } + catch (OperationCanceledException) when (!RunCancellation.Token.IsCancellationRequested) { return null; } + catch (System.ComponentModel.Win32Exception) { return null; } + } +} diff --git a/src/MagicQuant/Services/ScratchStorageService.cs b/src/MagicQuant/Services/ScratchStorageService.cs new file mode 100644 index 0000000..9622f52 --- /dev/null +++ b/src/MagicQuant/Services/ScratchStorageService.cs @@ -0,0 +1,234 @@ +using System.Text.Json; +using MagicQuant.Helpers; +using MQ.DB; + +namespace MagicQuant.Services; + +public enum ScratchArtifactKind +{ + QuantizedSample, + PureQ8Probe, + ExternalBaselineRebuild, + ExternalBaselineNormalizedSample, + ExportTemp, + MetadataRead, + Other +} + +public sealed class ScratchArtifactLease : IAsyncDisposable +{ + private readonly Func _dispose; + private int _disposed; + + internal ScratchArtifactLease( + Guid leaseId, + ScratchArtifactKind kind, + string scratchRoot, + string leaseDirectory, + string ggufPath, + string primaryLogPath, + Func dispose) + { + LeaseId = leaseId; + Kind = kind; + ScratchRoot = scratchRoot; + LeaseDirectory = leaseDirectory; + GgufPath = ggufPath; + PrimaryLogPath = primaryLogPath; + OwnsGgufLifecycle = true; + _dispose = dispose; + } + + public Guid LeaseId { get; } + public ScratchArtifactKind Kind { get; } + public string ScratchRoot { get; } + public string LeaseDirectory { get; } + public string GgufPath { get; } + public string PrimaryLogPath { get; } + public bool OwnsGgufLifecycle { get; private set; } + + public void PreserveOutput() => OwnsGgufLifecycle = false; + + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref _disposed, 1) != 0) + return; + + await _dispose(this); + } +} + +public sealed class ScratchStorageService +{ + private const string ScratchFolderName = ".MagicQuant_tmp"; + private readonly ModelArtifactPathService? _paths; + private readonly string _modelNamespace; + private readonly string _quantLogDir; + private readonly IReadOnlyList _roots; + private readonly SemaphoreSlim[] _rootLocks; + private readonly SemaphoreSlim _availableRoots; + private int _cursor = -1; + + public ScratchStorageService(ModelArtifactPathService? paths = null) + { + _paths = paths; + _modelNamespace = paths?.ScratchModelNamespace ?? "global"; + _quantLogDir = paths?.QuantizationLogsDir ?? Path.Combine(Cache.MagicQuantDirectory ?? Path.GetTempPath(), "Logs", "Quantization"); + var configured = Cache.ScratchRoots ?? []; + + var fallbackRoot = paths != null + ? Path.Combine(paths.ModelMagicQuantDirectory, ScratchFolderName) + : Path.Combine(Cache.MagicQuantDirectory ?? Path.GetTempPath(), ScratchFolderName); + + _roots = configured.Count == 0 + ? [fallbackRoot] + : configured.Select(x => Path.GetFullPath(x)).ToList(); + + _rootLocks = _roots.Select(_ => new SemaphoreSlim(1, 1)).ToArray(); + _availableRoots = new SemaphoreSlim(_roots.Count, _roots.Count); + } + + public int WriterCapacity => _roots.Count; + public IReadOnlyList ConfiguredScratchRoots => _roots; + + public async Task AcquireAsync( + ScratchArtifactKind kind, + string artifactBaseName, + string extension = ".gguf", + CancellationToken ct = default) + { + await _availableRoots.WaitAsync(ct); + + int rootIndex = -1; + try + { + while (rootIndex < 0) + { + int start = (Interlocked.Increment(ref _cursor) % _roots.Count + _roots.Count) % _roots.Count; + for (int i = 0; i < _roots.Count; i++) + { + int idx = (start + i) % _roots.Count; + if (_rootLocks[idx].Wait(0)) + { + rootIndex = idx; + break; + } + } + + if (rootIndex < 0) + await Task.Delay(20, ct); + } + + var leaseId = Guid.NewGuid(); + string root = _roots[rootIndex]; + string tmpRoot = root.EndsWith(ScratchFolderName, StringComparison.OrdinalIgnoreCase) + ? root + : Path.Combine(root, ScratchFolderName); + + string leaseDir = Path.Combine(tmpRoot, _modelNamespace, leaseId.ToString("N")); + Directory.CreateDirectory(leaseDir); + Directory.CreateDirectory(_quantLogDir); + + string safeBase = ModelArtifactPathService.MakeSafeFileComponent(artifactBaseName); + string ggufPath = Path.Combine(leaseDir, safeBase + extension); + string logPath = _paths?.GetQuantizationLogPath(safeBase, leaseId) ?? Path.Combine(_quantLogDir, $"{safeBase}-{leaseId:N}.quantize.log"); + + var marker = new + { + lease_id = leaseId, + process_id = Environment.ProcessId, + started_utc = DateTime.UtcNow, + kind = kind.ToString(), + artifact_name = artifactBaseName, + gguf_path = ggufPath + }; + await File.WriteAllTextAsync(Path.Combine(leaseDir, "lease.json"), JsonSerializer.Serialize(marker), ct); + + return new ScratchArtifactLease( + leaseId, + kind, + root, + leaseDir, + ggufPath, + logPath, + async lease => + { + try + { + if (lease.OwnsGgufLifecycle && Directory.Exists(lease.LeaseDirectory)) + await HardDeleteHelper.DeleteDirectoryIfExistsAsync(lease.LeaseDirectory); + } + finally + { + _rootLocks[rootIndex].Release(); + _availableRoots.Release(); + } + }); + } + catch + { + if (rootIndex >= 0) + _rootLocks[rootIndex].Release(); + _availableRoots.Release(); + throw; + } + } + + public async Task CleanupStaleScratchArtifactsAsync(CancellationToken ct = default) + { + foreach (var root in _roots) + { + ct.ThrowIfCancellationRequested(); + string tmpRoot = root.EndsWith(ScratchFolderName, StringComparison.OrdinalIgnoreCase) + ? root + : Path.Combine(root, ScratchFolderName); + + if (!Directory.Exists(tmpRoot)) + continue; + + foreach (var child in Directory.EnumerateDirectories(tmpRoot)) + { + ct.ThrowIfCancellationRequested(); + // Legacy single-level lease folder support. + if (IsLeaseDirectory(child)) + { + await HardDeleteHelper.DeleteDirectoryIfExistsAsync(child); + continue; + } + + // Model namespace folder: only remove known lease children. + bool containsOnlyLeaseDirs = !Directory.EnumerateFiles(child).Any(); + foreach (var leaseDir in Directory.EnumerateDirectories(child)) + { + ct.ThrowIfCancellationRequested(); + if (!IsLeaseDirectory(leaseDir)) + { + containsOnlyLeaseDirs = false; + continue; + } + + await HardDeleteHelper.DeleteDirectoryIfExistsAsync(leaseDir); + } + + if (containsOnlyLeaseDirs && + !Directory.EnumerateDirectories(child).Any() && + !Directory.EnumerateFiles(child).Any()) + { + await HardDeleteHelper.DeleteDirectoryIfExistsAsync(child); + } + } + } + } + + private static bool IsLeaseDirectory(string directoryPath) + { + if (!Directory.Exists(directoryPath)) + return false; + + string name = Path.GetFileName(directoryPath); + if (!Guid.TryParseExact(name, "N", out _)) + return false; + + return File.Exists(Path.Combine(directoryPath, "lease.json")); + } +} diff --git a/src/MagicQuant/Services/SelectionDiagnosticsLogService.cs b/src/MagicQuant/Services/SelectionDiagnosticsLogService.cs new file mode 100644 index 0000000..6527211 --- /dev/null +++ b/src/MagicQuant/Services/SelectionDiagnosticsLogService.cs @@ -0,0 +1,249 @@ +using System.Text.Json; +using MagicQuant.Models; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class SelectionDiagnosticsLogService +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + WriteIndented = true + }; + + public async Task WriteAsync( + IReadOnlyCollection benchmarkOverview, + IReadOnlyCollection validationFailures, + CancellationToken ct = default) + { + string directory = ResolveGgufDirectory(); + Directory.CreateDirectory(directory); + + string overviewPath = Path.Combine(directory, "magicquant-benchmark-overview.json"); + string missesPath = Path.Combine(directory, "magicquant-selection-validation-misses.json"); + + var overview = benchmarkOverview + .DistinctBy(x => TensorConfigIdentity.ToKey(x.Config)) + .OrderBy(x => x.Kld) + .ThenBy(x => x.SizeBytes) + .Select(ToSnapshotLog) + .ToList(); + + var misses = validationFailures + .Where(x => !x.Accepted) + .Select(ToFailureLog) + .ToList(); + + await File.WriteAllTextAsync(overviewPath, JsonSerializer.Serialize(overview, JsonOptions), ct); + await File.WriteAllTextAsync(missesPath, JsonSerializer.Serialize(misses, JsonOptions), ct); + + AnsiConsole.MarkupLine($"[green]Benchmark overview log:[/] {Markup.Escape(overviewPath)}"); + AnsiConsole.MarkupLine($"[green]Prediction miss log:[/] {Markup.Escape(missesPath)}"); + } + + private static object ToSnapshotLog(BenchmarkSnapshotRecord snap) + { + return new + { + key = TensorConfigIdentity.ToKey(snap.Config), + displayName = snap.DisplayName, + provider = snap.ProviderName, + baselineFamily = snap.BaselineFamily, + isHybrid = snap.IsHybrid, + isExternalPureBaseline = snap.IsExternalPureBaseline, + isExternalRebuiltBaseline = snap.IsExternalRebuiltBaseline, + isMaterializedTensorMapped = snap.IsMaterializedTensorMapped, + sizeBytes = snap.SizeBytes, + sizeGiB = ToGb(snap.SizeBytes), + kld = snap.Kld, + ppl = snap.Ppl, + outputModelPath = snap.OutputModelPath, + externalRepositoryUrl = snap.ExternalRepositoryUrl + }; + } + + private static object ToFailureLog(CandidateValidationResult failure) + { + var c = failure.Candidate; + var snap = failure.Snapshot; + + double? actualLine = null; + double? actualGainOverLine = null; + long? sizeMissBytes = null; + double? kldMiss = null; + bool? actualInsideSizeWindow = null; + bool? actualBeatLine = null; + + if (snap != null) + { + actualLine = InterpolateKldLine(snap.SizeBytes, c.HigherDamageAnchor, c.LowerDamageAnchor); + actualGainOverLine = actualLine.Value - snap.Kld; + actualInsideSizeWindow = snap.SizeBytes >= c.WindowMinSizeBytes && snap.SizeBytes <= c.WindowMaxSizeBytes; + actualBeatLine = snap.Kld + Config.SelectionMinimumKldImprovementEpsilon < actualLine.Value; + + if (snap.SizeBytes < c.WindowMinSizeBytes) + sizeMissBytes = (long)c.WindowMinSizeBytes - (long)snap.SizeBytes; + else if (snap.SizeBytes > c.WindowMaxSizeBytes) + sizeMissBytes = (long)snap.SizeBytes - (long)c.WindowMaxSizeBytes; + else + sizeMissBytes = 0; + + kldMiss = snap.Kld + Config.SelectionMinimumKldImprovementEpsilon - actualLine.Value; + } + + return new + { + reason = c.Reason.ToString(), + attemptOrder = c.AttemptOrder, + attemptLimit = c.CandidateAttemptLimit, + windowLabel = c.WindowLabel, + phaseWindowIndex = c.PhaseWindowIndex, + phaseWindowCount = c.PhaseWindowCount, + candidateKey = TensorConfigIdentity.ToKey(c.Prediction.Config), + candidateInternalName = HybridBenchmarkRepository.BuildDisplayName(c.Prediction.Quant), + bitSpace = DescribeBitSpace(c.Prediction.Config), + overrideSummary = DescribeOverrides(c.Prediction.Config), + baseQuant = c.Prediction.Quant.BaseQuant.Names[0], + baseBitRange = c.Prediction.Quant.BaseQuant.BitRange, + failureCode = failure.FailureCode, + predicted = new + { + sizeBytes = c.Prediction.PredictedSizeBytes, + sizeGiB = ToGb(c.Prediction.PredictedSizeBytes), + kld = c.Prediction.PredictedKld, + lineKldAtPredictedSize = c.LinearExpectedKld, + gainOverLine = c.PredictedGainOverLine, + confidence = c.Prediction.PredictionConfidence, + rank = c.Prediction.PredictedRank + }, + selectionContext = new + { + candidatePoolSize = c.CandidatePoolSize, + windowCandidateCount = c.WindowCandidateCount, + lineBeatingCandidateCount = c.LineBeatingCandidateCount, + fetchedCandidateCount = c.FetchedCandidateCount, + candidatesAfterBrutalityCount = c.CandidatesAfterBrutalityCount, + candidateAttemptLimit = c.CandidateAttemptLimit, + rawSelectionRank = c.RawSelectionRank, + diversityMode = c.DiversityMode, + candidateTheoryFamilyKey = c.CandidateTheoryFamilyKey, + candidateTheoryFamilyDisplay = c.CandidateTheoryFamilyDisplay, + candidateTheoryFamilyRank = c.CandidateTheoryFamilyRank, + candidateTheoryFamilyMemberRank = c.CandidateTheoryFamilyMemberRank, + notes = c.CandidateSelectionNotes + }, + actual = snap == null + ? null + : new + { + displayName = snap.DisplayName, + sizeBytes = snap.SizeBytes, + sizeGiB = ToGb(snap.SizeBytes), + kld = snap.Kld, + ppl = snap.Ppl, + lineKldAtActualSize = actualLine, + gainOverLine = actualGainOverLine, + insideSizeWindow = actualInsideSizeWindow, + beatLine = actualBeatLine, + sizeMissBytes, + kldMiss, + positiveKldShortfall = kldMiss.HasValue ? (double?)Math.Max(0d, kldMiss.Value) : null + }, + anchors = new + { + higherDamageSmaller = ToAnchorLog(c.HigherDamageAnchor), + lowerDamageLarger = ToAnchorLog(c.LowerDamageAnchor) + }, + acceptancePolicy = new + { + minimumKldImprovementEpsilon = Config.SelectionMinimumKldImprovementEpsilon, + windowMinSizeBytes = c.WindowMinSizeBytes, + windowMaxSizeBytes = c.WindowMaxSizeBytes, + mustBeatLineByEpsilon = true + }, + accepted = failure.Accepted, + message = failure.Message + }; + } + + private static object ToAnchorLog(BenchmarkSnapshotRecord anchor) + { + return new + { + key = TensorConfigIdentity.ToKey(anchor.Config), + displayName = anchor.DisplayName, + provider = anchor.ProviderName, + baselineFamily = anchor.BaselineFamily, + sizeBytes = anchor.SizeBytes, + sizeGiB = ToGb(anchor.SizeBytes), + kld = anchor.Kld, + ppl = anchor.Ppl, + bitRange = anchor.Quant.BaseQuant.BitRange, + quantizeBase = anchor.Quant.BaseQuant.QuantizeBaseArgumentName + }; + } + + private static double InterpolateKldLine( + ulong candidateSize, + BenchmarkSnapshotRecord higherDamageSmaller, + BenchmarkSnapshotRecord lowerDamageLarger) + { + ulong smallSize = higherDamageSmaller.SizeBytes; + ulong largeSize = lowerDamageLarger.SizeBytes; + + if (largeSize <= smallSize) + return Math.Min(higherDamageSmaller.Kld, lowerDamageLarger.Kld); + + double t = Math.Clamp((candidateSize - smallSize) / (double)(largeSize - smallSize), 0d, 1d); + return higherDamageSmaller.Kld + ((lowerDamageLarger.Kld - higherDamageSmaller.Kld) * t); + } + + + private static string DescribeBitSpace(TensorConfig config) + { + var baseQuant = BaselineQuants.FromId(config.BaseQuant); + var overrides = DescribeOverrides(config); + return string.IsNullOrWhiteSpace(overrides) + ? $"base={baseQuant.Names[0]}({baseQuant.BitRange}b); overrides=inherit-all" + : $"base={baseQuant.Names[0]}({baseQuant.BitRange}b); overrides={overrides}"; + } + + private static string DescribeOverrides(TensorConfig config) + { + var parts = new List(); + AddOverride(parts, "E", config.Embeddings); + AddOverride(parts, "H", config.LmHead); + AddOverride(parts, "Q", config.AttnQ); + AddOverride(parts, "K", config.AttnKV); + AddOverride(parts, "O", config.AttnOutput); + AddOverride(parts, "U", config.FfnUpGate); + AddOverride(parts, "D", config.FfnDown); + AddOverride(parts, "X", config.MoeExperts); + AddOverride(parts, "R", config.MoeRouter); + return string.Join(", ", parts); + } + + private static void AddOverride(List parts, string groupToken, byte storedSlot) + { + if (BaselineQuants.IsNullTensorConfigGroupSlot(storedSlot)) + return; + + var baseline = BaselineQuants.DecodeTensorConfigGroupSlotToBaseline(storedSlot); + parts.Add($"{groupToken}:{baseline.Names[0]}({baseline.BitRange}b)"); + } + + private static string ResolveGgufDirectory() + { + if (!string.IsNullOrWhiteSpace(Cache.ModelMagicQuantDirectory)) + return Path.Combine(Cache.ModelMagicQuantDirectory!, "GGUF"); + + if (!string.IsNullOrWhiteSpace(Cache.MagicQuantDirectory)) + return Path.Combine(Cache.MagicQuantDirectory!, "GGUF"); + + return Path.Combine(Directory.GetCurrentDirectory(), "GGUF"); + } + + private static string ToGb(ulong bytes) => (bytes / 1024d / 1024d / 1024d).ToString("0.00"); +} \ No newline at end of file diff --git a/src/MagicQuant/Services/SmartBaselineTuningFallbackService.cs b/src/MagicQuant/Services/SmartBaselineTuningFallbackService.cs new file mode 100644 index 0000000..3056ef7 --- /dev/null +++ b/src/MagicQuant/Services/SmartBaselineTuningFallbackService.cs @@ -0,0 +1,876 @@ +using MagicQuant.Helpers; +using MagicQuant.Models; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Services; + +/// +/// Conservative, non-DuckDB fallback used only after the normal prediction-guided +/// selector fails to validate a candidate for a strict/premium/interior phase. +/// +/// The service starts from the real benchmarked pure/uniform baseline anchor, keeps +/// that blanket baseline available for every group even if the baseline was pruned +/// from a group, and only swaps in group candidates that survived isolation pruning. +/// It does not create the normal prediction-space gremlin trades: shrinking is only +/// allowed when the isolated group sample is same-size-or-smaller and measurably lower +/// KLD than the blanket state; higher-fidelity protection is bounded by config and +/// must fit the target real-size window exactly. +/// +public sealed class SmartBaselineTuningFallbackService +{ + private const double KldEpsilon = 1e-12d; + + private readonly HybridBenchmarkRepository _repository; + private RankSafeKldPredictionService.RankSafePredictionModel? _context; + + public SmartBaselineTuningFallbackService(HybridBenchmarkRepository repository) + { + _repository = repository; + } + + public async Task> BuildStrictDominanceCandidatesAsync( + BenchmarkSnapshotRecord anchor, + int phaseWindowIndex, + int phaseWindowCount, + CancellationToken ct = default) + { + if (!Config.SelectionSmartFallbackEnabled) + return Array.Empty(); + + return await BuildCandidatesAsync(new SmartFallbackRequest + { + Reason = HybridSelectionReason.SmartStrictDominanceFallback, + WindowLabel = $"smart strict baseline tuning vs {anchor.DisplayName}", + BaselineAnchor = anchor, + HigherDamageAnchor = anchor, + LowerDamageAnchor = anchor, + WindowMinSizeBytes = 0, + WindowMaxSizeBytes = anchor.SizeBytes, + StrictDominance = true, + PhaseWindowIndex = phaseWindowIndex, + PhaseWindowCount = phaseWindowCount + }, ct); + } + + public async Task> BuildNearBaselineCandidatesAsync( + BenchmarkSnapshotRecord lowerSizeHigherDamage, + BenchmarkSnapshotRecord upperSizeLowerDamage, + ulong realMinSizeBytes, + ulong realMaxSizeBytes, + int phaseWindowIndex, + int phaseWindowCount, + CancellationToken ct = default) + { + if (!Config.SelectionSmartFallbackEnabled) + return Array.Empty(); + + return await BuildCandidatesAsync(new SmartFallbackRequest + { + Reason = HybridSelectionReason.SmartNearBaselineFallback, + WindowLabel = $"smart near-baseline tuning {lowerSizeHigherDamage.DisplayName} → {upperSizeLowerDamage.DisplayName}", + BaselineAnchor = lowerSizeHigherDamage, + HigherDamageAnchor = lowerSizeHigherDamage, + LowerDamageAnchor = upperSizeLowerDamage, + WindowMinSizeBytes = realMinSizeBytes, + WindowMaxSizeBytes = realMaxSizeBytes, + StrictDominance = false, + PhaseWindowIndex = phaseWindowIndex, + PhaseWindowCount = phaseWindowCount + }, ct); + } + + public async Task> BuildInteriorCandidatesAsync( + BenchmarkSnapshotRecord lowerSizeHigherDamage, + BenchmarkSnapshotRecord upperSizeLowerDamage, + ulong realMinSizeBytes, + ulong realMaxSizeBytes, + string windowLabel, + int phaseWindowIndex, + int phaseWindowCount, + CancellationToken ct = default) + { + if (!Config.SelectionSmartFallbackEnabled) + return Array.Empty(); + + return await BuildCandidatesAsync(new SmartFallbackRequest + { + Reason = HybridSelectionReason.SmartInteriorSubspaceFallback, + WindowLabel = $"smart interior tuning {windowLabel}", + BaselineAnchor = lowerSizeHigherDamage, + HigherDamageAnchor = lowerSizeHigherDamage, + LowerDamageAnchor = upperSizeLowerDamage, + WindowMinSizeBytes = realMinSizeBytes, + WindowMaxSizeBytes = realMaxSizeBytes, + StrictDominance = false, + PhaseWindowIndex = phaseWindowIndex, + PhaseWindowCount = phaseWindowCount + }, ct); + } + + private async Task> BuildCandidatesAsync( + SmartFallbackRequest request, + CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + + if (!TryResolveBaselineBlanket(request.BaselineAnchor, out var blanketBaseline, out var skipReason)) + { + AnsiConsole.MarkupLine($"[grey]Smart fallback skipped:[/] {Markup.Escape(skipReason)}"); + return Array.Empty(); + } + + var context = await GetContextAsync(ct); + var blanketAnchor = ResolveSmartBlanketAnchor(request.BaselineAnchor, blanketBaseline, context); + var missingIsolationGroups = GetMissingIsolationGroups(blanketBaseline, context); + var activeExplicitCandidates = BaselineQuants.GetGroupCombinationCandidates( + RuntimeSearchSpace.HasUsableImatrix(), + Config.Current.Flags.AllowHighPrecisionHybrids); + + if (!IsEligibleForSmartFallbackTuning( + blanketBaseline, + activeExplicitCandidates, + hasCompleteIsolationCoverage: missingIsolationGroups.Count == 0)) + { + AnsiConsole.MarkupLine( + $"[grey]Smart fallback skipped:[/] baseline [cyan]{Markup.Escape(blanketBaseline.Names[0])}[/] is learning/context-only in the active search space and intentionally lacks isolated group truth for " + + $"{missingIsolationGroups.Count:N0}/{context.ActiveGroups.Count:N0} active group(s). Enable it as an explicit group candidate before smart blanket tuning."); + return Array.Empty(); + } + + ValidateBlanketIsolationCoverage(blanketBaseline, context, missingIsolationGroups); + + var baseSize = blanketAnchor.SizeBytes; + var baseKld = Math.Max(0d, blanketAnchor.Kld); + + if (request.StrictDominance && baseSize > request.WindowMaxSizeBytes) + { + AnsiConsole.MarkupLine($"[grey]Smart strict fallback skipped:[/] {Markup.Escape(blanketBaseline.Names[0])} blanket is larger than the strict anchor."); + return Array.Empty(); + } + + var options = BuildGroupOptions(blanketBaseline, context) + .Where(x => request.StrictDominance ? x.SizeDeltaBytes <= 0 : true) + .ToList(); + + if (options.Count == 0) + { + AnsiConsole.MarkupLine($"[grey]Smart fallback found no isolated group trades for[/] [cyan]{Markup.Escape(blanketBaseline.Names[0])}[/]."); + return Array.Empty(); + } + + var plans = BuildPlans(request, blanketBaseline, blanketAnchor, baseSize, baseKld, options, context); + if (plans.Count == 0) + { + AnsiConsole.MarkupLine($"[grey]Smart fallback found no size-safe plans for[/] [cyan]{Markup.Escape(blanketBaseline.Names[0])}[/] in window {Markup.Escape(request.WindowLabel)}."); + return Array.Empty(); + } + + var orderedPlans = request.StrictDominance + ? plans.OrderByDescending(x => x.TotalKldGain).ThenBy(x => x.PredictedSizeBytes).ThenByDescending(x => x.Score).ToList() + : plans.OrderByDescending(x => x.Score).ThenByDescending(x => x.TotalKldGain).ThenByDescending(x => x.PredictedSizeBytes).ToList(); + + int limit = Config.SelectionSmartFallbackAttemptsPerFailure; + var selected = orderedPlans + .Take(limit) + .Select((plan, index) => ToCandidate(request, blanketBaseline, plan, index + 1, options.Count, orderedPlans.Count, orderedPlans.Count(x => x.PredictedGainOverLine > 0d))) + .ToList(); + + AnsiConsole.MarkupLine( + $"[yellow]Smart baseline fallback staged:[/] [cyan]{selected.Count:N0}[/] candidate(s) for {Markup.Escape(request.WindowLabel)} from [cyan]{Markup.Escape(blanketBaseline.Names[0])}[/] blanket."); + + foreach (var candidate in selected) + { + var swaps = string.Join(", ", candidate.CandidateSelectionNotes.Where(x => x.StartsWith("swap ", StringComparison.Ordinal)).Take(4)); + AnsiConsole.MarkupLine($" [grey]- {Markup.Escape(HybridBenchmarkRepository.BuildDisplayName(candidate.Prediction.Quant))}[/] size={ToGiB(candidate.Prediction.PredictedSizeBytes):0.00}GiB additiveKLD={candidate.Prediction.PredictedKld:0.000000} {Markup.Escape(swaps)}"); + } + + return selected; + } + + private static bool TryResolveBaselineBlanket( + BenchmarkSnapshotRecord anchor, + out BaselineQuants baseline, + out string reason) + { + baseline = anchor.Quant.BaseQuant; + reason = string.Empty; + + if (BaselineQuants.IsNativeExactAlias(baseline.UniqueId)) + { + reason = $"anchor '{anchor.DisplayName}' uses native/exact precision and cannot be used as a learned baseline blanket."; + return false; + } + + if (TensorConfigIdentity.IsPureBaseline(anchor.Config)) + return true; + + foreach (var (group, storedValue) in TensorConfigIdentity.EnumerateGroupSlots(anchor.Config)) + { + if (Cache.UnusedTensorGroups.Any(x => x.UniqueId == group.UniqueId)) + continue; + + if (BaselineQuants.IsNullTensorConfigGroupSlot(storedValue)) + continue; + + var decoded = BaselineQuants.DecodeTensorConfigGroupSlotToBaselineId(storedValue); + if (decoded != baseline.UniqueId) + { + reason = $"anchor '{anchor.DisplayName}' is already a non-uniform hybrid; smart fallback only starts from pure/uniform baseline blankets."; + return false; + } + } + + return true; + } + + private static BenchmarkSnapshotRecord ResolveSmartBlanketAnchor( + BenchmarkSnapshotRecord anchor, + BaselineQuants blanketBaseline, + RankSafeKldPredictionService.RankSafePredictionModel context) + { + if (TensorConfigIdentity.IsPureBaseline(anchor.Config)) + { + if (!context.PureSnapshotsByBaselineId.TryGetValue(blanketBaseline.UniqueId, out _)) + { + throw new InvalidOperationException( + $"Smart baseline fallback critical truth error: pure benchmark snapshot for '{blanketBaseline.Names[0]}' (id {blanketBaseline.UniqueId}) was not loaded, " + + $"but strict/near/interior fallback is trying to tune from anchor '{anchor.DisplayName}'. " + + "This is not a soft skip; the original baseline anchor is missing from the prediction context."); + } + + /* + * Preserve the exact anchor that triggered the fallback. The dictionary check above is a + * consistency guard proving the pure baseline truth exists in the loaded context; using + * request.BaselineAnchor keeps size/KLD aligned with the active strict/near/interior frontier. + */ + return anchor; + } + + if (!IsUniformLearnedBlanket(anchor, blanketBaseline)) + { + throw new InvalidOperationException( + $"Smart baseline fallback critical truth error: anchor '{anchor.DisplayName}' resolved to blanket '{blanketBaseline.Names[0]}', " + + "but the anchor is not a pure baseline and not a uniform learned-candidate blanket. This should have been rejected before anchor resolution."); + } + + return anchor; + } + + private static bool IsUniformLearnedBlanket(BenchmarkSnapshotRecord anchor, BaselineQuants blanketBaseline) + { + if (TensorConfigIdentity.IsPureBaseline(anchor.Config)) + return true; + + foreach (var (group, storedValue) in TensorConfigIdentity.EnumerateGroupSlots(anchor.Config)) + { + if (Cache.UnusedTensorGroups.Any(x => x.UniqueId == group.UniqueId)) + continue; + + if (BaselineQuants.IsNullTensorConfigGroupSlot(storedValue)) + continue; + + var decoded = BaselineQuants.DecodeTensorConfigGroupSlotToBaselineId(storedValue); + if (decoded != blanketBaseline.UniqueId) + return false; + } + + return true; + } + + internal static bool IsEligibleForSmartFallbackTuning( + BaselineQuants blanketBaseline, + IReadOnlyCollection activeExplicitCandidates, + bool hasCompleteIsolationCoverage) + { + if (hasCompleteIsolationCoverage) + return true; + + return activeExplicitCandidates.Any(x => x.UniqueId == blanketBaseline.UniqueId); + } + + private static IReadOnlyList GetMissingIsolationGroups( + BaselineQuants blanketBaseline, + RankSafeKldPredictionService.RankSafePredictionModel context) + { + return context.ActiveGroups + .Where(group => !context.IsolationByGroupAndBaseline.ContainsKey((group.UniqueId, blanketBaseline.UniqueId))) + .Select(group => group.Name) + .ToList(); + } + + private static void ValidateBlanketIsolationCoverage( + BaselineQuants blanketBaseline, + RankSafeKldPredictionService.RankSafePredictionModel context, + IReadOnlyList missingGroups) + { + + if (missingGroups.Count == 0) + return; + + throw new InvalidOperationException( + $"Smart baseline fallback critical truth error: baseline '{blanketBaseline.Names[0]}' (id {blanketBaseline.UniqueId}) is present as a fallback anchor, " + + $"but isolated group truth is missing for {missingGroups.Count:N0}/{context.ActiveGroups.Count:N0} active group(s): {string.Join(", ", missingGroups)}. " + + "Smart fallback must not silently skip groups when tuning from a real baseline anchor."); + } + + private async Task GetContextAsync(CancellationToken ct) + { + if (_context != null) + return _context; + + var activeGroups = TReg.All + .Where(x => !Cache.UnusedTensorGroups.Any(u => u.UniqueId == x.UniqueId)) + .OrderBy(x => x.UniqueId) + .ToList(); + + if (activeGroups.Count == 0) + throw new InvalidOperationException("No active tensor groups were available for smart baseline fallback."); + + var notes = new List(); + var pureSnapshots = await _repository.LoadPureBaselineSnapshotsAsync(ct); + var pureByBaselineId = pureSnapshots + .GroupBy(x => x.Quant.BaseQuant.UniqueId) + .ToDictionary(g => g.Key, g => g.OrderBy(x => x.Kld).ThenBy(x => x.SizeBytes).First()); + + if (!pureByBaselineId.TryGetValue(BaselineQuants.Q8_0.UniqueId, out var pureQ8)) + throw new InvalidOperationException("Smart baseline fallback requires a pure Q8_0 benchmark snapshot."); + + var nativeExactScheme = TensorWeightScheme.GetCurrentNativePrecisionScheme(); + var q8BaseOnlyQuant = HybridQuant.CreateExactBlanket( + baseQuant: BaselineQuants.Q8_0, + groups: activeGroups, + exactScheme: nativeExactScheme); + + var q8BaseOnly = await _repository.LoadBenchmarkSnapshotAsync((TensorConfig)q8BaseOnlyQuant, ct) + ?? throw new InvalidOperationException("Smart baseline fallback requires the Q8_0 native-exact base-only anchor."); + + var baseOnlyByBaselineId = new Dictionary + { + [BaselineQuants.Q8_0.UniqueId] = q8BaseOnly + }; + + foreach (var baseline in BaselineQuants.GetAllRecognizedBaselines() + .Where(x => !BaselineQuants.IsNativeExactAlias(x.UniqueId)) + .OrderBy(x => x.UniqueId)) + { + if (baseOnlyByBaselineId.ContainsKey(baseline.UniqueId)) + continue; + + var directBaseOnlyQuant = HybridQuant.CreateExactBlanket( + baseQuant: baseline, + groups: activeGroups, + exactScheme: nativeExactScheme); + + var directBaseOnlySnapshot = await _repository.LoadBenchmarkSnapshotAsync((TensorConfig)directBaseOnlyQuant, ct); + if (directBaseOnlySnapshot != null) + baseOnlyByBaselineId[baseline.UniqueId] = directBaseOnlySnapshot; + } + + var isolationByGroupAndBaseline = new Dictionary<(byte GroupId, byte BaselineId), BenchmarkSnapshotRecord>(); + + foreach (var group in activeGroups) + { + foreach (var baseline in BaselineQuants.GetAllRecognizedBaselines() + .Where(x => !BaselineQuants.IsNativeExactAlias(x.UniqueId)) + .OrderBy(x => x.UniqueId)) + { + if (isolationByGroupAndBaseline.ContainsKey((group.UniqueId, baseline.UniqueId))) + continue; + + var isolationQuant = HybridQuant.CreateExactBlanket( + baseQuant: BaselineQuants.Q8_0, + groups: activeGroups, + exactScheme: nativeExactScheme); + + isolationQuant.SetLearnedCandidateOverride(group, baseline); + var snapshot = await _repository.LoadBenchmarkSnapshotAsync((TensorConfig)isolationQuant, ct); + if (snapshot != null) + isolationByGroupAndBaseline[(group.UniqueId, baseline.UniqueId)] = snapshot; + } + } + + notes.Add($"Smart fallback isolation context loaded: activeGroups={activeGroups.Count:N0}, baseOnlyAnchors={baseOnlyByBaselineId.Count:N0}, groupIsolations={isolationByGroupAndBaseline.Count:N0}."); + + _context = new RankSafeKldPredictionService.RankSafePredictionModel( + activeGroups: activeGroups, + pureQ8: pureQ8, + q8BaseOnly: q8BaseOnly, + pureSnapshotsByBaselineId: pureByBaselineId, + baseOnlySnapshotsByBaselineId: baseOnlyByBaselineId, + isolationByGroupAndBaseline: isolationByGroupAndBaseline, + isolationDominanceBitTruthByGroupAndBaseline: new Dictionary<(byte GroupId, byte BaselineId), double>(), + notes: notes); + + return _context; + } + + private static IReadOnlyList BuildGroupOptions( + BaselineQuants blanketBaseline, + RankSafeKldPredictionService.RankSafePredictionModel context) + { + var result = new List(); + + foreach (var group in context.ActiveGroups) + { + if (!context.IsolationByGroupAndBaseline.TryGetValue((group.UniqueId, blanketBaseline.UniqueId), out var baseIsolation)) + continue; + + long baseContributionBytes = (long)baseIsolation.SizeBytes - (long)context.Q8BaseOnly.SizeBytes; + + var allowed = RuntimeSearchSpace.GetAllowedRealExplicitCombinationCandidatesForGroup(group) + .Where(x => x.UniqueId != blanketBaseline.UniqueId) + .Where(x => !BaselineQuants.IsNativeExactAlias(x.UniqueId)) + .OrderBy(x => x.ExplicitCandidateSortOrder) + .ThenBy(x => x.UniqueId) + .ToList(); + + foreach (var candidate in allowed) + { + if (!context.IsolationByGroupAndBaseline.TryGetValue((group.UniqueId, candidate.UniqueId), out var candidateIsolation)) + continue; + + double kldGain = baseIsolation.Kld - candidateIsolation.Kld; + if (kldGain <= Config.SelectionMinimumKldImprovementEpsilon + KldEpsilon) + continue; + + long candidateContributionBytes = (long)candidateIsolation.SizeBytes - (long)context.Q8BaseOnly.SizeBytes; + long sizeDeltaBytes = candidateContributionBytes - baseContributionBytes; + + int fidelitySteps = CountHigherFidelitySteps(blanketBaseline, candidate); + if (fidelitySteps > Config.SelectionSmartFallbackMaxHigherFidelitySteps) + continue; + + bool isShrink = sizeDeltaBytes < 0; + bool isFreeOrBetter = sizeDeltaBytes <= 0; + + if (isShrink && candidateIsolation.Kld + Config.SelectionMinimumKldImprovementEpsilon >= baseIsolation.Kld) + continue; + + double damageAvoidedPerMiB = kldGain / Math.Max(1d, Math.Abs(sizeDeltaBytes) / 1024d / 1024d); + double sensitivityScore = kldGain * Math.Log2(2d + Math.Max(0d, baseIsolation.Kld) / Math.Max(candidateIsolation.Kld, 1e-12d)); + double score = sensitivityScore + damageAvoidedPerMiB; + if (isFreeOrBetter) + score += kldGain * 1000d; + + result.Add(new SmartGroupOption + { + Group = group, + CandidateBaseline = candidate, + BaseIsolation = baseIsolation, + CandidateIsolation = candidateIsolation, + SizeDeltaBytes = sizeDeltaBytes, + KldGain = kldGain, + Score = score, + HigherFidelitySteps = fidelitySteps + }); + } + } + + return result; + } + + private static List BuildPlans( + SmartFallbackRequest request, + BaselineQuants blanketBaseline, + BenchmarkSnapshotRecord blanketAnchor, + ulong baseSize, + double baseKld, + IReadOnlyList options, + RankSafeKldPredictionService.RankSafePredictionModel context) + { + var plans = new List(); + var seen = new HashSet(StringComparer.Ordinal); + + void TryAddPlan(IEnumerable selected, string strategy) + { + var chosen = selected + .GroupBy(x => x.Group.UniqueId) + .Select(g => g.OrderByDescending(x => x.Score).ThenBy(x => x.SizeDeltaBytes).First()) + .OrderBy(x => x.Group.UniqueId) + .ToList(); + + if (chosen.Count == 0) + return; + + var quant = HybridQuant.CreateLearnedCandidateBlanket( + baseQuant: blanketBaseline, + groups: context.ActiveGroups, + candidateBaseline: blanketBaseline); + + foreach (var option in chosen) + quant.SetLearnedCandidateOverride(option.Group, option.CandidateBaseline); + + var config = (TensorConfig)quant; + string key = TensorConfigIdentity.ToKey(config); + if (!seen.Add(key)) + return; + + if (!TryPredictSize(config, blanketBaseline, blanketAnchor, context, out var predictedSize, out _) || + !TryComputeAdditiveKld(config, blanketBaseline, blanketAnchor, context, out var predictedKld, out _)) + return; + + if (request.StrictDominance) + { + if (predictedSize > request.WindowMaxSizeBytes) + return; + } + else if (predictedSize < request.WindowMinSizeBytes || predictedSize > request.WindowMaxSizeBytes) + { + return; + } + + double expectedLine = request.StrictDominance + ? request.HigherDamageAnchor.Kld + : InterpolateKldLine(predictedSize, request.HigherDamageAnchor, request.LowerDamageAnchor); + double gainOverLine = expectedLine - predictedKld; + double totalKldGain = baseKld - predictedKld; + long totalSizeDelta = (long)predictedSize - (long)baseSize; + + if (request.StrictDominance && totalKldGain <= Config.SelectionMinimumKldImprovementEpsilon) + return; + + double score = chosen.Sum(x => x.Score) + + Math.Max(0d, totalKldGain) * 100d + + Math.Max(0d, gainOverLine) * 25d; + + if (!request.StrictDominance) + { + // Premium/interior fallback is allowed to gamble, but it should still + // prefer plans that protect the most isolated damage per byte spent. + double budgetUsedFraction = request.WindowMaxSizeBytes <= request.WindowMinSizeBytes + ? 0d + : ((double)predictedSize - request.WindowMinSizeBytes) / Math.Max(1d, request.WindowMaxSizeBytes - request.WindowMinSizeBytes); + score += Math.Clamp(budgetUsedFraction, 0d, 1d) * Math.Max(0d, totalKldGain) * 50d; + } + + plans.Add(new SmartCandidatePlan + { + Quant = quant, + Config = config, + Strategy = strategy, + Options = chosen, + PredictedSizeBytes = predictedSize, + PredictedAdditiveKld = predictedKld, + LinearExpectedKld = expectedLine, + PredictedGainOverLine = gainOverLine, + TotalKldGain = totalKldGain, + TotalSizeDeltaBytes = totalSizeDelta, + Score = score + }); + } + + var freeLunch = options + .Where(x => x.SizeDeltaBytes <= 0) + .GroupBy(x => x.Group.UniqueId) + .Select(g => g.OrderByDescending(x => x.KldGain).ThenBy(x => x.SizeDeltaBytes).First()) + .ToList(); + + TryAddPlan(freeLunch, "free-lunch-same-or-smaller"); + + foreach (var single in options.OrderByDescending(x => x.Score).ThenBy(x => x.SizeDeltaBytes).Take(Math.Max(12, Config.SelectionSmartFallbackAttemptsPerFailure * 4))) + TryAddPlan(new[] { single }, "single-sensitive-group"); + + if (!request.StrictDominance) + { + var protectedSet = new List(); + protectedSet.AddRange(freeLunch); + + foreach (var option in options + .Where(x => x.SizeDeltaBytes > 0) + .OrderByDescending(x => x.Score) + .ThenBy(x => x.SizeDeltaBytes)) + { + var trial = protectedSet + .Where(x => x.Group.UniqueId != option.Group.UniqueId) + .Concat(new[] { option }) + .ToList(); + + var trialQuant = HybridQuant.CreateLearnedCandidateBlanket( + baseQuant: blanketBaseline, + groups: context.ActiveGroups, + candidateBaseline: blanketBaseline); + foreach (var selected in trial) + trialQuant.SetLearnedCandidateOverride(selected.Group, selected.CandidateBaseline); + + if (!TryPredictSize((TensorConfig)trialQuant, blanketBaseline, blanketAnchor, context, out var trialSize, out _)) + continue; + + if (trialSize <= request.WindowMaxSizeBytes) + protectedSet = trial; + } + + TryAddPlan(protectedSet, "balanced-brain-protection"); + + foreach (var groupedBySensitivity in options + .Where(x => x.SizeDeltaBytes >= 0) + .OrderByDescending(x => x.KldGain) + .ThenBy(x => x.SizeDeltaBytes) + .Take(Math.Max(8, Config.SelectionSmartFallbackAttemptsPerFailure * 3))) + { + var blend = freeLunch + .Where(x => x.Group.UniqueId != groupedBySensitivity.Group.UniqueId) + .Concat(new[] { groupedBySensitivity }); + TryAddPlan(blend, "sensitivity-first-blend"); + } + } + + return plans; + } + + private static HybridSelectionCandidate ToCandidate( + SmartFallbackRequest request, + BaselineQuants blanketBaseline, + SmartCandidatePlan plan, + int attemptOrder, + int optionCount, + int planCount, + int lineBeatingCount) + { + var notes = new List + { + "smartFallback=sqlite-isolation-truth; not selected from DuckDB prediction rows", + $"blanket={blanketBaseline.Names[0]}", + $"strategy={plan.Strategy}", + $"exactSizeWindow={ToGiB(request.WindowMinSizeBytes):0.00}..{ToGiB(request.WindowMaxSizeBytes):0.00}GiB", + $"smartAttemptsLimit={Config.SelectionSmartFallbackAttemptsPerFailure}", + $"maxHigherFidelitySteps={Config.SelectionSmartFallbackMaxHigherFidelitySteps}" + }; + + notes.AddRange(plan.Options.Select(option => + $"swap {option.Group.Name}: {blanketBaseline.Names[0]} -> {option.CandidateBaseline.Names[0]} " + + $"sizeDelta={option.SizeDeltaBytes:N0}B isolatedKldGain={option.KldGain:0.000000}")); + + var row = new RankSafePredictionRow + { + Config = plan.Config, + Quant = plan.Quant, + PredictedSizeBytes = plan.PredictedSizeBytes, + IsSizePredictable = true, + AdditiveKld = plan.PredictedAdditiveKld, + InteractionKld = plan.PredictedAdditiveKld, + PredictedKld = plan.PredictedAdditiveKld, + PredictionConfidence = 0.50d, + PredictedPpl = 0d, + CrossTerm = 0d, + IsPureBaseline = false, + IsPredictable = true, + HasUnknownMappings = false, + EffectiveStateKey = $"smart-fallback:{TensorConfigIdentity.ToKey(plan.Config)}", + PredictedRank = null, + Notes = notes + }; + + return new HybridSelectionCandidate + { + Prediction = row, + Reason = request.Reason, + LowerDamageAnchor = request.LowerDamageAnchor, + HigherDamageAnchor = request.HigherDamageAnchor, + WindowMinSizeBytes = request.WindowMinSizeBytes, + WindowMaxSizeBytes = request.WindowMaxSizeBytes, + PredictionWindowMinSizeBytes = request.WindowMinSizeBytes, + PredictionWindowMaxSizeBytes = request.WindowMaxSizeBytes, + LinearExpectedKld = plan.LinearExpectedKld, + PredictedGainOverLine = plan.PredictedGainOverLine, + AttemptOrder = attemptOrder, + WindowLabel = request.WindowLabel, + CandidatePoolSize = optionCount, + WindowCandidateCount = planCount, + LineBeatingCandidateCount = lineBeatingCount, + FetchedCandidateCount = planCount, + CandidatesAfterBrutalityCount = planCount, + CandidateAttemptLimit = Config.SelectionSmartFallbackAttemptsPerFailure, + PhaseWindowIndex = request.PhaseWindowIndex, + PhaseWindowCount = request.PhaseWindowCount, + RawSelectionRank = attemptOrder, + CandidateTheoryFamilyKey = "smart-baseline-tuning", + CandidateTheoryFamilyRank = 1, + CandidateTheoryFamilyMemberRank = attemptOrder, + CandidateTheoryFamilyDisplay = "smart baseline tuning", + DiversityMode = "sqlite-isolation-fallback", + CandidateSelectionNotes = notes + }; + } + + private static bool TryPredictSize( + TensorConfig config, + BaselineQuants blanketBaseline, + BenchmarkSnapshotRecord blanketAnchor, + RankSafeKldPredictionService.RankSafePredictionModel context, + out ulong sizeBytes, + out IReadOnlyList notes) + { + var localNotes = new List(); + notes = localNotes; + sizeBytes = 0; + + long total = (long)blanketAnchor.SizeBytes; + + foreach (var (group, effectiveBaselineId) in RankSafeKldPredictionService.EnumerateEffectiveBaselines(config, context.ActiveGroups)) + { + if (effectiveBaselineId == blanketBaseline.UniqueId) + continue; + + if (!TryResolveIsolationSnapshot(group, blanketBaseline.UniqueId, context, localNotes, "baseline size", out var baseSnapshot) || + !TryResolveIsolationSnapshot(group, effectiveBaselineId, context, localNotes, "candidate size", out var candidateSnapshot)) + { + return false; + } + + total += (long)candidateSnapshot.SizeBytes - (long)baseSnapshot.SizeBytes; + } + + if (total <= 0) + { + localNotes.Add($"Predicted size collapsed to {total:N0} bytes."); + return false; + } + + sizeBytes = (ulong)total; + return true; + } + + private static bool TryComputeAdditiveKld( + TensorConfig config, + BaselineQuants blanketBaseline, + BenchmarkSnapshotRecord blanketAnchor, + RankSafeKldPredictionService.RankSafePredictionModel context, + out double additiveKld, + out IReadOnlyList notes) + { + var localNotes = new List(); + notes = localNotes; + additiveKld = Math.Max(0d, blanketAnchor.Kld); + + foreach (var (group, effectiveBaselineId) in RankSafeKldPredictionService.EnumerateEffectiveBaselines(config, context.ActiveGroups)) + { + if (effectiveBaselineId == blanketBaseline.UniqueId) + continue; + + if (!TryResolveIsolationSnapshot(group, blanketBaseline.UniqueId, context, localNotes, "baseline KLD", out var baseSnapshot) || + !TryResolveIsolationSnapshot(group, effectiveBaselineId, context, localNotes, "candidate KLD", out var candidateSnapshot)) + { + return false; + } + + additiveKld += Math.Max(0d, candidateSnapshot.Kld) - Math.Max(0d, baseSnapshot.Kld); + } + + additiveKld = Math.Max(0d, additiveKld); + return true; + } + + private static bool TryResolveIsolationSnapshot( + TensorGroup group, + byte baselineId, + RankSafeKldPredictionService.RankSafePredictionModel context, + List notes, + string role, + out BenchmarkSnapshotRecord snapshot) + { + if (BaselineQuants.IsNativeExactAlias(baselineId)) + { + snapshot = context.Q8BaseOnly; + return true; + } + + if (context.IsolationByGroupAndBaseline.TryGetValue((group.UniqueId, baselineId), out var foundSnapshot)) + { + snapshot = foundSnapshot; + return true; + } + + var name = BaselineQuants.FromId(baselineId).Names[0]; + var message = $"Missing {role} isolation anchor for group '{group.Name}' and baseline '{name}' (id {baselineId})."; + notes.Add(message); + + if (BaselineQuants.FromId(baselineId).IsExternalRepositoryBaseline) + { + throw new InvalidOperationException( + $"Smart baseline fallback critical truth error: {message} " + + "External/custom fallback must use exact isolated truth and must not silently collapse or skip."); + } + + snapshot = default!; + return false; + } + + private static int CountHigherFidelitySteps(BaselineQuants baseBaseline, BaselineQuants candidate) + { + if (candidate.BitRange <= baseBaseline.BitRange) + return 0; + + var ladder = BaselineQuants.GetAllRecognizedBaselines() + .Where(x => !BaselineQuants.IsNativeExactAlias(x.UniqueId)) + .GroupBy(x => x.BitRange) + .Select(g => g.Key) + .OrderBy(x => x) + .ToList(); + + int baseIndex = ladder.IndexOf(baseBaseline.BitRange); + int candidateIndex = ladder.IndexOf(candidate.BitRange); + if (baseIndex < 0 || candidateIndex < 0) + return candidate.BitRange > baseBaseline.BitRange ? 1 : 0; + + return Math.Max(0, candidateIndex - baseIndex); + } + + private static double InterpolateKldLine( + ulong sizeBytes, + BenchmarkSnapshotRecord higherDamageSmaller, + BenchmarkSnapshotRecord lowerDamageLarger) + { + if (lowerDamageLarger.SizeBytes <= higherDamageSmaller.SizeBytes) + return Math.Min(higherDamageSmaller.Kld, lowerDamageLarger.Kld); + + double t = ((double)sizeBytes - higherDamageSmaller.SizeBytes) / + (lowerDamageLarger.SizeBytes - higherDamageSmaller.SizeBytes); + t = Math.Clamp(t, 0d, 1d); + return higherDamageSmaller.Kld + (lowerDamageLarger.Kld - higherDamageSmaller.Kld) * t; + } + + private static double ToGiB(ulong bytes) => bytes / 1024d / 1024d / 1024d; + + private sealed class SmartFallbackRequest + { + public HybridSelectionReason Reason { get; init; } + public string WindowLabel { get; init; } = string.Empty; + public BenchmarkSnapshotRecord BaselineAnchor { get; init; } = default!; + public BenchmarkSnapshotRecord HigherDamageAnchor { get; init; } = default!; + public BenchmarkSnapshotRecord LowerDamageAnchor { get; init; } = default!; + public ulong WindowMinSizeBytes { get; init; } + public ulong WindowMaxSizeBytes { get; init; } + public bool StrictDominance { get; init; } + public int PhaseWindowIndex { get; init; } + public int PhaseWindowCount { get; init; } + } + + private sealed class SmartGroupOption + { + public TensorGroup Group { get; init; } = default!; + public BaselineQuants CandidateBaseline { get; init; } = default!; + public BenchmarkSnapshotRecord BaseIsolation { get; init; } = default!; + public BenchmarkSnapshotRecord CandidateIsolation { get; init; } = default!; + public long SizeDeltaBytes { get; init; } + public double KldGain { get; init; } + public double Score { get; init; } + public int HigherFidelitySteps { get; init; } + } + + private sealed class SmartCandidatePlan + { + public HybridQuant Quant { get; init; } = default!; + public TensorConfig Config { get; init; } + public string Strategy { get; init; } = string.Empty; + public IReadOnlyList Options { get; init; } = Array.Empty(); + public ulong PredictedSizeBytes { get; init; } + public double PredictedAdditiveKld { get; init; } + public double LinearExpectedKld { get; init; } + public double PredictedGainOverLine { get; init; } + public double TotalKldGain { get; init; } + public long TotalSizeDeltaBytes { get; init; } + public double Score { get; init; } + } +} diff --git a/src/MagicQuant/Services/TargetedRelearnService.cs b/src/MagicQuant/Services/TargetedRelearnService.cs new file mode 100644 index 0000000..a1231a0 --- /dev/null +++ b/src/MagicQuant/Services/TargetedRelearnService.cs @@ -0,0 +1,222 @@ +using MagicQuant.Configuration; +using Microsoft.EntityFrameworkCore; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class TargetedRelearnService +{ + public async Task PlanConfirmAndExecuteAsync(IReadOnlyCollection resolvedCustomBaselines, CancellationToken ct = default) + { + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int tensorGroupProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + + await using var db = new MagicQuantContext(); + var plan = new TargetedRelearnPlan(architectureFamilyId, tensorGroupProfileId); + + if (Config.Current.Learning.ForceRelearnArchitectureFamily) + { + plan.ArchitectureFamilyWide = true; + } + + foreach (var raw in Config.Current.Learning.ForceRelearnStandardBaselines ?? []) + { + if (string.IsNullOrWhiteSpace(raw)) + continue; + + var baseline = BaselineQuants.ResolveBuiltInStandardBaseline(raw.Trim()) + ?? throw new InvalidOperationException($"Unknown standard baseline '{raw}' in learning.force_relearn_standard_baselines."); + + var definition = await db.BaselineQuantDefinitions.AsNoTracking().FirstOrDefaultAsync(x => + x.ArchitectureFamilyId == null && x.RuntimeBaselineId == baseline.UniqueId, ct); + + if (definition == null) + throw new InvalidOperationException($"SQLite baseline definition was not found for standard baseline '{raw}'."); + + plan.BaselineDefinitionIds.Add(definition.Id); + plan.TargetDescriptions.Add($"standard:{definition.DisplayName}"); + } + + foreach (var spec in resolvedCustomBaselines.Where(x => x.ForceRelearn)) + { + var definition = await db.BaselineQuantDefinitions.AsNoTracking().FirstOrDefaultAsync(x => + x.ArchitectureFamilyId == architectureFamilyId && + x.RuntimeBaselineId == spec.DynamicBaselineId, ct); + + if (definition == null) + throw new InvalidOperationException($"Custom include requested force_relearn, but no DB definition was found for '{spec.RepoId}/{spec.SourceFileName}'."); + + plan.BaselineDefinitionIds.Add(definition.Id); + plan.TargetDescriptions.Add($"custom:{definition.SourceRepository}/{definition.SourceFileName}"); + } + + if (!plan.HasTargets) + return; + + await PopulateCountsAsync(db, plan, ct); + PrintPlan(plan); + + if (!AnsiConsole.Confirm("Apply this targeted destructive relearn plan?", defaultValue: false)) + { + throw new OperationCanceledException("Targeted relearn was declined by the user. Aborting before any destructive changes."); + } + + await ExecuteAsync(db, plan, ct); + AnsiConsole.MarkupLine("[green]Targeted relearn cleanup complete.[/] The affected truth will be regenerated by this run."); + } + + private static async Task PopulateCountsAsync(MagicQuantContext db, TargetedRelearnPlan plan, CancellationToken ct) + { + IQueryable benchmarkQuery = BuildAffectedBenchmarkQuery(db, plan); + var benchmarkIds = await benchmarkQuery.Select(x => x.Id).Distinct().ToListAsync(ct); + + plan.AiBenchmarkRows = benchmarkIds.Count; + plan.CategoryBenchmarkRows = await db.Set().CountAsync(x => benchmarkIds.Contains(x.AiBenchmarkId), ct); + plan.BenchmarkRunRows = await db.BenchmarkRuns.CountAsync(x => benchmarkIds.Contains(x.AiBenchmarkId), ct); + plan.QuantizationRunRowsToDetach = await db.QuantizationRuns.CountAsync(x => x.AiBenchmarkId != null && benchmarkIds.Contains(x.AiBenchmarkId.Value), ct); + plan.AiBenchmarkLearnedSourceRows = await db.AiBenchmarkLearnedSources.CountAsync(x => benchmarkIds.Contains(x.AiBenchmarkId), ct); + + if (plan.ArchitectureFamilyWide) + { + plan.LearnedBaselineTensorQuantRows = await db.LearnedBaselineTensorQuants + .CountAsync(x => x.ArchitectureFamilyId == plan.ArchitectureFamilyId, ct); + plan.ExecutionPlanProbeCacheRows = await db.ExecutionPlanProbeCaches + .CountAsync(x => x.ArchitectureFamilyId == plan.ArchitectureFamilyId, ct); + } + else + { + plan.LearnedBaselineTensorQuantRows = await db.LearnedBaselineTensorQuants + .CountAsync(x => x.ArchitectureFamilyId == plan.ArchitectureFamilyId && + x.TensorGroupProfileId == plan.TensorGroupProfileId && + plan.BaselineDefinitionIds.Contains(x.BaselineQuantDefinitionId), ct); + plan.ExecutionPlanProbeCacheRows = await db.ExecutionPlanProbeCaches + .CountAsync(x => x.ArchitectureFamilyId == plan.ArchitectureFamilyId && + x.TensorGroupProfileId == plan.TensorGroupProfileId, ct); + } + } + + private static void PrintPlan(TargetedRelearnPlan plan) + { + AnsiConsole.Write(new Rule("[red]Targeted Relearn Deletion Plan[/]") { Justification = Justify.Left }); + if (plan.ArchitectureFamilyWide) + AnsiConsole.MarkupLine("[yellow]Scope:[/] active architecture family, all tensor group profiles"); + else + AnsiConsole.MarkupLine($"[yellow]Scope:[/] active architecture family + tensor group profile id [cyan]{plan.TensorGroupProfileId}[/]"); + + foreach (var target in plan.TargetDescriptions.Distinct(StringComparer.OrdinalIgnoreCase).OrderBy(x => x)) + AnsiConsole.MarkupLine($" [grey]- {Markup.Escape(target)}[/]"); + + AnsiConsole.MarkupLine($"[red]LearnedBaselineTensorQuant rows:[/] {plan.LearnedBaselineTensorQuantRows:N0}"); + AnsiConsole.MarkupLine($"[red]AiBenchmark rows:[/] {plan.AiBenchmarkRows:N0}"); + AnsiConsole.MarkupLine($"[red]CategoryBenchmark rows:[/] {plan.CategoryBenchmarkRows:N0}"); + AnsiConsole.MarkupLine($"[red]BenchmarkRun rows:[/] {plan.BenchmarkRunRows:N0}"); + AnsiConsole.MarkupLine($"[yellow]QuantizationRun rows to detach:[/] {plan.QuantizationRunRowsToDetach:N0}"); + AnsiConsole.MarkupLine($"[red]AiBenchmarkLearnedSource rows:[/] {plan.AiBenchmarkLearnedSourceRows:N0}"); + AnsiConsole.MarkupLine($"[red]ExecutionPlanProbeCache rows:[/] {plan.ExecutionPlanProbeCacheRows:N0}"); + } + + private static async Task ExecuteAsync(MagicQuantContext db, TargetedRelearnPlan plan, CancellationToken ct) + { + await using var transaction = await db.Database.BeginTransactionAsync(ct); + + IQueryable benchmarkQuery = BuildAffectedBenchmarkQuery(db, plan); + var benchmarkIds = await benchmarkQuery.Select(x => x.Id).Distinct().ToListAsync(ct); + + if (benchmarkIds.Count > 0) + { + await db.QuantizationRuns + .Where(x => x.AiBenchmarkId != null && benchmarkIds.Contains(x.AiBenchmarkId.Value)) + .ExecuteUpdateAsync(x => x.SetProperty(r => r.AiBenchmarkId, (Guid?)null), ct); + + await db.AiBenchmarks + .Where(x => benchmarkIds.Contains(x.Id)) + .ExecuteDeleteAsync(ct); + } + + if (plan.ArchitectureFamilyWide) + { + await db.LearnedBaselineTensorQuants + .Where(x => x.ArchitectureFamilyId == plan.ArchitectureFamilyId) + .ExecuteDeleteAsync(ct); + + await db.ExecutionPlanProbeCaches + .Where(x => x.ArchitectureFamilyId == plan.ArchitectureFamilyId) + .ExecuteDeleteAsync(ct); + } + else + { + await db.LearnedBaselineTensorQuants + .Where(x => x.ArchitectureFamilyId == plan.ArchitectureFamilyId && + x.TensorGroupProfileId == plan.TensorGroupProfileId && + plan.BaselineDefinitionIds.Contains(x.BaselineQuantDefinitionId)) + .ExecuteDeleteAsync(ct); + + await db.ExecutionPlanProbeCaches + .Where(x => x.ArchitectureFamilyId == plan.ArchitectureFamilyId && + x.TensorGroupProfileId == plan.TensorGroupProfileId) + .ExecuteDeleteAsync(ct); + } + + await transaction.CommitAsync(ct); + } + + private static IQueryable BuildAffectedBenchmarkQuery(MagicQuantContext db, TargetedRelearnPlan plan) + { + if (plan.ArchitectureFamilyWide) + return db.AiBenchmarks.Where(x => x.ArchitectureFamilyId == plan.ArchitectureFamilyId); + + var pureBaselineRuntimeIds = db.BaselineQuantDefinitions + .Where(x => plan.BaselineDefinitionIds.Contains(x.Id)) + .Select(x => x.RuntimeBaselineId) + .ToList(); + + var pureBaselineBenchmarks = db.AiBenchmarks + .Include(x => x.TensorCombo) + .Where(x => x.ArchitectureFamilyId == plan.ArchitectureFamilyId && + x.TensorGroupProfileId == plan.TensorGroupProfileId && + pureBaselineRuntimeIds.Contains(x.TensorCombo.BaseQuant) && + x.TensorCombo.Embeddings == 0 && x.TensorCombo.LmHead == 0 && + x.TensorCombo.AttnQ == 0 && x.TensorCombo.AttnKV == 0 && + x.TensorCombo.AttnOutput == 0 && x.TensorCombo.FfnUpGate == 0 && + x.TensorCombo.FfnDown == 0 && x.TensorCombo.MoeExperts == 0 && + x.TensorCombo.MoeRouter == 0); + + var dependentHybridBenchmarkIds = db.AiBenchmarkLearnedSources + .Where(x => x.ArchitectureFamilyId == plan.ArchitectureFamilyId && + x.TensorGroupProfileId == plan.TensorGroupProfileId && + plan.BaselineDefinitionIds.Contains(x.BaselineQuantDefinitionId)) + .Select(x => x.AiBenchmarkId); + + var dependentHybridBenchmarks = db.AiBenchmarks + .Where(x => dependentHybridBenchmarkIds.Contains(x.Id)); + + return pureBaselineBenchmarks.Concat(dependentHybridBenchmarks); + } + + private sealed class TargetedRelearnPlan + { + public TargetedRelearnPlan(int architectureFamilyId, int tensorGroupProfileId) + { + ArchitectureFamilyId = architectureFamilyId; + TensorGroupProfileId = tensorGroupProfileId; + } + + public int ArchitectureFamilyId { get; } + public int TensorGroupProfileId { get; } + public bool ArchitectureFamilyWide { get; set; } + public HashSet BaselineDefinitionIds { get; } = new(); + public List TargetDescriptions { get; } = new(); + public bool HasTargets => ArchitectureFamilyWide || BaselineDefinitionIds.Count > 0; + public int LearnedBaselineTensorQuantRows { get; set; } + public int AiBenchmarkRows { get; set; } + public int CategoryBenchmarkRows { get; set; } + public int BenchmarkRunRows { get; set; } + public int QuantizationRunRowsToDetach { get; set; } + public int AiBenchmarkLearnedSourceRows { get; set; } + public int ExecutionPlanProbeCacheRows { get; set; } + } +} diff --git a/src/MagicQuant/Services/TensorGroupProfileService.cs b/src/MagicQuant/Services/TensorGroupProfileService.cs new file mode 100644 index 0000000..dc75414 --- /dev/null +++ b/src/MagicQuant/Services/TensorGroupProfileService.cs @@ -0,0 +1,104 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.Encodings.Web; +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class TensorGroupProfileService +{ + private static readonly JsonSerializerOptions JsonOptions = new() + { + WriteIndented = false, + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + + public async Task EnsureCurrentProfileAsync(CancellationToken ct = default) + { + int architectureFamilyId = Cache.CurrentArchitectureFamilyId + ?? throw new InvalidOperationException("Architecture family must be resolved before resolving tensor group profile."); + + string snapshotJson = BuildSnapshotJson(); + string fingerprint = ComputeSnapshotHash(snapshotJson); + + await using var db = new MagicQuantContext(); + + var existing = await db.TensorGroupProfiles + .FirstOrDefaultAsync(x => x.ArchitectureFamilyId == architectureFamilyId && x.FingerprintHash == fingerprint, ct); + + if (existing == null) + { + existing = new TensorGroupProfile + { + ArchitectureFamilyId = architectureFamilyId, + FingerprintHash = fingerprint, + SnapshotJson = snapshotJson, + CreatedUtc = DateTime.UtcNow, + IsActive = true + }; + db.TensorGroupProfiles.Add(existing); + } + + var activeProfiles = await db.TensorGroupProfiles + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && x.Id != existing.Id && x.IsActive) + .ToListAsync(ct); + + foreach (var profile in activeProfiles) + profile.IsActive = false; + + existing.IsActive = true; + await db.SaveChangesAsync(ct); + + Cache.CurrentTensorGroupProfileId = existing.Id; + Cache.CurrentTensorGroupProfileFingerprintHash = existing.FingerprintHash; + + AnsiConsole.MarkupLine($"[green]Tensor group profile active:[/] id=[cyan]{existing.Id}[/] hash=[grey]{Markup.Escape(existing.FingerprintHash[..Math.Min(12, existing.FingerprintHash.Length)])}[/]"); + return existing; + } + + public static int RequireCurrentProfileId() => + Cache.CurrentTensorGroupProfileId + ?? throw new InvalidOperationException("Current tensor group profile is not set. Call TensorGroupProfileService.EnsureCurrentProfileAsync after architecture-family resolution."); + + public static int RequireCurrentArchitectureFamilyId() => + Cache.CurrentArchitectureFamilyId + ?? throw new InvalidOperationException("Current architecture family is not set."); + + public static string BuildSnapshotJson() + { + var snapshot = new + { + schema = 1, + groups = TReg.All + .OrderBy(x => x.UniqueId) + .Select(x => new + { + id = x.UniqueId, + name = x.Name, + patterns = x.Tensors + .Where(p => !string.IsNullOrWhiteSpace(p)) + .Select(p => p.Trim()) + .ToArray() + }) + .ToArray(), + baseQuantExceptions = TReg.GetBaseQuantExceptionPatterns() + .Where(p => !string.IsNullOrWhiteSpace(p)) + .Select(p => p.Trim()) + .ToArray() + }; + + return JsonSerializer.Serialize(snapshot, JsonOptions); + } + + public static string ComputeSnapshotHash(string snapshotJson) + { + using var sha = SHA256.Create(); + return Convert.ToHexString(sha.ComputeHash(Encoding.UTF8.GetBytes(snapshotJson))).ToLowerInvariant(); + } +} diff --git a/src/MagicQuant/Services/TensorGroupRebucketService.cs b/src/MagicQuant/Services/TensorGroupRebucketService.cs new file mode 100644 index 0000000..90e58ee --- /dev/null +++ b/src/MagicQuant/Services/TensorGroupRebucketService.cs @@ -0,0 +1,319 @@ +using MagicQuant.Models.Learning; +using MagicQuant.Services.Learning; +using Microsoft.EntityFrameworkCore; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class TensorGroupRebucketService +{ + private const byte UnknownTensorGroupId = 255; + private readonly TensorGroupingAuditService _auditService = new(); + + public async Task RebucketFromExistingProfileTruthAsync(CancellationToken ct = default) + { + int architectureFamilyId = TensorGroupProfileService.RequireCurrentArchitectureFamilyId(); + int currentProfileId = TensorGroupProfileService.RequireCurrentProfileId(); + + await using var db = new MagicQuantContext(); + + var sourceProfiles = await db.TensorGroupProfiles + .AsNoTracking() + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && x.Id != currentProfileId) + .OrderByDescending(x => x.CreatedUtc) + .ThenByDescending(x => x.Id) + .Select(x => new { x.Id, x.FingerprintHash, x.CreatedUtc }) + .ToListAsync(ct); + + if (sourceProfiles.Count == 0) + { + AnsiConsole.MarkupLine("[grey]Tensor-group rebucket requested, but no previous tensor group profiles exist for this architecture family.[/]"); + return TensorGroupRebucketSummary.Empty; + } + + uint scopedModelHashId = await ArchitectureFamilyService.ResolveScopedAiModelHashIdAsync(db, ct); + int? currentImatrixDefinitionId = await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync( + db, + scopedModelHashId, + createIfMissing: false, + ct); + + var candidateKeys = await db.LearnedBaselineTensorQuants + .AsNoTracking() + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && x.TensorGroupProfileId != currentProfileId) + .GroupBy(x => new { x.BaselineQuantDefinitionId, x.TensorWeightSchemeId }) + .Select(g => new + { + g.Key.BaselineQuantDefinitionId, + g.Key.TensorWeightSchemeId, + RowCount = g.Count(), + LatestProfileId = g.Max(x => x.TensorGroupProfileId) + }) + .ToListAsync(ct); + + if (candidateKeys.Count == 0) + { + AnsiConsole.MarkupLine("[grey]Tensor-group rebucket requested, but no previous learned tensor truth exists for this architecture family.[/]"); + return TensorGroupRebucketSummary.Empty; + } + + int copiedBaselines = 0; + int copiedRows = 0; + int clonedBenchmarks = 0; + int skippedExisting = 0; + int fatalSkipped = 0; + + AnsiConsole.Write(new Rule("[yellow]Tensor Group Rebucket From Existing Truth[/]") { Justification = Justify.Left }); + AnsiConsole.MarkupLine("[grey]Rebuilding learned tensor/group rows for the active regex profile from prior DB truth. Pure baseline benchmark rows are cloned when available; group-override isolation truth is intentionally not cloned.[/]"); + + foreach (var key in candidateKeys + .OrderBy(x => x.BaselineQuantDefinitionId) + .ThenBy(x => x.TensorWeightSchemeId)) + { + bool alreadyExists = await db.LearnedBaselineTensorQuants + .AsNoTracking() + .AnyAsync(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == currentProfileId && + x.BaselineQuantDefinitionId == key.BaselineQuantDefinitionId && + x.TensorWeightSchemeId == key.TensorWeightSchemeId, ct); + + if (alreadyExists) + { + skippedExisting++; + continue; + } + + var sourceProfileId = await PickBestSourceProfileForKeyAsync( + db, + architectureFamilyId, + currentProfileId, + key.BaselineQuantDefinitionId, + key.TensorWeightSchemeId, + sourceProfiles.Select(x => x.Id).ToList(), + ct); + + if (!sourceProfileId.HasValue) + continue; + + var sourceRows = await db.LearnedBaselineTensorQuants + .AsNoTracking() + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == sourceProfileId.Value && + x.BaselineQuantDefinitionId == key.BaselineQuantDefinitionId && + x.TensorWeightSchemeId == key.TensorWeightSchemeId) + .OrderBy(x => x.TensorName) + .ToListAsync(ct); + + if (sourceRows.Count == 0) + continue; + + var truth = sourceRows.ToDictionary( + x => x.TensorName, + x => new LearnedTensorTruth(x.TensorName, x.FinalQuantType, LearningSource.GgufOnly), + StringComparer.Ordinal); + + var audit = _auditService.Audit(truth.Keys.ToList(), truth); + if (audit.HasFatalIssues) + { + fatalSkipped++; + AnsiConsole.MarkupLine( + $"[red]Skipped rebucket for BaselineDefinitionId={key.BaselineQuantDefinitionId}, scheme={key.TensorWeightSchemeId}:[/] ambiguous={audit.Ambiguous.Count}, unresolved={audit.IllegalUnresolved.Count}. Fix tensor_groups.yaml first."); + continue; + } + + var targetBenchmarkId = await EnsurePureBaselineBenchmarkCloneAsync( + db, + sourceRows, + architectureFamilyId, + currentProfileId, + scopedModelHashId, + currentImatrixDefinitionId, + ct); + + if (targetBenchmarkId.Cloned) + clonedBenchmarks++; + + await db.LearnedBaselineTensorQuants + .Where(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == currentProfileId && + x.BaselineQuantDefinitionId == key.BaselineQuantDefinitionId && + x.TensorWeightSchemeId == key.TensorWeightSchemeId) + .ExecuteDeleteAsync(ct); + + var targetRows = sourceRows.Select(row => new LearnedBaselineTensorQuant + { + Id = Guid.NewGuid(), + ArchitectureFamilyId = architectureFamilyId, + TensorGroupProfileId = currentProfileId, + BaselineQuantDefinitionId = row.BaselineQuantDefinitionId, + TensorComboId = row.TensorComboId, + AiBenchmarkId = targetBenchmarkId.BenchmarkId, + AiModelHashId = scopedModelHashId, + BaselineQuantId = row.BaselineQuantId, + BaselineCanonicalKey = row.BaselineCanonicalKey, + BaselineSourceKind = row.BaselineSourceKind, + BaselineSourceRepository = row.BaselineSourceRepository, + BaselineSourceFileName = row.BaselineSourceFileName, + TensorWeightSchemeId = row.TensorWeightSchemeId, + TensorGroupId = ResolveRebucketedTensorGroupId(row.TensorName, audit), + TensorName = row.TensorName, + FinalQuantType = row.FinalQuantType + }).ToList(); + + db.LearnedBaselineTensorQuants.AddRange(targetRows); + await db.SaveChangesAsync(ct); + + copiedBaselines++; + copiedRows += targetRows.Count; + + string baselineName = await db.BaselineQuantDefinitions + .AsNoTracking() + .Where(x => x.Id == key.BaselineQuantDefinitionId) + .Select(x => x.DisplayName) + .FirstOrDefaultAsync(ct) ?? key.BaselineQuantDefinitionId.ToString(); + + AnsiConsole.MarkupLine( + $"[green]Rebucketed learned truth:[/] {Markup.Escape(baselineName)} scheme={key.TensorWeightSchemeId} tensors={targetRows.Count:N0} fromProfile={sourceProfileId.Value} -> currentProfile={currentProfileId}"); + } + + var summary = new TensorGroupRebucketSummary + { + BaselineSchemeSetsCopied = copiedBaselines, + LearnedRowsCopied = copiedRows, + PureBenchmarkRowsCloned = clonedBenchmarks, + ExistingCurrentProfileSetsSkipped = skippedExisting, + FatalSetsSkipped = fatalSkipped + }; + + AnsiConsole.MarkupLine( + $"[green]Tensor-group rebucket summary:[/] baseline/scheme sets={summary.BaselineSchemeSetsCopied:N0}, rows={summary.LearnedRowsCopied:N0}, pure benchmarks cloned={summary.PureBenchmarkRowsCloned:N0}, already-current skipped={summary.ExistingCurrentProfileSetsSkipped:N0}, fatal skipped={summary.FatalSetsSkipped:N0}"); + + return summary; + } + + private static byte ResolveRebucketedTensorGroupId( + string tensorName, + TensorGroupingAuditResult audit) + { + if (!audit.GroupedByTensor.TryGetValue(tensorName, out var grouped)) + { + throw new InvalidOperationException( + $"Cannot rebucket tensor '{tensorName}' because it was not present in the active grouping audit."); + } + + if (grouped.PrimaryGroup != null) + return grouped.PrimaryGroup.UniqueId; + + if (grouped.IsBaseQuantException) + return UnknownTensorGroupId; + + throw new InvalidOperationException( + $"Cannot rebucket tensor '{tensorName}' because it is unresolved under the active tensor_groups.yaml profile. " + + "BaseQuant fallback is only allowed when the tensor matches base_quant_exceptions."); + } + + private static async Task PickBestSourceProfileForKeyAsync( + MagicQuantContext db, + int architectureFamilyId, + int currentProfileId, + int baselineDefinitionId, + byte tensorWeightSchemeId, + IReadOnlyList preferredProfileOrder, + CancellationToken ct) + { + foreach (var profileId in preferredProfileOrder) + { + bool exists = await db.LearnedBaselineTensorQuants + .AsNoTracking() + .AnyAsync(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == profileId && + x.TensorGroupProfileId != currentProfileId && + x.BaselineQuantDefinitionId == baselineDefinitionId && + x.TensorWeightSchemeId == tensorWeightSchemeId, ct); + if (exists) + return profileId; + } + + return null; + } + + private static async Task<(Guid BenchmarkId, bool Cloned)> EnsurePureBaselineBenchmarkCloneAsync( + MagicQuantContext db, + IReadOnlyList sourceRows, + int architectureFamilyId, + int currentProfileId, + uint scopedModelHashId, + int? currentImatrixDefinitionId, + CancellationToken ct) + { + var sourceBenchmarkId = sourceRows.Select(x => x.AiBenchmarkId).FirstOrDefault(x => x != Guid.Empty); + if (sourceBenchmarkId == Guid.Empty) + throw new InvalidOperationException("Cannot rebucket learned rows because the source AiBenchmarkId snapshot is missing."); + + var sourceBenchmark = await db.AiBenchmarks + .Include(x => x.CategorBenchmarks) + .AsNoTracking() + .FirstOrDefaultAsync(x => x.Id == sourceBenchmarkId, ct); + + if (sourceBenchmark == null) + throw new InvalidOperationException($"Cannot rebucket learned rows because source AiBenchmarkId={sourceBenchmarkId} was not found."); + + var target = await db.AiBenchmarks + .FirstOrDefaultAsync(x => x.ArchitectureFamilyId == architectureFamilyId && + x.TensorGroupProfileId == currentProfileId && + x.AiModelHashId == scopedModelHashId && + x.ImatrixDefinitionId == currentImatrixDefinitionId && + x.TensorComboId == sourceBenchmark.TensorComboId, ct); + + if (target != null) + return (target.Id, false); + + target = new AiBenchmark + { + Id = Guid.NewGuid(), + ArchitectureFamilyId = architectureFamilyId, + TensorGroupProfileId = currentProfileId, + AiModelHashId = scopedModelHashId, + ImatrixDefinitionId = currentImatrixDefinitionId, + TensorComboId = sourceBenchmark.TensorComboId, + Ngl = sourceBenchmark.Ngl, + SizeBytes = sourceBenchmark.SizeBytes, + TokensPerSecond = sourceBenchmark.TokensPerSecond + }; + + db.AiBenchmarks.Add(target); + await db.SaveChangesAsync(ct); + + if (sourceBenchmark.CategorBenchmarks.Count > 0) + { + db.Set().AddRange(sourceBenchmark.CategorBenchmarks.Select(x => new CategoryBenchmark + { + Id = Guid.NewGuid(), + AiBenchmarkId = target.Id, + Category = x.Category, + Kld = x.Kld, + Ppl = x.Ppl, + PplError = x.PplError + })); + + await db.SaveChangesAsync(ct); + } + + return (target.Id, true); + } +} + +public sealed class TensorGroupRebucketSummary +{ + public int BaselineSchemeSetsCopied { get; init; } + public int LearnedRowsCopied { get; init; } + public int PureBenchmarkRowsCloned { get; init; } + public int ExistingCurrentProfileSetsSkipped { get; init; } + public int FatalSetsSkipped { get; init; } + + public static TensorGroupRebucketSummary Empty { get; } = new(); +} diff --git a/src/MagicQuant/Services/TensorGroupReviewService.cs b/src/MagicQuant/Services/TensorGroupReviewService.cs new file mode 100644 index 0000000..298ada4 --- /dev/null +++ b/src/MagicQuant/Services/TensorGroupReviewService.cs @@ -0,0 +1,138 @@ +using MagicQuant.Models.Learning; +using MagicQuant.Services.Learning; +using MQ.DB; +using MQ.DB.Models; +using Spectre.Console; + +namespace MagicQuant.Services; + +public sealed class TensorGroupReviewService +{ + private readonly TensorGroupingAuditService _auditService = new(); + + public async Task ReviewNativeTensorGroupingAsync( + QuantizationService quantizationService, + string nativeGgufPath, + bool requireConfirmation, + CancellationToken ct = default) + { + if (quantizationService == null) + throw new ArgumentNullException(nameof(quantizationService)); + + if (string.IsNullOrWhiteSpace(nativeGgufPath) || !File.Exists(nativeGgufPath)) + throw new FileNotFoundException($"Native GGUF path not found for tensor-group review: {nativeGgufPath}"); + + var tensorTypes = await quantizationService.ReadExactTensorTypesAsync(nativeGgufPath, ct); + var truth = tensorTypes + .OrderBy(x => x.Key, StringComparer.Ordinal) + .ToDictionary( + x => x.Key, + x => new LearnedTensorTruth(x.Key, x.Value, LearningSource.GgufOnly), + StringComparer.Ordinal); + + var audit = _auditService.Audit(truth.Keys.ToList(), truth); + + PrintReview(nativeGgufPath, truth, audit); + + if (audit.HasFatalIssues) + { + throw new InvalidOperationException( + "Tensor-group regex review found fatal grouping issues before learning/search could continue. " + + $"Ambiguous={audit.Ambiguous.Count}, IllegalUnresolved={audit.IllegalUnresolved.Count}. " + + "Fix tensor_groups.yaml and rerun."); + } + + if (requireConfirmation) + { + bool confirmed = AnsiConsole.Confirm( + "Continue with this tensor grouping profile? Review the counts above before saying yes."); + + if (!confirmed) + { + throw new OperationCanceledException( + "Pipeline run cancelled by user after tensor-group profile review. No tensor-group-scoped learning/search work was started."); + } + } + else + { + AnsiConsole.MarkupLine("[yellow]Tensor-group confirmation skipped by config/CLI.[/]"); + } + + return audit; + } + + private static void PrintReview( + string nativeGgufPath, + IReadOnlyDictionary truth, + TensorGroupingAuditResult audit) + { + string snapshotJson = TensorGroupProfileService.BuildSnapshotJson(); + string fingerprint = TensorGroupProfileService.ComputeSnapshotHash(snapshotJson); + + AnsiConsole.Write(new Rule("[yellow]Tensor Group Regex Review[/]") { Justification = Justify.Left }); + AnsiConsole.MarkupLine($"[grey]Native source:[/] {Markup.Escape(Path.GetFileName(nativeGgufPath))}"); + AnsiConsole.MarkupLine($"[grey]Tensor group profile hash:[/] [cyan]{Markup.Escape(fingerprint[..Math.Min(16, fingerprint.Length)])}[/]"); + AnsiConsole.MarkupLine($"[grey]Tensors inspected:[/] [cyan]{truth.Count:N0}[/]"); + AnsiConsole.MarkupLine($"[grey]BaseQuant exception tensors:[/] [cyan]{audit.BaseQuantExceptions.Count:N0}[/]"); + AnsiConsole.MarkupLine($"[grey]Ambiguous tensors:[/] [{(audit.Ambiguous.Count == 0 ? "green" : "red")}]{audit.Ambiguous.Count:N0}[/]"); + AnsiConsole.MarkupLine($"[grey]Illegal unresolved tensors:[/] [{(audit.IllegalUnresolved.Count == 0 ? "green" : "red")}]{audit.IllegalUnresolved.Count:N0}[/]"); + + var table = new Table() + .Border(TableBorder.Rounded) + .AddColumn("Id") + .AddColumn("Group") + .AddColumn(new TableColumn("Tensors").RightAligned()) + .AddColumn("Top native types") + .AddColumn("Examples"); + + foreach (var group in TReg.All.OrderBy(x => x.UniqueId)) + { + var tensors = audit.GroupedByTensor + .Where(x => x.Value.PrimaryGroup?.UniqueId == group.UniqueId) + .Select(x => x.Key) + .OrderBy(x => x, StringComparer.Ordinal) + .ToList(); + + var distribution = tensors + .Select(t => truth.TryGetValue(t, out var row) ? row.FinalQuantType : "unknown") + .GroupBy(x => x, StringComparer.Ordinal) + .OrderByDescending(x => x.Count()) + .ThenBy(x => x.Key, StringComparer.Ordinal) + .Take(4) + .Select(x => $"{x.Key}:{x.Count():N0}"); + + var examples = tensors.Take(4).Select(Markup.Escape); + + table.AddRow( + group.UniqueId.ToString(), + Markup.Escape(group.Name), + tensors.Count.ToString("N0"), + Markup.Escape(string.Join(", ", distribution)), + string.Join("\n", examples)); + } + + AnsiConsole.Write(table); + + PrintIssuePreview("Ambiguous group collisions", audit.Ambiguous); + PrintIssuePreview("Illegal unresolved tensors", audit.IllegalUnresolved); + PrintIssuePreview("BaseQuant exception tensors", audit.BaseQuantExceptions); + } + + private static void PrintIssuePreview(string heading, IReadOnlyList issues) + { + if (issues.Count == 0) + return; + + AnsiConsole.MarkupLine($"[yellow]{Markup.Escape(heading)}:[/] showing first {Math.Min(10, issues.Count):N0} of {issues.Count:N0}"); + foreach (var issue in issues.Take(10)) + { + var extra = issue.MatchedGroups.Count > 0 + ? $" groups=[{string.Join(", ", issue.MatchedGroups)}]" + : issue.MatchedExceptionPattern != null + ? $" pattern={issue.MatchedExceptionPattern}" + : string.Empty; + + AnsiConsole.MarkupLine($" [grey]-[/] {Markup.Escape(issue.TensorName)} [grey]{Markup.Escape(extra)}[/]"); + } + } +} diff --git a/src/MagicQuant/config.default.yaml b/src/MagicQuant/config.default.yaml new file mode 100644 index 0000000..b0809f5 --- /dev/null +++ b/src/MagicQuant/config.default.yaml @@ -0,0 +1,453 @@ +# ============================================================ +# MagicQuant - Model-neutral Starter Configuration +# ============================================================ +# +# This is a model-neutral starting profile, not a tuned preset for any model. +# Set the model path and architecture identity deliberately before running. +# Research thresholds are starting policies, not universal quality guarantees. +# +# General rules: +# - CLI flags override values from this YAML. +# - Debug and Release use this file unless --config is supplied. +# - Leave values blank when they must be provided per-machine or per-run. +# +# Notes: +# - "standard_baselines_mode: all" keeps the built-in baseline families active. +# - Custom repositories can add extra learned baseline sources, such as Unsloth. +# - External/custom baselines are LEARNED from, then rebuilt by MagicQuant +# under MagicQuant-controlled conditions before benchmarking. +# - The old early learned-baseline pruning logic is disabled in code now, +# so the isolation_pruning section mainly controls later tradeoff logic. +# ============================================================ + +paths: + # Root MagicQuant working directory. + # If blank, defaults to /MagicQuant. + magic_quant_root: + + # REQUIRED for real runs. + # This should point to the local source model folder containing safetensors. + model_dir: + + # Optional if MagicQuant auto-discovers or bootstraps llama.cpp. + llama_root: + llama_bin: + convert_script: + + # Scratch roots for temporary heavy GGUF writes. + # MagicQuant creates .MagicQuant_tmp under each root and enforces one heavy writer per root. + # Roots are not validated as separate physical disks; choose paths intentionally. + # When blank, MagicQuant falls back to single-root model-local scratch behavior. + scratch_roots: [] + + # Folder name created under the current model's MagicQuant directory for durable external/custom GGUF downloads. + external_baseline_cache_dir_name: ExternalBaselines + +flags: + # Whether MagicQuant should use an imatrix when supported and configured. + use_imatrix: false + + # Force rebuilding the active imatrix artifact even if one already exists. + force_imatrix_rebuild: false + + # Force rerunning hardware execution-plan probing. + force_refresh_hardware_probe: false + + # Whether exact high-precision hybrid aliases (BF16/F16-style explicit overrides) + # are allowed in hybrid generation logic. + allow_high_precision_hybrids: false + +learning: + # Destructive relearn options are intentionally targeted. + # These are transient runtime commands and are not persisted as DB state. + # When any option below is enabled, MagicQuant prints a count summary and asks + # for confirmation before deleting/relearning anything. + # + # Deletes learned mappings, benchmark truth, dependent benchmark/source rows, + # and execution probe cache rows scoped to the active architecture family. + # Does not delete AiModelHash, ArchitectureFamily, ImatrixDefinition, + # TensorCombo, or BaselineQuantDefinition rows. + force_relearn_architecture_family: false + + # Relearn built-in/standard baselines by display/canonical name for the current + # architecture family and active tensor group profile. + # Example: + # force_relearn_standard_baselines: + # - Q6_K + # - IQ4_XS + force_relearn_standard_baselines: [] + + # Safety gate for tensor group regex/profile changes. After MagicQuant reads the + # native BF16 GGUF tensor list, it prints group counts, example tensors, + # ambiguous matches, unresolved tensors, and base-quant exception counts, then + # asks before continuing. Keep this true unless running fully unattended. + confirm_tensor_group_profile: true + + # Safe/idempotent repair mode for accidental regex mistakes. + # + # Default true: on every run MagicQuant checks whether older DB learned tensor + # truth can be copied into the active TensorGroupProfile by reapplying the + # current regex/base_quant_exceptions rules. If nothing changed or current rows + # already exist, it skips cleanly and does not create duplicates. + # + # This avoids needless re-download/re-quantization of pure learning baselines + # after regex-only regrouping. Old benchmarks/learned rows remain attached to + # their original TensorGroupProfile and are ignored unless that profile becomes + # active again. + # + # Disable only when you intentionally want the slower/full path to regenerate + # learned grouping truth instead of rebucketing from DB snapshots. + # CLI disable aliases: + # --no-rebucket-learned-tensor-groups + # --disable-tensor-group-rebucket + # --full-relearn-tensor-groups + rebucket_learned_tensor_groups_from_existing_truth: true + +readme: + # Optional title model name override used in: + # # MagicQuant Hybrids - + # If blank, MagicQuant uses identity.architecture_family_name. + title_model_name_override: + + # Hugging Face README frontmatter. + # Scalars render as: + # license: apache-2.0 + # Arrays render as: + # tags: + # - gguf + # - text-generation + # + # Add more keys freely, such as base_model, datasets, language, pipeline_tag, etc. + frontmatter: + # Set license and base_model to match the actual source model before publishing. + tags: + - gguf + - text-generation + - magicquant + +hardware: + # Optional per-GPU usable VRAM limits in GB. + # Leave empty for automatic/default llama.cpp placement. + # When provided and a benchmark slot uses multiple GPUs, MagicQuant will pass + # --tensor-split in visible GPU order. + # + # Example: + # gpu_memory_limits_gb: + # 0: 19 + # 1: 23 + gpu_memory_limits_gb: {} + +imatrix: + # Optional remote imatrix URL if your flow supports fetching one. + imatrix_url: + + # Optional dataset repo for building an imatrix from a dataset source. + dataset_repo: + + # Optional dataset split, for example: text, train, validation + dataset_split: + + # Optional dataset config / subset name. + dataset_config: + + # Optional local dataset file path for imatrix generation. + # Example: + # dataset_local_file: /data/datasets/imatrix-general-v1-1m.jsonl + dataset_local_file: + +# Legacy evolution survivor knobs were removed from YAML. +# Final hybrid selection is now driven by rank-safe isolation prediction plus candidate_selection. + +isolation_pruning: + # NOTE: + # Early learned-baseline pruning has been removed from the code path. + # These values still matter for later isolation / bad-trade reasoning, + # not for the old "skip candidate because learned tensor usage looked redundant" path. + + # Minimum carrier-relative size reduction required before generating the remaining + # candidate-isolation samples for a tensor group. Keep this at 0 when prediction, + # contextual probing, or anomaly analysis needs complete isolation truth; later + # quality and bad-trade filters still remove unhelpful candidates. + minimum_isolation_reduction_to_continue_ratio: 0.00 + + # Minimum reduction ratio before BF16 suppression logic is allowed to kick in. + minimum_isolation_reduction_to_suppress_bf16_ratio: 0.10 + + # Maximum allowed isolation PPL delta percent before considering the trade poor. + maximum_isolation_ppl_delta_percent: 5.0 + + # Maximum allowed isolation KLD before considering the trade poor. + maximum_isolation_kld: 0.1 + + # If size savings are below this percent, the trade may be treated as bad. + bad_trade_max_size_delta_percent: 4.0 + + # Multiplier thresholds for bad-trade reasoning. + bad_trade_kld_multiplier: 2.5 + bad_trade_ppl_multiplier: 3.5 + + # Float comparison tolerance. + floating_point_epsilon: 1.0e-8 + + # Minimum meaningful reduction ratio for base-only comparisons. + minimum_meaningful_base_only_reduction_ratio: 0.01 + + +prediction: + # Rank-safe isolation KLD predictor. + # + # manual_max_predicted_size_bytes is retained only as an emergency compatibility + # field for older helper code. Leave it at 0 for the new chooser. + manual_max_predicted_size_bytes: 0 + + # Candidate bit-stress thresholds for the low-bit interaction correction. + # The predictor fits each candidate threshold against existing category=General + # benchmark truth and keeps the best MAE fit for the active model/imatrix bucket. + bit_stress_threshold_candidates: + - 4.0 + - 5.0 + - 6.0 + - 7.0 + - 8.0 + - 9.0 + - 10.0 + - 11.0 + - 12.0 + + # Fallback threshold when too few benchmark rows exist to fit the interaction model. + default_bit_stress_threshold: 8.0 + + # Minimum benchmark rows required before fitting the interaction correction. + minimum_fit_rows: 12 + +candidate_selection: + + validate_all_anomaly_strict_candidates_after_success: false + + # Phase 2: a hybrid can replace the smaller/higher-damage anchor when it fits + # inside this size premium and beats the real linear KLD improvement line. + near_baseline_max_size_growth_percent: 1.0 + + # Phase 3: interior windows between adjacent final anchors. + # [0.35, 0.35] means test the first 35% of the size span, then the next 35%. + interior_window_fractions: + - 0.35 + - 0.35 + + # Number of predicted winners to keep per interior window. + max_candidates_per_interior_window: 1 + + # If the first predicted candidate fails real validation, try this many fallbacks. + max_fallback_attempts_per_anchor: 5 + + # Conservative SQLite/isolation-truth fallback. This runs only after the + # normal DuckDB prediction-guided attempts fail for a strict/premium/interior + # phase window. It starts from the anchor baseline blanket and only swaps + # tensor groups using surviving isolated group candidates, plus the baseline + # itself as the blanket state. + smart_fallback_enabled: true + smart_fallback_attempts_per_failure: 3 + smart_fallback_max_higher_fidelity_steps: 2 + + # Strict epsilon for lower-KLD comparisons after real benchmark validation. + minimum_kld_improvement_epsilon: 1.0e-9 + + # Final spacing pass: candidates closer than this fraction of the global survivor + # size span are collapsed unless one genuinely earns the slot. + minimum_neighbor_gap_fraction_of_global_span: 0.03 + + # Extra-brutal zone near the smaller anchor. A candidate this close to the smaller + # anchor must provide a stronger KLD gain to justify its existence. + near_lower_anchor_brutal_zone_fraction_of_pair_span: 0.02 + near_anchor_required_kld_gain_fraction_of_pair_gap: 0.05 + + # This distributed config enables final prediction/build attempts to replace + # 8-bit anchors such as Q8_0 during strict dominance or near-anchor replacement. + # Set false to keep Q8 as the highest-fidelity practical anchor. + allow_eight_bit_anchor_replacements: true + +anomaly_detection: + enabled: true + + # One anomaly refinement pass after smoke/probe/rule generation. + max_anomaly_refinement_rounds: 1 + + # Minimum actual KLD gain versus higher-bit counterfactual twin to confirm anomaly. + min_actual_gain_vs_twin_kld: 0.00025 + + # Minimum predicted size savings versus higher-bit twin/reference to probe. + min_predicted_size_savings_vs_twin_percent: 1.0 + + # Max changed groups in a candidate that can seed contextual probes. + max_probe_group_count: 4 + + # Max probes generated per anomaly seed. + max_probes_per_seed: 16 + + # Max anomaly probes in one run. + max_total_probes_per_run: 32 + + # Strong smoke if a monotone downgrade candidate is this close to or better than its twin in prediction space. + max_prediction_space_gap_vs_twin_kld: 0.00050 + + # Optional relative cap for prediction-space gap normalized by local anchor gap. + max_relative_prediction_penalty_vs_twin: 0.35 + + # Minimum margin used when forcing confirmed anomalies below their higher-bit twin in prediction space. + prediction_space_violation_margin: 0.00005 + + # Shrink applied to prediction-space adjustment after a rule is confirmed. + anomaly_adjustment_shrink_factor: 0.50 + + # Minimum confidence required before applying a confirmed anomaly rule. + min_rule_confidence_to_apply: 0.50 + + # Absolute cap on total negative anomaly adjustment in prediction-space KLD units. + max_negative_adjustment_kld: 0.00075 + + # Absolute cap on positive harmful interaction adjustment in prediction-space KLD units. + max_positive_adjustment_kld: 0.00075 + + # Fractional cap relative to BaseRankSafeKld. + max_adjustment_fraction_of_base_kld: 0.75 + + # Number of top smoke candidates to consider per reference quant zone. + max_smoke_candidates_per_reference_zone: 12 + + # Store suppression-only results so false smoke is not repeatedly probed. + persist_suppression_results: true + + # Emit detailed anomaly logs. + verbose_anomaly_logging: true + + # Small bounded sniff pass around already-confirmed beneficial contextual anomalies. + confirmed_anomaly_expansion: + enabled: true + max_neighbors_per_confirmed_rule: 6 + max_total_expansion_probes: 12 + allowed_reference_quants: + - Q8_0 + # Built-in families only. Add exact custom baseline names explicitly after + # configuring a compatible provider; see examples/pipeline-external.yaml. + allowed_candidate_quants: + - Q6_K + - Q5_K + +output: + # Optional explicit output directory. + # If blank, MagicQuant will default to: + # /MagicQuant/Final_Outputs + output_dir: + + # Prefix used when generating exported GGUF file names. + output_name_prefix: Model + + # By default MagicQuant will not locally export pure learned external/custom baselines + # such as Unsloth. They remain upstream references in the README/output unless enabled. + # + # Set true if you explicitly want MagicQuant to rebuild/export those external learned + # baselines locally under MagicQuant-controlled conditions (for example when testing a + # modified model where the upstream artifact does not really exist for your case). + # Recommended: leave off when the original provider hosts the same model; + # link to their release and credit their work. Enable for pipeline runs on + # variants they do not host. clone-repository-quants already rebuilds every + # manifest entry locally and does not use this pipeline export setting. + # Learning tensor assignments does not reproduce other provider techniques. + export_external_learned_baselines: false + + # false = normal behavior; delete/rebuild final outputs from scratch. + # true = preserve valid existing GGUFs and skip rebuilding them only when + # exact file name + byte size match benchmark truth. + # CLI --reuse-existing-final-artifacts overrides YAML. + reuse_existing_final_artifacts: false + +# Legacy bit-range bucket survival settings were removed. +# See candidate_selection above for the active final chooser settings. + +identity: + architecture_family_name: + allow_architecture_family_alias_override: false + +baselines: + # ---------------------------------------------------------- + # standard_baselines_mode options + # ---------------------------------------------------------- + # all + # Keep built-in standard baselines active AND allow custom repositories. + # + # selected + # Use the explicit built-in role lists below; empty lists enable none for that role. + # + # none + # Disable built-in learning/carrier/explicit-group roles. Internal native anchors + # may still be required. Custom repositories are configured independently. + standard_baselines_mode: all + + # These lists apply only in selected mode. In all mode, built-in defaults apply. + # + # Example: + # enabled_standard_learning_baselines: [Q8_0, Q6_K, Q5_K, Q4_K_M] + enabled_standard_learning_baselines: [] + + # Example: + # enabled_standard_combination_carriers: [Q8_0, Q6_K, Q5_K] + enabled_standard_combination_carriers: [] + + # Example: + # enabled_standard_explicit_group_candidates: [Q8_0, Q6_K, Q5_K, Q4_K_M, IQ4_NL, IQ4_XS] + enabled_standard_explicit_group_candidates: [] + + # Optional external tensor-configuration sources. No provider/model is selected. + # See examples/pipeline-external.yaml and docs/best-practices.md for a template. + # Use the exact source model and provider filenames; pin the provider revision. + custom_repositories: [] + +# Counterfactual synergy templates generalize confirmed contextual anomaly evidence. +# anomaly_detection remains the low-level compatibility section; synergy_detection controls +# template transfer, composition probes, contamination suppression, and wing diagnostics. +synergy_detection: + enabled: true + max_refinement_rounds: 1 + exact_context_confidence_multiplier: 1.00 + same_selected_groups_confidence_multiplier: 0.55 + equivalent_quant_family_confidence_multiplier: 0.30 + group_family_suspicion_confidence_multiplier: 0.15 + min_confidence_to_apply_adjustment: 0.35 + min_confidence_to_schedule_transfer_probe: 0.25 + max_negative_adjustment_kld: 0.002 + max_negative_adjustment_fraction_of_base_kld: 0.75 + transfer_probe_enabled: true + max_transfer_probes_per_template: 6 + max_total_transfer_probes_per_run: 24 + # Controlled all-surrounding-groups blankets. This measures whether a template's + # marginal behavior transfers—or flips—without importing an old winning mixture. + transfer_probe_context_strata: + high_fidelity_reference_quants: [Q6_K, Q5_K] + mid_fidelity_reference_quants: [Q4_K_M] + low_fidelity_reference_quants: [IQ3_S] + low_fidelity_enabled: false + # Remeasure the best and closest-size non-equivalent same-bit isolation recipes head-to-head inside + # controlled contexts. This detects context-dependent rank flips without replaying + # an old winning mixture. Low-fidelity contexts remain governed by the opt-in above. + exploratory_context_pair_enabled: true + max_exploratory_context_pairs_per_run: 14 + exploratory_pair_bit_ranges: [4] + exploratory_pair_context_strata: [mid-fidelity, low-fidelity] + # Match rules against their measured effective surrounding-group context, + # independent of the search row's carrier quant. Rule-selected groups are excluded. + context_scoped_rule_application_enabled: true + max_non_rule_group_context_mismatches: 1 + verbose_synergy_logging: true + min_smoke_score: 0.55 + max_smoke_gap_kld: 0.004 + top_rejected_smoke_preview: 25 + composition_probe_enabled: true + max_template_composition_group_count: 4 + max_composition_probes_per_run: 8 + max_templates_to_compose: 4 + min_template_confidence_for_composition: 0.50 + min_combined_expected_size_savings_percent: 1.0 + contaminating_passenger_detection_enabled: true + min_failure_margin_for_contamination_kld: 0.00050 + contamination_penalty_confidence_multiplier: 0.45 + suppress_repeated_contaminated_attempts: true diff --git a/src/MagicQuant/packages.lock.json b/src/MagicQuant/packages.lock.json new file mode 100644 index 0000000..1784fe7 --- /dev/null +++ b/src/MagicQuant/packages.lock.json @@ -0,0 +1,259 @@ +{ + "version": 1, + "dependencies": { + "net10.0": { + "Blake3": { + "type": "Direct", + "requested": "[2.2.0, )", + "resolved": "2.2.0", + "contentHash": "RM6sZLZDx2wGi00aTj9s2jUcrI4s9dS2ibcT7lSujpUpBGp+TLf71F3XdBKJyYSxHnZ+FL7Dm36Pl0Y+cfcXvw==" + }, + "DuckDB.NET.Data.Full": { + "type": "Direct", + "requested": "[1.4.3, )", + "resolved": "1.4.3", + "contentHash": "tg1FWmePN+k536O1cx2VhKWa3xT7DXrcGg4kGgiSWQyur9UWwZ2i2YMpD+XVYv1ARozyy1Tt7OW2cMrNPoPj9g==", + "dependencies": { + "DuckDB.NET.Bindings.Full": "1.4.3" + } + }, + "LibGit2Sharp": { + "type": "Direct", + "requested": "[0.31.0, )", + "resolved": "0.31.0", + "contentHash": "b3+UfV7LjKMjAHWwl7VawejiOv2gJIC6dTCA/S0puLTHACAA/Oeb5JJmWUQMeyH/T/WR/LaIK8bk2RbdFnrZvg==", + "dependencies": { + "LibGit2Sharp.NativeBinaries": "[2.0.323]" + } + }, + "Spectre.Console": { + "type": "Direct", + "requested": "[0.54.0, )", + "resolved": "0.54.0", + "contentHash": "StDXCFayfy0yB1xzUHT2tgEpV1/HFTiS4JgsAQS49EYTfMixSwwucaQs/bIOCwXjWwIQTMuxjUIxcB5XsJkFJA==" + }, + "System.Management": { + "type": "Direct", + "requested": "[10.0.11, )", + "resolved": "10.0.11", + "contentHash": "xyNn8KGbWI88LoUwg3rB8qcpFFST6dr8Ro/qS8GBu2GOwR0v7J82kVFHTiiPtvEKS79VbMTxs/sIKQ+Cq1Zs1g==", + "dependencies": { + "System.CodeDom": "10.0.11" + } + }, + "YamlDotNet": { + "type": "Direct", + "requested": "[17.0.1, )", + "resolved": "17.0.1", + "contentHash": "qVir5fehR/W5nTJyoJUibypETXaW4iRAF9cQa0FQIC9TJ3VC0qDOwm4o/RxANewj8KzPF8WMF2abBfUgi6LC4w==" + }, + "DuckDB.NET.Bindings.Full": { + "type": "Transitive", + "resolved": "1.4.3", + "contentHash": "hZwm0zTKJ5HdUGKcase2JX52Lquyh7dCUFweECvR877QEA2gF8gSl3qrtb71BvRlgZ7pfjh0bRBCiAKOJMLE+A==" + }, + "LibGit2Sharp.NativeBinaries": { + "type": "Transitive", + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" + }, + "Microsoft.Data.Sqlite": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "7je7UELzm131GiLYc4PpZvfKXIgIyzPM+v+tjcd/nbnuWRfgcONYKzDTqJlURxwVCFsVnlpmq6y6yn4qvR8QXQ==", + "dependencies": { + "Microsoft.Data.Sqlite.Core": "10.0.11", + "SQLitePCLRaw.bundle_e_sqlite3": "2.1.12", + "SQLitePCLRaw.core": "2.1.12" + } + }, + "Microsoft.Data.Sqlite.Core": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "hubA20AGenQ4Sx0ElWaPpB8DISjXpdx463+1zOGRslsT0e/t/06ITv+pHsop8CcJ0d8PZLfgnT7juCDVD79Dkw==", + "dependencies": { + "SQLitePCLRaw.core": "2.1.12" + } + }, + "Microsoft.EntityFrameworkCore": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "VOSGU8en6HZJs8t7UMFN+9vGcRgVOOn6fA44Ngcg2NyvJ3P1KE94iAb0XzaVaGhXGtt+qaM/VtEn0/hzluQJeg==", + "dependencies": { + "Microsoft.EntityFrameworkCore.Abstractions": "10.0.11", + "Microsoft.EntityFrameworkCore.Analyzers": "10.0.11", + "Microsoft.Extensions.Caching.Memory": "10.0.11", + "Microsoft.Extensions.Logging": "10.0.11" + } + }, + "Microsoft.EntityFrameworkCore.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "6auJR+9+9VunznKfH7WGrHMrnrmA0F7JZ22EXzwXvVhjfnbu9Xq7NSIWaOf3KJsOanM2qf5ajJ2JR5TlcPZTLA==" + }, + "Microsoft.EntityFrameworkCore.Analyzers": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "Bv7X4wSSnzCQED9WYXKJ8fwgyvKwf0xZM1GO8xkf6CF9zl+UBnvjxmcPnokJRy0JKjc1SlHSzzhx1HcL4jitTQ==" + }, + "Microsoft.EntityFrameworkCore.Relational": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "grznnTJgEYxaWpdKAsTzg6j+89jHgCXWYp+QGtlX5O92+w/VuhWM6JLPYb+uw8M9VhGUvOTsO76dYOy9vNPd5Q==", + "dependencies": { + "Microsoft.EntityFrameworkCore": "10.0.11", + "Microsoft.Extensions.Caching.Memory": "10.0.11", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.Logging": "10.0.11" + } + }, + "Microsoft.EntityFrameworkCore.Sqlite": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "jc7iVrhQyInR3loraMESfEFaFOtQOB1mRKHjX6QYC9o7YDbfMNbAPnIwlpffnFwhXd6/27FKaaV+sWSoLd4F1g==", + "dependencies": { + "Microsoft.EntityFrameworkCore.Sqlite.Core": "10.0.11", + "Microsoft.Extensions.Caching.Memory": "10.0.11", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.DependencyModel": "10.0.11", + "Microsoft.Extensions.Logging": "10.0.11", + "SQLitePCLRaw.bundle_e_sqlite3": "2.1.12", + "SQLitePCLRaw.core": "2.1.12" + } + }, + "Microsoft.EntityFrameworkCore.Sqlite.Core": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "ywTQKt32xnVhCzjEQAqFufpEyXkOUfvW/EC/s4xnS8Xaor2xXE+TMUyzhgACqXtZEU5IR95y94RDzHto55Fx7w==", + "dependencies": { + "Microsoft.Data.Sqlite.Core": "10.0.11", + "Microsoft.EntityFrameworkCore.Relational": "10.0.11", + "Microsoft.Extensions.Caching.Memory": "10.0.11", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.DependencyModel": "10.0.11", + "Microsoft.Extensions.Logging": "10.0.11", + "SQLitePCLRaw.core": "2.1.12" + } + }, + "Microsoft.Extensions.Caching.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "vUl798SmruTqqlt/xH2gDk3tJlhk6k3HdOXAHirlRfbNKDym4g/kRpUL9S4sl6F6FsOTOMW+ZsDapqlZMOOiEw==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Caching.Memory": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "el1g0mBEbDBGY2bT9mcSfrTWO8QlPdq2nOCnvQugioOFwHV+bVBMeiakoI0dNOdj8d6Hi9K6HY2xzRUWJiDR3w==", + "dependencies": { + "Microsoft.Extensions.Caching.Abstractions": "10.0.11", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11", + "Microsoft.Extensions.Logging.Abstractions": "10.0.11", + "Microsoft.Extensions.Options": "10.0.11", + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "fVi053xdpda9Em7vSkmgVxO/PtgC2m78ekReKWsgcyskqY0U82Bz/MONwxpGzI0hElYKJfw+fupqMVeKW3fSaA==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "PSmotV19c7E3lKed++uYo1kSiXFI+uTl37CBSrhq+CfLC3FCHjG7R91+xPnNehQfHS1b0Tzo/CCLPWH3qaEheg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "/a1aJz4m7ylhEDf25ugQChLQoN5XwoGjWw/BoR/ZWWKsO1v4DdJElS1uyngahz4B/eOzjFk1KNTkarRLE5wsIg==" + }, + "Microsoft.Extensions.DependencyModel": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "PJPtFYsZ+r+uz9qqXWUTEyKeJ1EiBGIJtqavkg9ZXijjGSFAk4Fgi5sqIxj+uAyLZwEKgexDUQXhWhvU6l3+og==" + }, + "Microsoft.Extensions.Logging": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "nUOJwgFkSiLHiVGFpU22pIJtuWYewuSYQ3JVuP/gdK8ASMT807Px+TYQiRWs6uSsOmoyFTaVCwKXTasczV6BpA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "10.0.11", + "Microsoft.Extensions.Logging.Abstractions": "10.0.11", + "Microsoft.Extensions.Options": "10.0.11" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "Ljd0Uxoq5XpScD2Bg0nM/r3mwx7Ao5Uq24eo2ARxbGvqJ7Zht6rt2cJtwVRH4Cv+1ZVMdXz6TB43KbpmsxRrvQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11" + } + }, + "Microsoft.Extensions.Options": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "eY1GAKcTfD2maP27J84X9IovT3yjHJ2dVDzPmDg6/XqYvt3jMzJhtfQCLjG9pVsZGAd+8DQ2QrjaDcs2+VQLGw==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11", + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "SXcz+kF+4Oo9b1+55zntpJFYfwb1jw66ioxptyNOOTDc8g2FHnBFWjZpsWfCvZIhzr0x+4e2trVTs4OKwQfBtw==" + }, + "SQLitePCLRaw.bundle_e_sqlite3": { + "type": "Transitive", + "resolved": "2.1.12", + "contentHash": "mAgscpQMLw5/nfA1Q5oJVAT29yROUo1ifZGbbTpx/lwZpSxMUGoYbKfmvdm8oXER+RzxqBmmQzeBEVKfeHv2nw==", + "dependencies": { + "SQLitePCLRaw.lib.e_sqlite3": "2.1.12", + "SQLitePCLRaw.provider.e_sqlite3": "2.1.12" + } + }, + "SQLitePCLRaw.core": { + "type": "Transitive", + "resolved": "2.1.12", + "contentHash": "ETpNw9DY3ckWLgRRAeCHj+GKOuPi61aeczkXhgHexUvqoZBAYg8RYESE2J7O1M7+o6QbdSEZwrw9bfqztUVWXg==" + }, + "SQLitePCLRaw.lib.e_sqlite3": { + "type": "Transitive", + "resolved": "2.1.12", + "contentHash": "fWi8Dbknuhgg72fWinIdjXVaqO1hHL4YBBwVLnr7e1c9TAZwJ0QE38j9syW1hwx6HaqEVTwI+O07WPdZn8Rp0w==" + }, + "SQLitePCLRaw.provider.e_sqlite3": { + "type": "Transitive", + "resolved": "2.1.12", + "contentHash": "W3oH4XIfCzFrgUSDKHhN6N+dgzA5YHOR2VxX8GB6Qy7CyrJJgxPEG8NirgYWlPQC5P2jz2knSsexWu4tDUL33g==", + "dependencies": { + "SQLitePCLRaw.core": "2.1.12" + } + }, + "System.CodeDom": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "v40pNeBoZTYsiVxz+PzyZmmIr2JIhpK4VsFpQqZSZCXa51PDlNXIN2ESm8kDU0voZYVfLhxF9HvmBsxCJmkiRg==" + }, + "mq.db": { + "type": "Project", + "dependencies": { + "Microsoft.Data.Sqlite": "[10.0.11, )", + "Microsoft.EntityFrameworkCore": "[10.0.11, )", + "Microsoft.EntityFrameworkCore.Sqlite": "[10.0.11, )", + "YamlDotNet": "[17.0.1, )" + } + } + } + } +} \ No newline at end of file diff --git a/tests/MagicQuant.ProcessFixture/MagicQuant.ProcessFixture.csproj b/tests/MagicQuant.ProcessFixture/MagicQuant.ProcessFixture.csproj new file mode 100644 index 0000000..0fda630 --- /dev/null +++ b/tests/MagicQuant.ProcessFixture/MagicQuant.ProcessFixture.csproj @@ -0,0 +1,9 @@ + + + Exe + net10.0 + enable + enable + false + + diff --git a/tests/MagicQuant.ProcessFixture/Program.cs b/tests/MagicQuant.ProcessFixture/Program.cs new file mode 100644 index 0000000..bd451b0 --- /dev/null +++ b/tests/MagicQuant.ProcessFixture/Program.cs @@ -0,0 +1,40 @@ +using System.Diagnostics; + +namespace MagicQuant.ProcessFixture; + +/// Offline child-process fixture. Used only by process lifetime regression tests. +public static class Program +{ + public static async Task Main(string[] args) + { + switch (args[0]) + { + case "echo": + foreach (string arg in args.Skip(1)) Console.WriteLine(arg); + return 0; + case "flood": + for (int i = 0; i < 12000; i++) + { + Console.WriteLine($"stdout-{i:D5}"); + Console.Error.WriteLine($"stderr-{i:D5}"); + } + return 7; + case "tree": + var start = new ProcessStartInfo("dotnet"); + start.ArgumentList.Add(typeof(Program).Assembly.Location); + start.ArgumentList.Add("wait"); + using (var child = Process.Start(start)!) + { + Console.WriteLine($"child:{child.Id}"); + await Task.Delay(TimeSpan.FromMinutes(5)); + } + return 0; + case "wait": + Console.WriteLine($"ready:{Environment.ProcessId}"); + await Task.Delay(TimeSpan.FromMinutes(5)); + return 0; + default: + return 2; + } + } +} diff --git a/tests/MagicQuant.ProcessFixture/packages.lock.json b/tests/MagicQuant.ProcessFixture/packages.lock.json new file mode 100644 index 0000000..4a91a8c --- /dev/null +++ b/tests/MagicQuant.ProcessFixture/packages.lock.json @@ -0,0 +1,6 @@ +{ + "version": 1, + "dependencies": { + "net10.0": {} + } +} \ No newline at end of file diff --git a/tests/MagicQuant.Tests/AnomalyContextScopeTests.cs b/tests/MagicQuant.Tests/AnomalyContextScopeTests.cs new file mode 100644 index 0000000..d4b4a59 --- /dev/null +++ b/tests/MagicQuant.Tests/AnomalyContextScopeTests.cs @@ -0,0 +1,182 @@ +using MagicQuant.Configuration; +using MagicQuant.Models; +using MagicQuant.Services; +using MQ.DB; +using MQ.DB.Models; +using MQ.DB.Models.DbModels; +using Xunit; + +namespace MagicQuant.Tests; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class AnomalyContextScopeCollection +{ + public const string Name = "Anomaly context scope"; +} + +[Collection(AnomalyContextScopeCollection.Name)] +public sealed class AnomalyContextScopeTests +{ + [Fact] + public void BuildContextFidelityPredicate_BoundsOnlyNonRuleGroups() + { + var priorConfig = Config.Current; + var priorUnusedGroups = Cache.UnusedTensorGroups.ToList(); + + try + { + var config = MagicQuantYamlConfig.CreateDefault(); + config.SynergyDetection.ContextScopedRuleApplicationEnabled = true; + config.SynergyDetection.MaxNonRuleGroupContextMismatches = 1; + Config.Load(config); + Cache.UnusedTensorGroups.Clear(); + + var rule = new AnomalyInteractionRule + { + ReferenceQuantId = BaselineQuants.Q8_0.UniqueId, + ReferenceContextKey = $"{TReg.LmHead.UniqueId}:{BaselineQuants.Q4_K_M.UniqueId}", + GroupStates = + [ + new AnomalyInteractionRuleGroupState + { + TensorGroupId = TReg.Embeddings.UniqueId, + ReferenceQuantId = BaselineQuants.Q8_0.UniqueId, + CandidateQuantId = BaselineQuants.Q6_K.UniqueId + } + ] + }; + + string predicate = AnomalyAdjustedPredictionService.BuildContextFidelityPredicate(rule, "c"); + + Assert.Contains("c.LmHead", predicate, StringComparison.Ordinal); + Assert.Contains($"= {BaselineQuants.Q4_K_M.UniqueId} THEN 0", predicate, StringComparison.Ordinal); + Assert.Contains("c.AttnQ", predicate, StringComparison.Ordinal); + Assert.DoesNotContain("c.Embeddings", predicate, StringComparison.Ordinal); + Assert.EndsWith("<= 1)", predicate, StringComparison.Ordinal); + } + finally + { + Config.Load(priorConfig); + Cache.UnusedTensorGroups.Clear(); + Cache.UnusedTensorGroups.AddRange(priorUnusedGroups); + } + } + + [Fact] + public void BuildRuleCandidateWhere_UsesEffectiveContextInsteadOfCarrierIdentity() + { + var priorConfig = Config.Current; + var priorUnusedGroups = Cache.UnusedTensorGroups.ToList(); + + try + { + var config = MagicQuantYamlConfig.CreateDefault(); + config.SynergyDetection.ContextScopedRuleApplicationEnabled = true; + config.SynergyDetection.MaxNonRuleGroupContextMismatches = 0; + Config.Load(config); + Cache.UnusedTensorGroups.Clear(); + + var rule = new AnomalyInteractionRule + { + ReferenceQuantId = BaselineQuants.Q4_K_M.UniqueId, + ReferenceContextKey = string.Join("|", TReg.All.Select(x => $"{x.UniqueId}:{BaselineQuants.Q4_K_M.UniqueId}")), + GroupStates = + [ + new AnomalyInteractionRuleGroupState + { + TensorGroupId = TReg.Embeddings.UniqueId, + ReferenceQuantId = BaselineQuants.Q4_K_M.UniqueId, + CandidateQuantId = BaselineQuants.IQ4_NL.UniqueId + } + ] + }; + + string predicate = AnomalyAdjustedPredictionService.BuildRuleCandidateWhere(rule, "c"); + + Assert.DoesNotContain($"c.BaseQuant = {BaselineQuants.Q4_K_M.UniqueId}", predicate, StringComparison.Ordinal); + Assert.Contains("c.Embeddings", predicate, StringComparison.Ordinal); + Assert.Contains("c.LmHead", predicate, StringComparison.Ordinal); + Assert.Contains($"= {BaselineQuants.Q4_K_M.UniqueId} THEN 0", predicate, StringComparison.Ordinal); + } + finally + { + Config.Load(priorConfig); + Cache.UnusedTensorGroups.Clear(); + Cache.UnusedTensorGroups.AddRange(priorUnusedGroups); + } + } + + [Fact] + public void BuildContextFidelityPredicate_ReturnsEmptyWhenDisabled() + { + var priorConfig = Config.Current; + + try + { + var config = MagicQuantYamlConfig.CreateDefault(); + config.SynergyDetection.ContextScopedRuleApplicationEnabled = false; + Config.Load(config); + + var rule = new AnomalyInteractionRule + { + ReferenceQuantId = BaselineQuants.Q8_0.UniqueId, + GroupStates = + [ + new AnomalyInteractionRuleGroupState + { + TensorGroupId = TReg.Embeddings.UniqueId, + ReferenceQuantId = BaselineQuants.Q8_0.UniqueId, + CandidateQuantId = BaselineQuants.Q6_K.UniqueId + } + ] + }; + + Assert.Empty(AnomalyAdjustedPredictionService.BuildContextFidelityPredicate(rule, "c")); + } + finally + { + Config.Load(priorConfig); + } + } + + [Fact] + public void RuleSuppressionKey_DistinguishesEffectiveSurroundingContext() + { + var priorUnusedGroups = Cache.UnusedTensorGroups.ToList(); + + try + { + Cache.UnusedTensorGroups.Clear(); + var movement = new QuantFidelityComparerService(); + var repository = new AnomalyRuleRepository(movement); + var q8Context = movement.CreateActivatedContextBlanket(BaselineQuants.Q8_0.UniqueId); + var q4PassengerContext = movement.WithStoredSlot( + q8Context, + TReg.LmHead, + BaselineQuants.EncodeTensorConfigGroupSlot(BaselineQuants.Q4_K_M)); + var changed = new List + { + new() + { + Group = TReg.Embeddings, + ReferenceQuantId = BaselineQuants.Q8_0.UniqueId, + CandidateQuantId = BaselineQuants.Q6_K.UniqueId, + ReferenceStoredSlot = BaselineQuants.EncodeTensorConfigGroupSlot(BaselineQuants.Q8_0), + CandidateStoredSlot = BaselineQuants.EncodeTensorConfigGroupSlot(BaselineQuants.Q6_K), + Movement = QuantMovementKind.Downgrade + } + }; + + string q8Key = repository.BuildRuleSuppressionKey(q8Context, changed); + string q4PassengerKey = repository.BuildRuleSuppressionKey(q4PassengerContext, changed); + + Assert.NotEqual(q8Key, q4PassengerKey); + Assert.Contains($"{TReg.LmHead.UniqueId}:{BaselineQuants.Q4_K_M.UniqueId}", q4PassengerKey, StringComparison.Ordinal); + } + finally + { + Cache.UnusedTensorGroups.Clear(); + Cache.UnusedTensorGroups.AddRange(priorUnusedGroups); + } + } +} diff --git a/tests/MagicQuant.Tests/AssemblyInfo.cs b/tests/MagicQuant.Tests/AssemblyInfo.cs new file mode 100644 index 0000000..2171200 --- /dev/null +++ b/tests/MagicQuant.Tests/AssemblyInfo.cs @@ -0,0 +1,3 @@ +using Xunit; + +[assembly: CollectionBehavior(DisableTestParallelization = true)] diff --git a/tests/MagicQuant.Tests/AuthorityUsageRegressionTests.cs b/tests/MagicQuant.Tests/AuthorityUsageRegressionTests.cs new file mode 100644 index 0000000..56d4360 --- /dev/null +++ b/tests/MagicQuant.Tests/AuthorityUsageRegressionTests.cs @@ -0,0 +1,24 @@ +using Xunit; + +namespace MagicQuant.Tests; + +public class AuthorityUsageRegressionTests +{ + [Fact] + public void ComboGenerationPaths_DoNotUseLegacyAllAllowedHybridQuantsAuthority() + { + string repositoryRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "../../../../..")); + var files = new[] + { + Path.Combine(repositoryRoot, "src", "MagicQuant", "Helpers", "ComboLogic.cs"), + Path.Combine(repositoryRoot, "src", "MagicQuant", "Helpers", "TensorConfigGenerator.cs"), + Path.Combine(repositoryRoot, "src", "MagicQuant", "Services", "IsolationOptimizationService.cs") + }; + + foreach (var file in files) + { + var text = File.ReadAllText(file); + Assert.DoesNotContain("All_Allowed_Hybrid_Quants", text); + } + } +} diff --git a/tests/MagicQuant.Tests/BaselineCandidatePolicyTests.cs b/tests/MagicQuant.Tests/BaselineCandidatePolicyTests.cs new file mode 100644 index 0000000..6a77672 --- /dev/null +++ b/tests/MagicQuant.Tests/BaselineCandidatePolicyTests.cs @@ -0,0 +1,178 @@ +using MagicQuant.Helpers; +using MQ.DB.Models; +using Xunit; + +namespace MagicQuant.Tests; + +public class BaselineCandidatePolicyTests +{ + [Fact] + public void Iq1Families_AreRegisteredAsImatrixLearningAndExplicitCandidates() + { + Assert.Equal((byte)1, BaselineQuants.IQ1_S.BitRange); + Assert.Equal((byte)1, BaselineQuants.IQ1_M.BitRange); + Assert.True(BaselineQuants.IQ1_S.RequiresImatrix); + Assert.True(BaselineQuants.IQ1_M.RequiresImatrix); + Assert.True(BaselineQuants.IQ1_S.IsLearningBaseline); + Assert.True(BaselineQuants.IQ1_M.IsLearningBaseline); + Assert.True(BaselineQuants.IQ1_S.IsExplicitGroupCombinationCandidate); + Assert.True(BaselineQuants.IQ1_M.IsExplicitGroupCombinationCandidate); + Assert.False(BaselineQuants.IQ1_S.IsCombinationCarrierCandidate); + Assert.False(BaselineQuants.IQ1_M.IsCombinationCarrierCandidate); + Assert.Same(BaselineQuants.IQ1_S, BaselineQuants.ResolveBuiltInStandardBaseline("IQ1_S")); + Assert.Same(BaselineQuants.IQ1_M, BaselineQuants.ResolveBuiltInStandardBaseline("IQ1_M")); + Assert.Equal("IQ1_S", TensorWeightScheme.FromId(TensorWeightScheme.IQ1_S.UniqueId).Names[0]); + Assert.Equal("IQ1_M", TensorWeightScheme.FromId(TensorWeightScheme.IQ1_M.UniqueId).Names[0]); + } + + [Fact] + public void ResolveBuiltInStandardRoleBaseline_PrefersCanonicalNameOverSharedTensorSchemeAlias() + { + Assert.Same(BaselineQuants.IQ3_S, BaselineQuants.ResolveBuiltInStandardRoleBaseline("IQ3_S")); + Assert.Same(BaselineQuants.IQ3_M, BaselineQuants.ResolveBuiltInStandardRoleBaseline("IQ3_M")); + } + + [Fact] + public void GetPureBaselineCandidates_NoImatrix_ReturnsAllNonImatrixLearningBaselines() + { + var ids = BaselineQuants.GetPureBaselineCandidates(hasUsableImatrix: false) + .Select(x => x.UniqueId) + .ToArray(); + + Assert.Equal( + [ + BaselineQuants.Q8_0.UniqueId, + BaselineQuants.Q6_K.UniqueId, + BaselineQuants.Q5_K.UniqueId, + BaselineQuants.Q4_K_M.UniqueId, + BaselineQuants.IQ4_NL.UniqueId, + BaselineQuants.IQ4_XS.UniqueId, + BaselineQuants.Q5_K_S.UniqueId, + BaselineQuants.Q4_K_S.UniqueId + ], ids); + } + + [Fact] + public void GetCombinationCarrierBaselines_UsesCanonicalQ8Carrier() + { + var ids = BaselineQuants.GetCombinationCarrierBaselines(hasUsableImatrix: false) + .Select(x => x.UniqueId) + .ToArray(); + + Assert.Equal([BaselineQuants.Q8_0.UniqueId], ids); + } + + [Fact] + public void GetGroupCombinationCandidates_NoImatrix_ReturnsAllEligibleFourBitAndHigherBaselines() + { + var ids = BaselineQuants.GetGroupCombinationCandidates(hasUsableImatrix: false, allowHighPrecisionHybrids: false) + .Select(x => x.UniqueId) + .ToArray(); + + Assert.Equal( + [ + BaselineQuants.IQ4_XS.UniqueId, + BaselineQuants.IQ4_NL.UniqueId, + BaselineQuants.Q4_K_S.UniqueId, + BaselineQuants.Q4_K_M.UniqueId, + BaselineQuants.Q5_K_S.UniqueId, + BaselineQuants.Q5_K.UniqueId, + BaselineQuants.Q6_K.UniqueId, + BaselineQuants.Q8_0.UniqueId + ], ids); + + Assert.DoesNotContain(BaselineQuants.IQ3_S.UniqueId, ids); + Assert.DoesNotContain(BaselineQuants.IQ3_XS.UniqueId, ids); + Assert.DoesNotContain(BaselineQuants.IQ3_XXS.UniqueId, ids); + Assert.DoesNotContain(BaselineQuants.IQ2_S.UniqueId, ids); + Assert.DoesNotContain(BaselineQuants.IQ2_XS.UniqueId, ids); + Assert.DoesNotContain(BaselineQuants.IQ2_XXS.UniqueId, ids); + Assert.DoesNotContain(BaselineQuants.BF16_Hybrid.UniqueId, ids); + Assert.DoesNotContain(BaselineQuants.F16_Hybrid.UniqueId, ids); + } + + [Fact] + public void RuntimeSearchSpace_GetActiveCombinationBaselines_ReturnsCanonicalQ8Carrier() + { + RuntimeSearchSpace.ResetForNewModel(); + RuntimeSearchSpace.SetImatrixAvailability(false); + + var ids = RuntimeSearchSpace.GetActiveCombinationBaselines() + .Select(x => x.UniqueId) + .ToArray(); + + Assert.Equal([BaselineQuants.Q8_0.UniqueId], ids); + } + + [Fact] + public void ExplicitCandidateExhaustion_UsesQ8FallbackPolicy() + { + RuntimeSearchSpace.ResetForNewModel(); + RuntimeSearchSpace.SetImatrixAvailability(true); + + RuntimeSearchSpace.BanAllExplicitCombinationCandidatesForGroup(TReg.AttnQ); + + Assert.True(RuntimeSearchSpace.IsGroupExplicitCandidateBanned(TReg.AttnQ)); + Assert.Equal(BaselineQuants.Q8_0.UniqueId, BaselineQuants.GetDefaultExplicitFallbackBaseline().UniqueId); + } + + [Fact] + public void CandidateBanAuthority_DrivesAllowedCandidateSet() + { + RuntimeSearchSpace.ResetForNewModel(); + RuntimeSearchSpace.SetImatrixAvailability(true); + + RuntimeSearchSpace.BanCombinationCandidateForGroup(TReg.AttnQ, BaselineQuants.Q6_K); + var allowed = ComboLogic.GetAllowedCandidateIdsPerGroup(BaselineQuants.Q8_0); + var attnQIndex = TReg.All.OrderBy(x => x.UniqueId).ToList().FindIndex(x => x.UniqueId == TReg.AttnQ.UniqueId); + + Assert.DoesNotContain(BaselineQuants.EncodeTensorConfigGroupSlot(BaselineQuants.Q6_K), allowed[attnQIndex]); + } + + [Fact] + public void ComboLogic_WhenHighPrecisionDisabled_DoesNotInjectBf16OrF16() + { + RuntimeSearchSpace.ResetForNewModel(); + RuntimeSearchSpace.SetImatrixAvailability(true); + RuntimeSearchSpace.AllowHighPrecisionHybrids = false; + + var allowed = ComboLogic.GetAllowedCandidateIdsPerGroup(BaselineQuants.Q8_0); + var attnQIndex = TReg.All.OrderBy(x => x.UniqueId).ToList().FindIndex(x => x.UniqueId == TReg.AttnQ.UniqueId); + + Assert.DoesNotContain(BaselineQuants.EncodeTensorConfigGroupSlot(BaselineQuants.BF16_Hybrid), allowed[attnQIndex]); + Assert.DoesNotContain(BaselineQuants.EncodeTensorConfigGroupSlot(BaselineQuants.F16_Hybrid), allowed[attnQIndex]); + } + + [Fact] + public void ComboLogic_UsesCandidateLevelBannedGroups() + { + RuntimeSearchSpace.ResetForNewModel(); + RuntimeSearchSpace.SetImatrixAvailability(false); + var candidate = BaselineQuants.RegisterCustomExternalBaseline(new BaselineQuants.ExternalBaselineRegistration + { + CanonicalKey = "test:moe-router-banned", + DisplayName = "TEST-Q5-BANNED", + QuantizeBaseArgumentName = "Q5_K", + Repository = "test/repository", + RepositoryFileName = "test-q5.gguf", + OwnerShortName = "test", + BaselineFamilyName = "Q5_K", + TensorScheme = TensorWeightScheme.Q5_K, + AddAsGroupCandidate = true, + BitRange = 5, + BannedGroupIds = [TReg.MoeRouter.UniqueId] + }); + + try + { + var allowed = ComboLogic.GetAllowedCandidateIdsPerGroup(BaselineQuants.Q8_0); + var moeRouterIndex = TReg.All.OrderBy(x => x.UniqueId).ToList().FindIndex(x => x.UniqueId == TReg.MoeRouter.UniqueId); + + Assert.DoesNotContain(BaselineQuants.EncodeTensorConfigGroupSlot(candidate), allowed[moeRouterIndex]); + } + finally + { + BaselineQuants.ResetDynamicCustomBaselines(); + } + } +} diff --git a/tests/MagicQuant.Tests/BenchmarkContractTests.cs b/tests/MagicQuant.Tests/BenchmarkContractTests.cs new file mode 100644 index 0000000..401eb0c --- /dev/null +++ b/tests/MagicQuant.Tests/BenchmarkContractTests.cs @@ -0,0 +1,61 @@ +using MagicQuant.Services; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class BenchmarkContractTests +{ + [Fact] + public void Cpu_and_gpu_benchmark_arguments_preserve_runtime_policy() + { + var cpu = BenchmarkCommands.Bench("bench", "/models/a b.gguf", false, 99, ""); + Assert.Equal(["-m", "/models/a b.gguf", "-p", "8", "-t", "16", "-ngl", "0", "-o", "md"], cpu.Arguments); + var gpu = BenchmarkCommands.Perplexity("ppl", "model", "corpus", true, 32, " --tensor-split 19,23", "logits", true); + Assert.Equal(["-m", "model", "-ngl", "32", "--tensor-split", "19,23", "-t", "4", "-c", "2048", "--file", "corpus", "--kl-divergence-base", "logits", "--kl-divergence"], gpu.Arguments); + Assert.DoesNotContain("--kl-divergence", BenchmarkCommands.Perplexity("ppl", "model", "corpus", false, 32, "", "logits", false).Arguments); + } + + [Theory] + [InlineData("PPL = 12.5 +/- 0.2\nMean KLD: 1.2e-3", 12.5, 0.0012)] + [InlineData("\u001b[32mMean PPL(Q) : 8.1 ± 0.1\u001b[0m\nKL-divergence = 0.02", 8.1, 0.02)] + public void Parses_plain_and_ansi_scientific_notation_logs(string content, double ppl, double kld) + { + WithLog(content, path => + { + var metrics = BenchmarkLogParser.ParsePerplexity(path, false); + Assert.Equal(ppl, metrics.Ppl); + Assert.Equal(kld, metrics.Kld); + }); + } + + [Fact] + public void Missing_kld_is_allowed_only_for_native_reference_logs() + { + WithLog("PPL = 12.5 +/- 0.2", path => + { + Assert.Null(BenchmarkLogParser.ParsePerplexity(path, true).Kld); + Assert.Throws(() => BenchmarkLogParser.ParsePerplexity(path, false)); + }); + WithLog("process failed before measurement", path => + Assert.Throws(() => BenchmarkLogParser.ParsePerplexity(path, true))); + } + + [Fact] + public void Parses_benchmark_table_by_column_name() + { + WithLog("| test | backend | t/s | ngl |\n|---|---|---|---|\n| pp8 | CPU | 123.45 ± 0.1 | 0 |", path => + { + var metrics = BenchmarkLogParser.ParseLlamaBench(path); + Assert.Equal(123.45, metrics.Tps); + Assert.Equal("CPU", metrics.Backend); + Assert.Equal("pp8", metrics.Test); + }); + } + + private static void WithLog(string text, Action test) + { + string path = Path.GetTempFileName(); + try { File.WriteAllText(path, text); test(path); } + finally { File.Delete(path); } + } +} diff --git a/tests/MagicQuant.Tests/BenchmarkCorpusTests.cs b/tests/MagicQuant.Tests/BenchmarkCorpusTests.cs new file mode 100644 index 0000000..5ff80a2 --- /dev/null +++ b/tests/MagicQuant.Tests/BenchmarkCorpusTests.cs @@ -0,0 +1,33 @@ +using MagicQuant.Services; +using Xunit; + +namespace MagicQuant.Tests; + +public class BenchmarkCorpusTests +{ + [Fact] + public void DatasetIds_AreCanonicalNamespacedIds() + { + Assert.Equal("Salesforce/wikitext", BenchmarkService.GeneralPplDatasetId); + Assert.Equal("openai/gsm8k", BenchmarkService.MathPplDatasetId); + } + + [Fact] + public void IsPplCorpusUsable_RequiresTheFullCharacterTarget() + { + string path = Path.GetTempFileName(); + + try + { + File.WriteAllText(path, new string('x', 31)); + Assert.False(BenchmarkService.IsPplCorpusUsable(path, tokenTarget: 8)); + + File.AppendAllText(path, "x"); + Assert.True(BenchmarkService.IsPplCorpusUsable(path, tokenTarget: 8)); + } + finally + { + File.Delete(path); + } + } +} diff --git a/tests/MagicQuant.Tests/BenchmarkGpuPlanningTests.cs b/tests/MagicQuant.Tests/BenchmarkGpuPlanningTests.cs new file mode 100644 index 0000000..65077d6 --- /dev/null +++ b/tests/MagicQuant.Tests/BenchmarkGpuPlanningTests.cs @@ -0,0 +1,182 @@ +using MagicQuant.Services; +using Xunit; + +namespace MagicQuant.Tests; + +public class BenchmarkGpuPlanningTests +{ + private const ulong Q8Size = 29_047_084_736UL; + + [Theory] + [InlineData(29_047_084_736UL, 44)] + [InlineData(21_998_012_096UL, 58)] + [InlineData(17_624_999_616UL, 66)] + public void ResolveNglForModel_ScalesAndClampsAtFullOffload(ulong size, int expected) + { + int result = BenchmarkGpuPlanner.ResolveNglForModel(Q8Size, 44, 66, size); + Assert.Equal(expected, result); + } + + [Fact] + public void EstimateIndependentCrossover_UsesMeasuredPerSlotScaling() + { + var slots = new[] + { + Slot(0, 44, (35, 6.59), (41, 5.83), (44, 5.45)), + Slot(1, 57, (48, 4.38), (56, 3.28), (57, 3.17)) + }; + + ulong crossover = BenchmarkGpuPlanner.EstimateIndependentCrossoverBytes( + Q8Size, + maxOffloadNgl: 66, + sharedSecondsPerPass: 1.60, + slots); + + double gib = crossover / 1024d / 1024d / 1024d; + Assert.InRange(gib, 22d, 27d); + } + + [Theory] + [InlineData(true, 20_000_000_000UL, true)] + [InlineData(false, 20_000_000_000UL, false)] + [InlineData(true, 26_000_000_000UL, false)] + public void ShouldUseIndependentTopology_RequiresConcurrentBatchIntent( + bool allowIndependentTopology, + ulong modelSizeBytes, + bool expected) + { + bool result = BenchmarkGpuPlanner.ShouldUseIndependentTopology( + modelSizeBytes, + independentMaxModelSizeBytes: 25_000_000_000UL, + independentSlotCount: 2, + allowIndependentTopology); + + Assert.Equal(expected, result); + } + + [Fact] + public void RankIndependentSlots_NearFullCandidateUsesWeakerDevice() + { + var slots = new[] + { + Slot(0, 44, (35, 6.5), (44, 5.4)), + Slot(1, 57, (48, 4.3), (57, 3.1)) + }; + + var ranked = BenchmarkGpuPlanner.RankIndependentSlotsForModel( + slots, + Q8Size, + maxOffloadNgl: 66, + modelSizeBytes: 19_500_000_000UL); + + Assert.Equal(0, ranked[0].DeviceIndices[0]); + Assert.Equal(65, BenchmarkGpuPlanner.ResolveNglForModel(Q8Size, 44, 66, 19_500_000_000UL)); + } + + [Fact] + public void RankIndependentSlots_LargeCandidateUsesStrongerDevice() + { + var slots = new[] + { + Slot(0, 44, (35, 6.5), (44, 5.4)), + Slot(1, 57, (48, 4.3), (57, 3.1)) + }; + + var ranked = BenchmarkGpuPlanner.RankIndependentSlotsForModel( + slots, + Q8Size, + maxOffloadNgl: 66, + modelSizeBytes: 22_900_000_000UL); + + Assert.Equal(1, ranked[0].DeviceIndices[0]); + Assert.True( + BenchmarkGpuPlanner.ResolveNglForModel(Q8Size, 44, 66, 22_900_000_000UL) < + BenchmarkGpuPlanner.ResolveNglForModel(Q8Size, 57, 66, 22_900_000_000UL)); + } + + [Fact] + public async Task ResourceScheduler_UsesCandidateSpecificSlotRanking() + { + var scheduler = new GpuResourceScheduler(); + var slots = new[] + { + Slot(0, 44, (35, 6.5), (44, 5.4)), + Slot(1, 57, (48, 4.3), (57, 3.1)) + }; + var smaller = BenchmarkGpuPlanner.RankIndependentSlotsForModel( + slots, Q8Size, 66, 19_500_000_000UL); + var larger = BenchmarkGpuPlanner.RankIndependentSlotsForModel( + slots, Q8Size, 66, 22_900_000_000UL); + + await using var first = await scheduler.AcquireAsync(smaller); + await using var second = await scheduler.AcquireAsync(larger); + + Assert.Equal(0, first.Slot.DeviceIndices[0]); + Assert.Equal(1, second.Slot.DeviceIndices[0]); + } + + [Fact] + public async Task ResourceScheduler_ReservesDisjointSingleGpuSlotsConcurrently() + { + var scheduler = new GpuResourceScheduler(); + var slots = new[] { Slot(0, 44, (35, 6.5), (44, 5.4)), Slot(1, 57, (48, 4.3), (57, 3.1)) }; + + await using var first = await scheduler.AcquireAsync(slots); + await using var second = await scheduler.AcquireAsync(slots); + + Assert.NotEqual(first.Slot.DeviceIndices[0], second.Slot.DeviceIndices[0]); + } + + [Fact] + public async Task ResourceScheduler_SharedSlotWaitsUntilAllDevicesAreFree() + { + var scheduler = new GpuResourceScheduler(); + var singleSlots = new[] { Slot(0, 44, (35, 6.5), (44, 5.4)), Slot(1, 57, (48, 4.3), (57, 3.1)) }; + var sharedSlots = new[] { new BenchmarkSlot(2, "shared", [0, 1], 66, []) }; + + var first = await scheduler.AcquireAsync(singleSlots); + var sharedTask = scheduler.AcquireAsync(sharedSlots).AsTask(); + + Assert.False(sharedTask.IsCompleted); + await first.DisposeAsync(); + + await using var shared = await sharedTask.WaitAsync(TimeSpan.FromSeconds(2)); + Assert.Equal([0, 1], shared.Slot.DeviceIndices); + } + + [Fact] + public void TopologyCacheCodec_RoundTripsMeasuredProfiles() + { + var independentSlots = new[] + { + Slot(0, 44, (35, 6.59), (44, 5.45)), + Slot(1, 57, (48, 4.38), (57, 3.17)) + }; + var shared = new BenchmarkTopologyProfile( + "shared", [new BenchmarkSlot(2, "shared", [0, 1], 66, [])], 0.1, 1.6); + var independent = new BenchmarkTopologyProfile("independent", independentSlots, 0.08, 0); + + string json = BenchmarkTopologyCacheCodec.Serialize(66, 26_000_000_000UL, shared, independent); + bool ok = BenchmarkTopologyCacheCodec.TryDeserialize( + json, out int maxNgl, out ulong crossover, out var loadedShared, out var loadedIndependent); + + Assert.True(ok); + Assert.Equal(66, maxNgl); + Assert.Equal(26_000_000_000UL, crossover); + Assert.Equal([0, 1], loadedShared!.Slots.Single().DeviceIndices); + Assert.Equal([44, 57], loadedIndependent!.Slots.Select(x => x.Q8StableNgl).ToArray()); + } + + private static BenchmarkSlot Slot( + int device, + int stableNgl, + params (int Ngl, double Seconds)[] samples) + => new( + SlotId: device, + ProfileName: "independent", + DeviceIndices: [device], + Q8StableNgl: stableNgl, + ProbeSamples: samples + .Select(x => new GpuProbeSample(x.Ngl, true, x.Seconds, x.Seconds * 3d)) + .ToArray()); +} diff --git a/tests/MagicQuant.Tests/CliArgumentParsingTests.cs b/tests/MagicQuant.Tests/CliArgumentParsingTests.cs new file mode 100644 index 0000000..705cc7e --- /dev/null +++ b/tests/MagicQuant.Tests/CliArgumentParsingTests.cs @@ -0,0 +1,33 @@ +using MagicQuant.Helpers; +using Xunit; + +namespace MagicQuant.Tests; + +public class CliArgumentParsingTests +{ + [Fact] + public void ArgvParser_PreservesHyphensInOrdinaryShellValues() + { + var parsed = CliHelpers.ParseArguments([ + "--config", "/repo/MagicQuant-Pipeline/config.dev.yaml", + "--architecture-family", "Qwen3.8-27B", + "--recheck-hardware-probe" + ]); + + Assert.Equal("/repo/MagicQuant-Pipeline/config.dev.yaml", parsed[0].Value); + Assert.Equal("Qwen3.8-27B", parsed[1].Value); + Assert.Equal(string.Empty, parsed[2].Value); + } + + [Fact] + public void ArgvParser_SupportsEqualsAndLegacyLiteralQuotes() + { + var parsed = CliHelpers.ParseArguments([ + "--model-dir=/models/Qwen-27B", + "--output-dir", "\"/models/Agent-Run\"" + ]); + + Assert.Equal("/models/Qwen-27B", parsed[0].Value); + Assert.Equal("/models/Agent-Run", parsed[1].Value); + } +} diff --git a/tests/MagicQuant.Tests/CliOptionValidationTests.cs b/tests/MagicQuant.Tests/CliOptionValidationTests.cs new file mode 100644 index 0000000..0cb3236 --- /dev/null +++ b/tests/MagicQuant.Tests/CliOptionValidationTests.cs @@ -0,0 +1,30 @@ +using MagicQuant.Configuration; +using MagicQuant.Models; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class CliOptionValidationTests +{ + [Theory] + [InlineData("modle-dir", "/model")] + [InlineData("model-dir", "")] + [InlineData("use-imatrix", "false")] + [InlineData("config", "")] + [InlineData("prediction-minimum-fit-rows", "one")] + [InlineData("prediction-minimum-fit-rows", "1")] + [InlineData("prediction-default-bit-stress-threshold", "NaN")] + [InlineData("selection-interior-window-fractions", "0.3,broken")] + [InlineData("selection-diversify-validation-candidates", "maybe")] + public void Rejects_typos_missing_values_and_ambiguous_flags(string name, string value) + { + Assert.Throws(() => CliOptionValidator.Validate([new CliArg { Name = name, Value = value }])); + } + + [Fact] + public void Duplicate_options_are_not_silently_resolved_by_order() + { + Assert.Throws(() => CliOptionValidator.Validate( + [new CliArg { Name = "model-dir", Value = "one" }, new CliArg { Name = "MODEL-DIR", Value = "two" }])); + } +} diff --git a/tests/MagicQuant.Tests/CliStartupTests.cs b/tests/MagicQuant.Tests/CliStartupTests.cs new file mode 100644 index 0000000..28093e2 --- /dev/null +++ b/tests/MagicQuant.Tests/CliStartupTests.cs @@ -0,0 +1,149 @@ +using System.Diagnostics; +using MagicQuant.Commands; +using Xunit; + +namespace MagicQuant.Tests; + +/// Exercise the real executable so startup cannot hide side effects behind command help. +public sealed class CliStartupTests +{ + [Theory] + [InlineData("")] + [InlineData("help")] + [InlineData("--help")] + [InlineData("-h")] + [InlineData("init-config")] + [InlineData("pipeline")] + [InlineData("evolution")] + [InlineData("build-hybrids")] + [InlineData("clone-repository-quants")] + [InlineData("validate-predictions")] + [InlineData("initialize-llama-cpp")] + public async Task Help_succeeds_without_config_or_runtime_artifacts(string command) + { + string[] arguments = command switch + { + "" => [], + "help" or "--help" or "-h" => [command], + _ => [command, "--help", "--config", "does-not-exist.yaml"] + }; + var result = await RunAsync(arguments); + Assert.True(result.ExitCode == 0, result.Output); + Assert.DoesNotContain("Using config:", result.Output); + Assert.DoesNotContain("Checking environment", result.Output); + Assert.Empty(result.CreatedFiles); + } + + [Fact] + public async Task Unknown_command_is_escaped_and_returns_usage_error() + { + var result = await RunAsync(["[invalid]"]); + Assert.Equal(2, result.ExitCode); + Assert.Contains("does not exist", result.Output); + Assert.Empty(result.CreatedFiles); + } + + [Fact] + public async Task Missing_config_returns_failure_before_setup() + { + var result = await RunAsync(["pipeline", "--config", "missing.yaml"]); + Assert.Equal(1, result.ExitCode); + Assert.Contains("config file was not found", result.Output); + Assert.Empty(result.CreatedFiles); + } + + [Fact] + public void Historical_alias_uses_the_same_pipeline_implementation() + { + var commands = CommandCatalog.Create(); + Assert.IsType(commands["pipeline"].Factory()); + Assert.IsType(commands["EVOLUTION"].Factory()); + Assert.IsAssignableFrom(new Evolution()); + } + + [Fact] + public async Task Invalid_model_fails_before_config_application_or_dependency_setup() + { + var result = await RunAsync(["pipeline", "--config", "bad.yaml"], directory => + File.WriteAllText(Path.Combine(directory, "bad.yaml"), "paths:\n magic_quant_root: runtime-must-not-exist\n model_dir: missing-model\n")); + Assert.Equal(1, result.ExitCode); + Assert.DoesNotContain("Using config:", result.Output); + Assert.DoesNotContain("Checking environment", result.Output); + Assert.Single(result.CreatedFiles); + } + + [Fact] + public async Task Check_config_does_not_initialize_or_clean_runtime_state() + { + var result = await RunAsync(["initialize-llama-cpp", "--config", "check.yaml", "--check-config", "--strict-config"], directory => + File.WriteAllText(Path.Combine(directory, "check.yaml"), "paths:\n magic_quant_root: runtime-must-not-exist\n")); + Assert.Equal(0, result.ExitCode); + Assert.Contains("No runtime setup was performed", result.Output); + Assert.Single(result.CreatedFiles); + } + + [Fact] + public async Task Init_config_copies_the_packaged_profile_without_runtime_setup() + { + var result = await RunAsync(["init-config", "--output", "my config.yaml"]); + Assert.Equal(0, result.ExitCode); + Assert.Single(result.CreatedFiles); + Assert.Contains("Created", result.Output); + } + + [Fact] + public async Task Init_config_refuses_to_overwrite_an_existing_file() + { + var result = await RunAsync(["init-config", "--output", "existing.yaml"], directory => + File.WriteAllText(Path.Combine(directory, "existing.yaml"), "user-owned")); + Assert.Equal(1, result.ExitCode); + Assert.Single(result.CreatedFiles); + } + + [Fact] + public async Task Init_config_rejects_a_positional_filename_without_writing_a_default() + { + var result = await RunAsync(["init-config", "unexpected.yaml"]); + Assert.Equal(1, result.ExitCode); + Assert.Empty(result.CreatedFiles); + } + + private static async Task<(int ExitCode, string Output, string[] CreatedFiles)> RunAsync(string[] args, Action? setup = null) + { + string directory = Path.Combine(Path.GetTempPath(), $"mq-cli-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directory); + setup?.Invoke(directory); + try + { + var start = new ProcessStartInfo("dotnet") + { + WorkingDirectory = directory, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + start.ArgumentList.Add(typeof(QuantizationPipeline).Assembly.Location); + foreach (string arg in args) + start.ArgumentList.Add(arg); + using var process = Process.Start(start)!; + Task stdout = process.StandardOutput.ReadToEndAsync(); + Task stderr = process.StandardError.ReadToEndAsync(); + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + try + { + await process.WaitForExitAsync(timeout.Token); + } + catch (OperationCanceledException) + { + process.Kill(entireProcessTree: true); + await process.WaitForExitAsync(); + throw; + } + return (process.ExitCode, await stdout + await stderr, Directory.GetFileSystemEntries(directory)); + } + finally + { + Directory.Delete(directory, recursive: true); + } + } +} diff --git a/tests/MagicQuant.Tests/CombinationDatabasePathTests.cs b/tests/MagicQuant.Tests/CombinationDatabasePathTests.cs new file mode 100644 index 0000000..186c984 --- /dev/null +++ b/tests/MagicQuant.Tests/CombinationDatabasePathTests.cs @@ -0,0 +1,40 @@ +using MagicQuant.Helpers; +using MagicQuant.Services; +using MQ.DB; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class CombinationDatabasePathTests +{ + [Theory] + [InlineData(false, null, false, "no-imatrix_hp-off")] + [InlineData(true, "abc123", false, "abc123_hp-off")] + [InlineData(true, null, true, "imatrix-unknown_hp-on")] + public void Reader_path_preserves_context_filename(bool imatrix, string? hash, bool highPrecision, string suffix) + { + var previous = (Cache.ModelMagicQuantDirectory, Cache.MagicQuantDirectory, Cache.CurrentModelId, + Cache.IsImatrixAvailable, Cache.ActiveImatrixIdentityHash, RuntimeSearchSpace.AllowHighPrecisionHybrids); + try + { + Cache.ModelMagicQuantDirectory = Path.Combine(Path.GetTempPath(), "model", "MagicQuant"); + Cache.MagicQuantDirectory = Path.Combine(Path.GetTempPath(), "shared"); + Cache.CurrentModelId = "model123"; + Cache.IsImatrixAvailable = imatrix; + Cache.ActiveImatrixIdentityHash = hash; + RuntimeSearchSpace.AllowHighPrecisionHybrids = highPrecision; + string expected = Path.Combine(Cache.ModelMagicQuantDirectory, $"MagicQuant_Combinations_model123_{suffix}.duckdb"); + Assert.Equal(expected, CombinationDatabasePathService.GetPath()); + Assert.Equal(expected, new RemainingCombinationStore().GetDatabaseFilePath()); + Cache.ModelMagicQuantDirectory = null; + Assert.Equal(Cache.MagicQuantDirectory, CombinationDatabasePathService.GetDirectory()); + Cache.MagicQuantDirectory = null; + Assert.Throws(() => CombinationDatabasePathService.GetPath()); + } + finally + { + (Cache.ModelMagicQuantDirectory, Cache.MagicQuantDirectory, Cache.CurrentModelId, + Cache.IsImatrixAvailable, Cache.ActiveImatrixIdentityHash, RuntimeSearchSpace.AllowHighPrecisionHybrids) = previous; + } + } +} diff --git a/tests/MagicQuant.Tests/ConfigurationContractTests.cs b/tests/MagicQuant.Tests/ConfigurationContractTests.cs new file mode 100644 index 0000000..f76ce9d --- /dev/null +++ b/tests/MagicQuant.Tests/ConfigurationContractTests.cs @@ -0,0 +1,105 @@ +using MagicQuant.Configuration; +using MagicQuant.Models; +using Xunit; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace MagicQuant.Tests; + +public sealed class ConfigurationContractTests +{ + [Fact] + public void Default_config_selection_does_not_depend_on_build_configuration() + { + Assert.Equal(Path.Combine(AppContext.BaseDirectory, "config.default.yaml"), MagicQuantYamlLoader.ResolveConfigPath([])); + } + + [Fact] + public void Explicit_config_path_is_relative_to_working_directory() + { + Assert.Equal(Path.GetFullPath("configs/my campaign.yaml"), MagicQuantYamlLoader.ResolveConfigPath( + [new CliArg { Name = "CONFIG", Value = "configs/my campaign.yaml" }])); + } + + [Theory] + [InlineData("src/MagicQuant/config.default.yaml")] + [InlineData("examples/pipeline.yaml")] + [InlineData("examples/clone.yaml")] + [InlineData("examples/pipeline-external.yaml")] + public void Distributed_configs_have_no_unknown_keys(string relativePath) + { + string root = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "../../../../..")); + var config = new DeserializerBuilder() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .Build() + .Deserialize(File.ReadAllText(Path.Combine(root, relativePath))); + Assert.NotNull(config.Paths); + Assert.NotNull(config.CandidateSelection); + } + [Theory] + [InlineData("README.md")] + [InlineData("docs/best-practices.md")] + public void Onboarding_yaml_examples_match_the_configuration_contract(string relativePath) + { + string root = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "../../../../..")); + string markdown = File.ReadAllText(Path.Combine(root, relativePath)); + var snippets = System.Text.RegularExpressions.Regex.Matches(markdown, @"```yaml\r?\n(.*?)```", + System.Text.RegularExpressions.RegexOptions.Singleline); + Assert.NotEmpty(snippets); + foreach (System.Text.RegularExpressions.Match snippet in snippets) + { + string yaml = snippet.Groups[1].Value; + Assert.Empty(YamlConfigurationDiagnostics.Inspect(yaml)); + var config = new DeserializerBuilder().WithNamingConvention(UnderscoredNamingConvention.Instance) + .Build().Deserialize(yaml); + ConfigurationShapeValidator.Validate(config); + } + } + + [Fact] + public void Starter_and_typed_defaults_do_not_select_a_model_or_external_provider() + { + var yaml = File.ReadAllText(Path.Combine(AppContext.BaseDirectory, "config.default.yaml")); + var starter = new DeserializerBuilder().WithNamingConvention(UnderscoredNamingConvention.Instance) + .Build().Deserialize(yaml); + Assert.DoesNotContain("Qwen", yaml, StringComparison.OrdinalIgnoreCase); + foreach (var config in new[] { starter, MagicQuantYamlConfig.CreateDefault() }) + { + Assert.True(string.IsNullOrWhiteSpace(config.Paths.ModelDir)); + Assert.True(string.IsNullOrWhiteSpace(config.Paths.MagicQuantRoot)); + Assert.True(string.IsNullOrWhiteSpace(config.Identity.ArchitectureFamilyName)); + Assert.True(string.IsNullOrWhiteSpace(config.Readme.TitleModelNameOverride)); + Assert.True(string.IsNullOrWhiteSpace(config.Imatrix.DatasetRepo)); + Assert.Empty(config.Paths.ScratchRoots); + Assert.Empty(config.Hardware.GpuMemoryLimitsGb); + Assert.Empty(config.Baselines.CustomRepositories); + Assert.False(config.Output.ExportExternalLearnedBaselines); + Assert.False(config.Learning.ForceRelearnArchitectureFamily); + Assert.True(config.Learning.ConfirmTensorGroupProfile); + Assert.Equal(new[] { "Q6_K", "Q5_K" }, config.AnomalyDetection.ConfirmedAnomalyExpansion.AllowedCandidateQuants); + Assert.False(config.Readme.Frontmatter.ContainsKey("license")); + Assert.False(config.Readme.Frontmatter.ContainsKey("base_model")); + } + } + + [Fact] + public void Legacy_inactive_yaml_remains_compatible_with_current_selection_settings() + { + const string yaml = """ + evolution: + max_survival_rounds: 100 + survival: + max_selected_choices_per_bucket: 50 + brain_layers: [embeddings] + candidate_selection: + max_fallback_attempts_per_anchor: 7 + """; + var config = new DeserializerBuilder() + .IgnoreUnmatchedProperties() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .Build() + .Deserialize(yaml); + Assert.Equal(7, config.CandidateSelection.MaxFallbackAttemptsPerAnchor); + } + +} diff --git a/tests/MagicQuant.Tests/ConfigurationReadTests.cs b/tests/MagicQuant.Tests/ConfigurationReadTests.cs new file mode 100644 index 0000000..4023973 --- /dev/null +++ b/tests/MagicQuant.Tests/ConfigurationReadTests.cs @@ -0,0 +1,46 @@ +using System.Globalization; +using MagicQuant.Configuration; +using MagicQuant.Models; +using MQ.DB; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class ConfigurationReadTests +{ + [Fact] + public void Read_does_not_mutate_runtime_state_and_cli_decimal_is_culture_independent() + { + string file = Path.GetTempFileName(); + var oldCulture = CultureInfo.CurrentCulture; + string? oldRoot = Cache.MagicQuantDirectory; + var oldConfig = Config.Current; + try + { + File.WriteAllText(file, "prediction:\n default_bit_stress_threshold: 5.0\n"); + CultureInfo.CurrentCulture = CultureInfo.GetCultureInfo("de-DE"); + var loaded = MagicQuantYamlLoader.Read([ + new CliArg { Name = "config", Value = file }, + new CliArg { Name = "prediction-default-bit-stress-threshold", Value = "7.5" } + ]); + Assert.Equal(7.5, loaded.Settings.Prediction.DefaultBitStressThreshold); + Assert.Same(oldConfig, Config.Current); + Assert.Equal(oldRoot, Cache.MagicQuantDirectory); + } + finally { CultureInfo.CurrentCulture = oldCulture; File.Delete(file); } + } + + [Fact] + public void Strict_config_rejects_typos_that_normal_mode_reports() + { + string file = Path.GetTempFileName(); + try + { + File.WriteAllText(file, "paths:\n modle_dir: /model\n"); + var arg = new CliArg { Name = "config", Value = file }; + Assert.Single(MagicQuantYamlLoader.Read([arg]).Warnings); + Assert.Throws(() => MagicQuantYamlLoader.Read([arg, new CliArg { Name = "strict-config", Value = "" }])); + } + finally { File.Delete(file); } + } +} diff --git a/tests/MagicQuant.Tests/ExternalBaselineCacheCleanupServiceTests.cs b/tests/MagicQuant.Tests/ExternalBaselineCacheCleanupServiceTests.cs new file mode 100644 index 0000000..13fbd27 --- /dev/null +++ b/tests/MagicQuant.Tests/ExternalBaselineCacheCleanupServiceTests.cs @@ -0,0 +1,119 @@ +using MagicQuant.Services; +using MagicQuant.Configuration; +using MQ.DB; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class ExternalBaselineCacheCleanupServiceTests +{ + [Fact] + public async Task CleanupStaleArtifactsAsync_HardDeletesCacheTree() + { + string temp = Path.Combine(Path.GetTempPath(), "mq-external-cache-test-" + Guid.NewGuid().ToString("N")); + string modelRoot = Path.Combine(temp, "MagicQuant"); + string cacheRoot = Path.Combine(modelRoot, "ExternalBaselines"); + + Directory.CreateDirectory(Path.Combine(cacheRoot, ".cache", "huggingface")); + await File.WriteAllTextAsync(Path.Combine(cacheRoot, "baseline.gguf"), "GGUF"); + await File.WriteAllTextAsync(Path.Combine(cacheRoot, ".cache", "huggingface", "metadata"), "x"); + + string? priorModelMagicQuantDirectory = Cache.ModelMagicQuantDirectory; + string? priorExternalBaselineCacheDirectory = Cache.ExternalBaselineCacheDirectory; + var priorConfig = Config.Current; + + try + { + Config.Load(MagicQuantYamlConfig.CreateDefault()); + Cache.ModelMagicQuantDirectory = modelRoot; + Cache.ExternalBaselineCacheDirectory = cacheRoot; + + bool cleaned = await new ExternalBaselineCacheCleanupService().CleanupStaleArtifactsAsync(); + + Assert.True(cleaned); + Assert.False(Directory.Exists(cacheRoot)); + } + finally + { + Cache.ModelMagicQuantDirectory = priorModelMagicQuantDirectory; + Cache.ExternalBaselineCacheDirectory = priorExternalBaselineCacheDirectory; + Config.Load(priorConfig); + + if (Directory.Exists(temp)) + Directory.Delete(temp, recursive: true); + } + } + + [Fact] + public async Task CleanupStaleArtifactsAsync_PreservesCompletedAndInProgressResumableDownloads() + { + string temp = Path.Combine(Path.GetTempPath(), "mq-external-cache-resume-test-" + Guid.NewGuid().ToString("N")); + string modelRoot = Path.Combine(temp, "MagicQuant"); + string cacheRoot = Path.Combine(modelRoot, "ExternalBaselines"); + string hubCache = Path.Combine(cacheRoot, ".cache", "huggingface"); + string completed = Path.Combine(cacheRoot, "baseline.gguf"); + string incomplete = Path.Combine(hubCache, "baseline.gguf.incomplete"); + string transient = Path.Combine(cacheRoot, "baseline.gguf.partial.interrupted"); + + Directory.CreateDirectory(hubCache); + await File.WriteAllTextAsync(completed, "GGUF-complete"); + await File.WriteAllTextAsync(incomplete, "partial-download"); + await File.WriteAllTextAsync(transient, "partial-copy"); + + string? priorModelMagicQuantDirectory = Cache.ModelMagicQuantDirectory; + string? priorExternalBaselineCacheDirectory = Cache.ExternalBaselineCacheDirectory; + var priorConfig = Config.Current; + + try + { + var config = MagicQuantYamlConfig.CreateDefault(); + config.Baselines.CustomRepositories.Add(new CustomBaselineRepositoryConfig + { + RepoId = "owner/model", + Enabled = true, + ResumeOrRetryDownloads = true + }); + Config.Load(config); + Cache.ModelMagicQuantDirectory = modelRoot; + Cache.ExternalBaselineCacheDirectory = cacheRoot; + + bool cleaned = await new ExternalBaselineCacheCleanupService().CleanupStaleArtifactsAsync(); + + Assert.True(cleaned); + Assert.True(File.Exists(completed)); + Assert.True(File.Exists(incomplete)); + Assert.False(File.Exists(transient)); + } + finally + { + Cache.ModelMagicQuantDirectory = priorModelMagicQuantDirectory; + Cache.ExternalBaselineCacheDirectory = priorExternalBaselineCacheDirectory; + Config.Load(priorConfig); + + if (Directory.Exists(temp)) + Directory.Delete(temp, recursive: true); + } + } + + [Fact] + public void ValidateCleanupRoot_RejectsPathOutsideModelWorkDirectory() + { + string temp = Path.Combine(Path.GetTempPath(), "mq-external-cache-test-" + Guid.NewGuid().ToString("N")); + string modelRoot = Path.Combine(temp, "model", "MagicQuant"); + string outsideRoot = Path.Combine(temp, "outside"); + + var ex = Assert.Throws(() => + ExternalBaselineCacheCleanupService.ValidateCleanupRoot(outsideRoot, modelRoot)); + + Assert.Contains("unsafe external baseline cache path", ex.Message, StringComparison.OrdinalIgnoreCase); + } + + [Fact] + public void ValidateCleanupRoot_RejectsModelWorkDirectoryItself() + { + string modelRoot = Path.Combine(Path.GetTempPath(), "mq-external-cache-test-" + Guid.NewGuid().ToString("N")); + + Assert.Throws(() => + ExternalBaselineCacheCleanupService.ValidateCleanupRoot(modelRoot, modelRoot)); + } +} diff --git a/tests/MagicQuant.Tests/ExternalBaselineTensorParityTests.cs b/tests/MagicQuant.Tests/ExternalBaselineTensorParityTests.cs new file mode 100644 index 0000000..a8f2734 --- /dev/null +++ b/tests/MagicQuant.Tests/ExternalBaselineTensorParityTests.cs @@ -0,0 +1,120 @@ +using MagicQuant.Services; +using Xunit; + +namespace MagicQuant.Tests; + +public class ExternalBaselineTensorParityTests +{ + [Fact] + public void ExactTensorSet_IsAccepted() + { + var native = Metadata(nextnLayers: 1, "token_embd.weight", "blk.0.attn_q.weight", "blk.64.nextn.eh_proj.weight"); + var external = Metadata(nextnLayers: 1, "token_embd.weight", "blk.0.attn_q.weight", "blk.64.nextn.eh_proj.weight"); + + var result = ExternalBaselineTensorParity.ValidateOrThrow(native, external); + + Assert.Empty(result.InheritedOptionalTensorNames); + Assert.Equal(0, result.OmittedNextnLayerCount); + } + + [Fact] + public void MetadataDeclaredTrailingMtpOmission_IsAccepted() + { + var native = Metadata( + nextnLayers: 1, + "token_embd.weight", + "blk.0.attn_q.weight", + "blk.63.ffn_down.weight", + "blk.64.attn_q.weight", + "blk.64.nextn.eh_proj.weight"); + var external = Metadata( + nextnLayers: 0, + "token_embd.weight", + "blk.0.attn_q.weight", + "blk.63.ffn_down.weight"); + + var result = ExternalBaselineTensorParity.ValidateOrThrow(native, external); + + Assert.Equal(1, result.OmittedNextnLayerCount); + Assert.Equal( + ["blk.64.attn_q.weight", "blk.64.nextn.eh_proj.weight"], + result.InheritedOptionalTensorNames); + } + + [Fact] + public void MissingModelTrunkTensor_IsRejected() + { + var native = Metadata( + nextnLayers: 1, + "token_embd.weight", + "blk.0.attn_q.weight", + "blk.64.nextn.eh_proj.weight"); + var external = Metadata(nextnLayers: 0, "token_embd.weight"); + + var ex = Assert.Throws( + () => ExternalBaselineTensorParity.ValidateOrThrow(native, external)); + + Assert.Contains("blk.0.attn_q.weight", ex.Message, StringComparison.Ordinal); + Assert.Contains("model-trunk tensors", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void MissingMtpTensorWithoutReducedMetadata_IsRejected() + { + var native = Metadata( + nextnLayers: 1, + "token_embd.weight", + "blk.64.nextn.eh_proj.weight"); + var external = Metadata(nextnLayers: 1, "token_embd.weight"); + + Assert.Throws( + () => ExternalBaselineTensorParity.ValidateOrThrow(native, external)); + } + + [Fact] + public void PartialOmittedMtpBlock_IsRejected() + { + var native = Metadata( + nextnLayers: 1, + "token_embd.weight", + "blk.64.attn_q.weight", + "blk.64.nextn.eh_proj.weight"); + var external = Metadata( + nextnLayers: 0, + "token_embd.weight", + "blk.64.attn_q.weight"); + + Assert.Throws( + () => ExternalBaselineTensorParity.ValidateOrThrow(native, external)); + } + + [Fact] + public void UnexpectedTensor_IsRejectedEvenWhenMtpIsOmitted() + { + var native = Metadata( + nextnLayers: 1, + "token_embd.weight", + "blk.64.nextn.eh_proj.weight"); + var external = Metadata( + nextnLayers: 0, + "token_embd.weight", + "unexpected.weight"); + + var ex = Assert.Throws( + () => ExternalBaselineTensorParity.ValidateOrThrow(native, external)); + + Assert.Contains("unexpected.weight", ex.Message, StringComparison.Ordinal); + } + + private static GgufTensorReadResult Metadata(int nextnLayers, params string[] tensorNames) + { + return new GgufTensorReadResult + { + Architecture = "qwen35", + BlockCount = 64 + nextnLayers, + NextnPredictLayers = nextnLayers, + TensorNames = tensorNames.ToList(), + TensorTypes = tensorNames.ToDictionary(x => x, _ => "BF16", StringComparer.Ordinal) + }; + } +} diff --git a/tests/MagicQuant.Tests/HardwareInitializationTests.cs b/tests/MagicQuant.Tests/HardwareInitializationTests.cs new file mode 100644 index 0000000..c1037b0 --- /dev/null +++ b/tests/MagicQuant.Tests/HardwareInitializationTests.cs @@ -0,0 +1,93 @@ +using MagicQuant.Commands; +using MagicQuant.Configuration; +using MagicQuant.Models; +using MQ.DB; +using Xunit; + +namespace MagicQuant.Tests; + +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class HardwareInitializationCollection +{ + public const string Name = "Hardware initialization"; +} + +[Collection(HardwareInitializationCollection.Name)] +public sealed class HardwareInitializationTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Custom_environment_validation_populates_system_info(bool useYamlPaths) + { + string testRoot = Path.Combine( + Path.GetTempPath(), + $"magicquant-hardware-init-{Guid.NewGuid():N}"); + string llamaRoot = Path.Combine(testRoot, "llama.cpp"); + string llamaBin = Path.Combine(llamaRoot, "build", "bin"); + string convertScript = Path.Combine(llamaRoot, "convert_hf_to_gguf.py"); + var previous = Cache.SysInfo; + var previousPaths = (Cache.LlamaRoot, Cache.LlamaBin, Cache.ConvertScript); + var previousConfig = Config.Current; + + try + { + Directory.CreateDirectory(llamaBin); + await File.WriteAllTextAsync(convertScript, "# test"); + Cache.SysInfo = null; + + var config = MagicQuantYamlConfig.CreateDefault(); + Config.Load(config); + List args = [new() { Name = "validate", Value = string.Empty }]; + if (useYamlPaths) + { + config.Paths.LlamaRoot = llamaRoot; + config.Paths.LlamaBin = llamaBin; + config.Paths.ConvertScript = convertScript; + } + else + { + args.AddRange([ + new CliArg { Name = "llama-root", Value = llamaRoot }, + new CliArg { Name = "llama-bin", Value = llamaBin }, + new CliArg { Name = "convert-script", Value = convertScript } + ]); + } + await new InitializeLlamaCpp().Run(args); + + Assert.Equal(llamaRoot, Cache.LlamaRoot); + Assert.Equal(llamaBin, Cache.LlamaBin); + Assert.Equal(convertScript, Cache.ConvertScript); + + Assert.NotNull(Cache.SysInfo); + Assert.True(Cache.SysInfo.ThreadCount > 0); + Assert.True(Cache.SysInfo.RamGb > 0); + } + finally + { + Cache.SysInfo = previous; + (Cache.LlamaRoot, Cache.LlamaBin, Cache.ConvertScript) = previousPaths; + Config.Load(previousConfig); + if (Directory.Exists(testRoot)) + Directory.Delete(testRoot, recursive: true); + } + } + [Fact] + public async Task Partial_custom_paths_fail_before_setup() + { + var previous = Config.Current; + try + { + Config.Load(MagicQuantYamlConfig.CreateDefault()); + await Assert.ThrowsAsync(() => new InitializeLlamaCpp().Run( + [new CliArg { Name = "llama-root", Value = "/missing/llama.cpp" }])); + await Assert.ThrowsAsync(() => new InitializeLlamaCpp().Run( + [new CliArg { Name = "llama-bin", Value = "/missing/bin" }])); + } + finally + { + Config.Load(previous); + } + } + +} diff --git a/tests/MagicQuant.Tests/HuggingFaceBaselineCacheTests.cs b/tests/MagicQuant.Tests/HuggingFaceBaselineCacheTests.cs new file mode 100644 index 0000000..942ab5e --- /dev/null +++ b/tests/MagicQuant.Tests/HuggingFaceBaselineCacheTests.cs @@ -0,0 +1,112 @@ +using MagicQuant.Services; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class HuggingFaceBaselineCacheTests +{ + [Fact] + public void MatchingGgufLengthAndTimestamp_IsReusable() + { + using var files = new TemporaryFiles(); + File.WriteAllBytes(files.Source, "GGUF-source-payload"u8.ToArray()); + File.Copy(files.Source, files.Destination); + File.SetLastWriteTimeUtc(files.Destination, File.GetLastWriteTimeUtc(files.Source)); + + Assert.True(HuggingFaceBaselineService.CanReuseDownloadedFile(files.Source, files.Destination)); + } + + [Fact] + public void TruncatedDestination_IsNotReusable() + { + using var files = new TemporaryFiles(); + File.WriteAllBytes(files.Source, "GGUF-complete-payload"u8.ToArray()); + File.WriteAllBytes(files.Destination, "GGUF-partial"u8.ToArray()); + File.SetLastWriteTimeUtc(files.Destination, File.GetLastWriteTimeUtc(files.Source)); + + Assert.False(HuggingFaceBaselineService.CanReuseDownloadedFile(files.Source, files.Destination)); + } + + [Fact] + public void SameSizeButDifferentTimestamp_IsNotReusable() + { + using var files = new TemporaryFiles(); + File.WriteAllBytes(files.Source, "GGUF-source-payload"u8.ToArray()); + File.Copy(files.Source, files.Destination); + File.SetLastWriteTimeUtc(files.Destination, File.GetLastWriteTimeUtc(files.Source).AddSeconds(-1)); + + Assert.False(HuggingFaceBaselineService.CanReuseDownloadedFile(files.Source, files.Destination)); + } + + [Fact] + public void InvalidGgufMagic_IsNotReusable() + { + using var files = new TemporaryFiles(); + File.WriteAllBytes(files.Source, "NOPE-source-payload"u8.ToArray()); + File.Copy(files.Source, files.Destination); + File.SetLastWriteTimeUtc(files.Destination, File.GetLastWriteTimeUtc(files.Source)); + + Assert.False(HuggingFaceBaselineService.CanReuseDownloadedFile(files.Source, files.Destination)); + } + + [Fact] + public void StagingCleanupPath_MustRemainInsideDestinationDirectory() + { + string root = Path.Combine(Path.GetTempPath(), "mq-hf-path-test", Guid.NewGuid().ToString("N")); + string cache = Path.Combine(root, "ExternalBaselines"); + + Assert.True(HuggingFaceBaselineService.IsPathInsideDirectory( + Path.Combine(cache, "source.gguf"), cache)); + Assert.False(HuggingFaceBaselineService.IsPathInsideDirectory( + Path.Combine(root, "outside.gguf"), cache)); + } + + [Fact] + public void StagingCleanupPath_ResolvesSymlinkedParentDirectory() + { + string root = Path.Combine(Path.GetTempPath(), "mq-hf-symlink-test-" + Guid.NewGuid().ToString("N")); + string physical = Path.Combine(root, "physical-model"); + string alias = Path.Combine(root, "model-alias"); + string cache = Path.Combine(physical, "MagicQuant", "ExternalBaselines"); + + try + { + Directory.CreateDirectory(cache); + Directory.CreateSymbolicLink(alias, physical); + + string downloadedPath = Path.Combine(cache, "source.gguf"); + File.WriteAllBytes(downloadedPath, "GGUF-source-payload"u8.ToArray()); + + string aliasedCache = Path.Combine(alias, "MagicQuant", "ExternalBaselines"); + Assert.True(HuggingFaceBaselineService.IsPathInsideDirectory(downloadedPath, aliasedCache)); + Assert.True(HuggingFaceBaselineService.PathsReferToSameLocation( + downloadedPath, + Path.Combine(aliasedCache, "source.gguf"))); + } + finally + { + if (Directory.Exists(root)) + Directory.Delete(root, recursive: true); + } + } + + private sealed class TemporaryFiles : IDisposable + { + private readonly string _directory = Path.Combine( + Path.GetTempPath(), "mq-hf-cache-test-" + Guid.NewGuid().ToString("N")); + + public TemporaryFiles() + { + Directory.CreateDirectory(_directory); + } + + public string Source => Path.Combine(_directory, "source.gguf"); + public string Destination => Path.Combine(_directory, "destination.gguf"); + + public void Dispose() + { + if (Directory.Exists(_directory)) + Directory.Delete(_directory, recursive: true); + } + } +} diff --git a/tests/MagicQuant.Tests/HuggingFaceRevisionConfigTests.cs b/tests/MagicQuant.Tests/HuggingFaceRevisionConfigTests.cs new file mode 100644 index 0000000..842f44a --- /dev/null +++ b/tests/MagicQuant.Tests/HuggingFaceRevisionConfigTests.cs @@ -0,0 +1,49 @@ +using MagicQuant.Configuration; +using Xunit; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace MagicQuant.Tests; + +public sealed class HuggingFaceRevisionConfigTests +{ + [Fact] + public void CustomRepositoryRevision_DeserializesPinnedCommit() + { + const string yaml = """ + baselines: + custom_repositories: + - repo_id: owner/model-GGUF + revision: 313447f257f7ebde0b968e4778feef774546ed81 + """; + + var config = Deserialize(yaml); + var repository = Assert.Single(config.Baselines.CustomRepositories); + + Assert.Equal("owner/model-GGUF", repository.RepoId); + Assert.Equal("313447f257f7ebde0b968e4778feef774546ed81", repository.Revision); + } + + [Fact] + public void CustomRepositoryRevision_RemainsOptional() + { + const string yaml = """ + baselines: + custom_repositories: + - repo_id: owner/model-GGUF + """; + + var repository = Assert.Single(Deserialize(yaml).Baselines.CustomRepositories); + + Assert.Null(repository.Revision); + } + + private static MagicQuantYamlConfig Deserialize(string yaml) + { + return new DeserializerBuilder() + .IgnoreUnmatchedProperties() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .Build() + .Deserialize(yaml); + } +} diff --git a/tests/MagicQuant.Tests/ImatrixIdentityServiceTests.cs b/tests/MagicQuant.Tests/ImatrixIdentityServiceTests.cs new file mode 100644 index 0000000..c9f8a97 --- /dev/null +++ b/tests/MagicQuant.Tests/ImatrixIdentityServiceTests.cs @@ -0,0 +1,88 @@ +using MagicQuant.Services; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models.DbModels; +using Microsoft.EntityFrameworkCore; +using Xunit; + +namespace MagicQuant.Tests; + +public class ImatrixIdentityServiceTests +{ + [Fact] + public async Task EnsureActiveImatrixIdentityHash_IsStableForSameArtifact() + { + string temp = Path.GetTempFileName(); + await File.WriteAllTextAsync(temp, "imatrix-test-content"); + + Cache.IsImatrixAvailable = true; + Cache.ActiveImatrixPath = temp; + Cache.ActiveImatrixIdentityHash = null; + + var first = await ImatrixIdentityService.EnsureActiveImatrixIdentityHashAsync(); + Cache.ActiveImatrixIdentityHash = null; + var second = await ImatrixIdentityService.EnsureActiveImatrixIdentityHashAsync(); + + Assert.False(string.IsNullOrWhiteSpace(first)); + Assert.Equal(first, second); + + File.Delete(temp); + } + + [Fact] + public async Task ResolveCurrentImatrixDefinitionId_SameModelSameImatrix_ReusesSameRow() + { + string tempRoot = Path.Combine(Path.GetTempPath(), "mq-imatrix-test-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempRoot); + + Cache.MagicQuantDirectory = tempRoot; + Cache.CurrentModelId = "model-test-same"; + + string tempImatrix = Path.Combine(tempRoot, "imatrix.dat"); + await File.WriteAllTextAsync(tempImatrix, "same-imatrix"); + + Cache.IsImatrixAvailable = true; + Cache.ActiveImatrixPath = tempImatrix; + Cache.ActiveImatrixIdentityHash = null; + + await using var db = new MagicQuantContext(); + var model = await db.AiModelHashes.FirstOrDefaultAsync(x => x.UniqueHash == Cache.CurrentModelId); + if (model == null) + { + model = new AiModelHash { UniqueHash = Cache.CurrentModelId }; + db.AiModelHashes.Add(model); + await db.SaveChangesAsync(); + } + + var first = await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, model.Id, createIfMissing: true); + var second = await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, model.Id, createIfMissing: true); + + Assert.NotNull(first); + Assert.Equal(first, second); + } + + [Fact] + public async Task ResolveCurrentImatrixDefinitionId_NoImatrix_ReturnsNull() + { + string tempRoot = Path.Combine(Path.GetTempPath(), "mq-imatrix-test-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempRoot); + + Cache.MagicQuantDirectory = tempRoot; + Cache.CurrentModelId = "model-test-none"; + Cache.IsImatrixAvailable = false; + Cache.ActiveImatrixPath = null; + Cache.ActiveImatrixIdentityHash = null; + + await using var db = new MagicQuantContext(); + var model = await db.AiModelHashes.FirstOrDefaultAsync(x => x.UniqueHash == Cache.CurrentModelId); + if (model == null) + { + model = new AiModelHash { UniqueHash = Cache.CurrentModelId }; + db.AiModelHashes.Add(model); + await db.SaveChangesAsync(); + } + + var id = await ImatrixIdentityService.ResolveCurrentImatrixDefinitionIdAsync(db, model.Id, createIfMissing: false); + Assert.Null(id); + } +} diff --git a/tests/MagicQuant.Tests/LearnedBaselinePruningServiceTests.cs b/tests/MagicQuant.Tests/LearnedBaselinePruningServiceTests.cs new file mode 100644 index 0000000..691c0ac --- /dev/null +++ b/tests/MagicQuant.Tests/LearnedBaselinePruningServiceTests.cs @@ -0,0 +1,33 @@ +using MagicQuant.Services; +using Xunit; + +namespace MagicQuant.Tests; + +public class LearnedBaselinePruningServiceTests +{ + [Fact] + public async Task CoverageStatus_ReportsEarlyPruningDisabled() + { + var service = new LearnedBaselinePruningService(); + + var status = await service.GetCoverageStatusAsync(); + + Assert.False(status.HasAnyLearnedRows); + Assert.False(status.SafeToApplyBeforeStartup); + Assert.Equal(0, status.ExpectedCandidateGroupPairs); + Assert.Equal(0, status.PresentCandidateGroupPairs); + Assert.Contains(status.MissingPairs, x => x.Contains("disabled", StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public async Task AnalyzeAndApply_DoesNotPruneWhileFeatureIsDisabled() + { + var service = new LearnedBaselinePruningService(); + + var result = await service.AnalyzeAndApplyAsync(); + + Assert.Equal(0, result.GroupCandidateEliminations); + Assert.Equal(0, result.BaselinesSkippedWithoutLearnedRows); + Assert.Contains(result.Notes, x => x.Contains("disabled", StringComparison.OrdinalIgnoreCase)); + } +} diff --git a/tests/MagicQuant.Tests/LlamaBinaryPathTests.cs b/tests/MagicQuant.Tests/LlamaBinaryPathTests.cs new file mode 100644 index 0000000..8b1e9e0 --- /dev/null +++ b/tests/MagicQuant.Tests/LlamaBinaryPathTests.cs @@ -0,0 +1,25 @@ +using MQ.DB; +using MQ.DB.Models; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class LlamaBinaryPathTests +{ + [Fact] + public void Explicit_binary_directory_wins_and_checkout_root_is_a_real_fallback() + { + string? old = Cache.LlamaBin; + try + { + string suffix = OperatingSystem.IsWindows() ? ".exe" : ""; + Cache.LlamaBin = Path.Combine(Path.GetTempPath(), "custom bin"); + Assert.Equal(Path.Combine(Cache.LlamaBin, "llama-bench" + suffix), new LlamaBinaries("ignored").Bench); + Cache.LlamaBin = null; + string root = Path.Combine(Path.GetTempPath(), "llama"); + Assert.Equal(Path.Combine(root, "build", "bin", "llama-cli" + suffix), new LlamaBinaries(root).Cli); + Assert.Throws(() => new LlamaBinaries(null)); + } + finally { Cache.LlamaBin = old; } + } +} diff --git a/tests/MagicQuant.Tests/LlamaGpuArgumentBuilderTests.cs b/tests/MagicQuant.Tests/LlamaGpuArgumentBuilderTests.cs new file mode 100644 index 0000000..d7f2d6a --- /dev/null +++ b/tests/MagicQuant.Tests/LlamaGpuArgumentBuilderTests.cs @@ -0,0 +1,51 @@ +using MagicQuant.Services; +using Xunit; + +namespace MagicQuant.Tests; + +public class LlamaGpuArgumentBuilderTests +{ + private static readonly IReadOnlyDictionary Limits = + new Dictionary + { + [0] = 19, + [1] = 23 + }; + + [Fact] + public void CommonCli_UsesCommaSeparatedMembers() + { + string args = LlamaGpuArgumentBuilder.BuildTensorSplitArgs( + [0, 1], Limits, LlamaGpuTool.CommonCli); + + Assert.Equal(" --tensor-split 19,23", args); + } + + [Fact] + public void LlamaBench_UsesSlashSeparatedMembers() + { + string args = LlamaGpuArgumentBuilder.BuildTensorSplitArgs( + [0, 1], Limits, LlamaGpuTool.LlamaBench); + + Assert.Equal(" --tensor-split 19/23", args); + } + + [Fact] + public void SingleGpu_DoesNotEmitTensorSplit() + { + string args = LlamaGpuArgumentBuilder.BuildTensorSplitArgs( + [1], Limits, LlamaGpuTool.CommonCli); + + Assert.Equal(string.Empty, args); + } + + [Fact] + public void MissingConfiguredLimit_IsRejected() + { + var error = Assert.Throws(() => + LlamaGpuArgumentBuilder.BuildTensorSplitArgs( + [0, 2], Limits, LlamaGpuTool.CommonCli)); + + Assert.Contains("2", error.Message, StringComparison.Ordinal); + } +} diff --git a/tests/MagicQuant.Tests/MagicQuant.Tests.csproj b/tests/MagicQuant.Tests/MagicQuant.Tests.csproj new file mode 100644 index 0000000..3ae77ff --- /dev/null +++ b/tests/MagicQuant.Tests/MagicQuant.Tests.csproj @@ -0,0 +1,19 @@ + + + net10.0 + enable + enable + false + + + + + + + + + + + + + diff --git a/tests/MagicQuant.Tests/ModelSmokeTests.cs b/tests/MagicQuant.Tests/ModelSmokeTests.cs new file mode 100644 index 0000000..3f9b0a6 --- /dev/null +++ b/tests/MagicQuant.Tests/ModelSmokeTests.cs @@ -0,0 +1,131 @@ +using System.Diagnostics; +using System.Text.Json; +using MagicQuant.Configuration; +using MagicQuant.Helpers; +using MagicQuant.Runtime; +using MagicQuant.Services; +using MQ.DB; +using MQ.DB.Models; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class ModelSmokeFactAttribute : FactAttribute +{ + public ModelSmokeFactAttribute() + { + if (Environment.GetEnvironmentVariable("MQ_RUN_MODEL_SMOKE") != "1") + Skip = "Opt in with MQ_RUN_MODEL_SMOKE=1 and the documented model/toolchain paths."; + } +} + +/// Small-model integration, isolated from ordinary PR checks and existing campaign state. +public sealed class ModelSmokeTests +{ + [ModelSmokeFact] + [Trait("Category", "ModelSmoke")] + public async Task Convert_quantize_read_metadata_benchmark_and_reuse_native_artifact() + { + string Required(string key) => Environment.GetEnvironmentVariable(key) + ?? throw new InvalidOperationException($"Set {key}; see docs/testing.md."); + string source = Path.GetFullPath(Required("MQ_SMOKE_MODEL")); + string llama = Path.GetFullPath(Required("MQ_SMOKE_LLAMA_ROOT")); + string runtime = Path.GetFullPath(Required("MQ_SMOKE_RUNTIME_ROOT")); + string output = Path.GetFullPath(Required("MQ_SMOKE_OUTPUT")); + string root = Path.Combine(output, $"smoke-{Guid.NewGuid():N}"); + string model = Path.Combine(root, "model with spaces"); + Assert.True(Directory.Exists(source)); + Directory.CreateDirectory(model); + // Inputs are copied/linked into a new directory; no source-model files are modified. + foreach (string file in Directory.EnumerateFiles(source)) + { + string destination = Path.Combine(model, Path.GetFileName(file)); + if (file.EndsWith(".safetensors", StringComparison.Ordinal)) File.CreateSymbolicLink(destination, file); + else File.Copy(file, destination); + } + + var oldConfig = Config.Current; + var oldPaths = (Cache.ModelDirectory, Cache.ModelMagicQuantDirectory, Cache.MagicQuantDirectory, + Cache.LlamaRoot, Cache.LlamaBin, Cache.ConvertScript, Cache.CurrentModelId); + var oldPrecision = Cache.TorchType; + var oldScratch = Cache.ScratchRoots; + var oldImatrix = (Cache.UseImatrix, Cache.IsImatrixAvailable); + try + { + using var timeout = new CancellationTokenSource(TimeSpan.FromMinutes(20)); + using var scope = RunCancellation.Use(timeout.Token); + Config.Load(MagicQuantYamlConfig.CreateDefault()); + Cache.ModelDirectory = model; + Cache.ModelMagicQuantDirectory = Path.Combine(model, "MagicQuant"); + Cache.MagicQuantDirectory = runtime; + Cache.LlamaRoot = llama; + Cache.LlamaBin = Path.Combine(llama, "build", "bin"); + Cache.ConvertScript = Path.Combine(llama, "convert_hf_to_gguf.py"); + Cache.CurrentModelId = "isolated-smoke"; + Cache.ScratchRoots = [root]; + Cache.UseImatrix = false; + Cache.IsImatrixAvailable = false; + JsonHelper.DetectAndSetTorchType(model); + var python = new PythonManager(runtime); + var quantizer = new QuantizationService(new BenchmarkService(python)); + string native = await quantizer.EnsureBaseModelFileAsync(); + DateTime nativeTimestamp = File.GetLastWriteTimeUtc(native); + Assert.Equal(native, await quantizer.EnsureBaseModelFileAsync()); + Assert.Equal(nativeTimestamp, File.GetLastWriteTimeUtc(native)); + + string export = Path.Combine(root, "export"); + Directory.CreateDirectory(export); + string q8 = Path.Combine(export, "smoke Q8_0.gguf"); + string scratch; + await using (var lease = await quantizer.BuildPureQ8ProbeLeaseAsync(timeout.Token)) + { + scratch = lease.GgufPath; + Assert.True(new FileInfo(scratch).Length > 0); + + } + Assert.False(File.Exists(scratch)); + await quantizer.BuildExportArtifactAsync(HybridQuant.CreatePureBaseline(BaselineQuants.Q8_0), q8, forceRebuild: true, ct: timeout.Token); + DateTime exportTimestamp = File.GetLastWriteTimeUtc(q8); + await quantizer.BuildExportArtifactAsync(HybridQuant.CreatePureBaseline(BaselineQuants.Q8_0), q8, ct: timeout.Token); + Assert.Equal(exportTimestamp, File.GetLastWriteTimeUtc(q8)); + var reader = new GgufMetadataReader(python); + var nativeMetadata = await reader.ReadAsync(native, root, timeout.Token); + var q8Metadata = await reader.ReadAsync(q8, root, timeout.Token); + Assert.NotEmpty(nativeMetadata.TensorNames); + Assert.Equal(nativeMetadata.TensorNames.OrderBy(x => x), q8Metadata.TensorNames.OrderBy(x => x)); + string benchLog = Path.Combine(export, "llamabench.md"); + var command = BenchmarkCommands.Bench(Path.Combine(Cache.LlamaBin, OperatingSystem.IsWindows() ? "llama-bench.exe" : "llama-bench"), q8, false, 0, ""); + var result = await new ProcessRunner().RunAsync(command.CreateStartInfo(), benchLog, ct: timeout.Token); + Assert.True(result.Success, result.CombinedOutput); + var metrics = BenchmarkLogParser.ParseLlamaBench(benchLog); + Assert.True(metrics.Tps > 0); + string manifest = MagicQuantManifestPathService.GetManifestFilePath(export, "smoke.tensor-map.json"); + await File.WriteAllTextAsync(manifest, JsonSerializer.Serialize(q8Metadata.TensorTypes)); + await File.WriteAllTextAsync(Path.Combine(root, "smoke-result.json"), JsonSerializer.Serialize(new + { + SourceModel = source, + LlamaRoot = llama, + RuntimeRoot = runtime, + NativeBytes = new FileInfo(native).Length, + Q8Bytes = new FileInfo(q8).Length, + TensorCount = q8Metadata.TensorNames.Count, + metrics.Tps, + NativeReuseVerified = true, + ScratchCleanupVerified = true, + CompletedUtc = DateTimeOffset.UtcNow + }, new JsonSerializerOptions { WriteIndented = true })); + } + finally + { + Config.Load(oldConfig); + (Cache.ModelDirectory, Cache.ModelMagicQuantDirectory, Cache.MagicQuantDirectory, + Cache.LlamaRoot, Cache.LlamaBin, Cache.ConvertScript, Cache.CurrentModelId) = oldPaths; + Cache.TorchType = oldPrecision; + Cache.ScratchRoots = oldScratch; + (Cache.UseImatrix, Cache.IsImatrixAvailable) = oldImatrix; + // Keep only logs, metadata and the result report. Always remove heavy test weights. + foreach (string file in Directory.EnumerateFiles(root, "*.gguf", SearchOption.AllDirectories)) File.Delete(file); + foreach (string file in Directory.EnumerateFiles(model, "*.safetensors")) File.Delete(file); + } + } +} diff --git a/tests/MagicQuant.Tests/NativeConversionTests.cs b/tests/MagicQuant.Tests/NativeConversionTests.cs new file mode 100644 index 0000000..115b5b9 --- /dev/null +++ b/tests/MagicQuant.Tests/NativeConversionTests.cs @@ -0,0 +1,69 @@ +using System.Diagnostics; +using MagicQuant.Helpers; +using MagicQuant.Runtime; +using MagicQuant.Services; +using MQ.DB; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class NativeConversionTests +{ + [Theory] + [InlineData(0)] + [InlineData(7)] + [InlineData(-1)] + public async Task Only_completed_conversion_is_reusable(int exitCode) + { + string root = Path.Combine(Path.GetTempPath(), $"mq-conversion-{Guid.NewGuid():N}"); + var old = (Cache.ModelDirectory, Cache.ModelMagicQuantDirectory, Cache.ConvertScript, Cache.TorchType); + try + { + Cache.ModelDirectory = Path.Combine(root, "source with spaces"); + Cache.ModelMagicQuantDirectory = Path.Combine(Cache.ModelDirectory, "MagicQuant"); + Cache.ConvertScript = Path.Combine(root, "converter with spaces.py"); + Cache.TorchType = Cache.MainTorchType.BF16; + var paths = new ModelArtifactPathService(); + Directory.CreateDirectory(paths.GgufDir); + var runner = new ConverterStub(exitCode); + var converter = new NativeModelConversionService(paths, new PythonManager(root), runner); + string native = paths.GetNativeBaseGgufPath(); + if (exitCode == 0) + { + Assert.Equal(native, await converter.EnsureAsync()); + Assert.Equal(native, await converter.EnsureAsync()); + Assert.Equal(1, runner.Calls); + // A stale marker cannot make a truncated artifact look complete. + File.WriteAllText(native, ""); + await converter.EnsureAsync(); + Assert.Equal(2, runner.Calls); + } + else + { + await Assert.ThrowsAnyAsync(() => converter.EnsureAsync()); + Assert.False(File.Exists(native)); + Assert.False(File.Exists(native + ".success.json")); + } + } + finally + { + (Cache.ModelDirectory, Cache.ModelMagicQuantDirectory, Cache.ConvertScript, Cache.TorchType) = old; + if (Directory.Exists(root)) Directory.Delete(root, true); + } + } + + private sealed class ConverterStub(int exitCode) : IProcessRunner + { + public int Calls { get; private set; } + public Task RunAsync(ProcessStartInfo start, string? logPath = null, Action? onLine = null, CancellationToken ct = default) + { + Calls++; + Assert.Equal(Cache.ConvertScript, start.ArgumentList[0]); + Assert.Equal(Cache.ModelDirectory, start.ArgumentList[1]); + int outputIndex = start.ArgumentList.IndexOf("--outfile") + 1; + File.WriteAllText(start.ArgumentList[outputIndex], "GGUF fixture"); + if (exitCode == -1) throw new OperationCanceledException(); + return Task.FromResult(new ProcessResult(exitCode, "", "")); + } + } +} diff --git a/tests/MagicQuant.Tests/OutputPathTests.cs b/tests/MagicQuant.Tests/OutputPathTests.cs new file mode 100644 index 0000000..4852817 --- /dev/null +++ b/tests/MagicQuant.Tests/OutputPathTests.cs @@ -0,0 +1,65 @@ +using MagicQuant.Services; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class OutputPathTests +{ + private static readonly string ModelWork = Path.Combine(Path.GetTempPath(), "model with spaces", "MagicQuant"); + + [Fact] + public void Default_destinations_preserve_existing_command_layouts() + { + Assert.Equal(Path.Combine(ModelWork, "Final_Outputs"), OutputPathService.Pipeline(ModelWork, null)); + Assert.Equal(Path.Combine(ModelWork, "FinalOutput"), OutputPathService.Clone(ModelWork, null, null)); + Assert.Equal(Path.Combine(ModelWork, "PredictionValidation"), OutputPathService.PredictionValidation(ModelWork, null, null)); + } + + [Fact] + public void Relative_pipeline_output_is_model_local_but_clone_output_is_cwd_relative() + { + Assert.Equal(Path.Combine(ModelWork, "exports"), OutputPathService.Pipeline(ModelWork, "exports")); + Assert.Equal(Path.GetFullPath("exports"), OutputPathService.Clone(ModelWork, null, "exports")); + } + + [Fact] + public void Absolute_pipeline_output_overrides_model_directory() + { + string output = Path.Combine(Path.GetTempPath(), "other exports"); + Assert.Equal(output, OutputPathService.Pipeline(ModelWork, output)); + } + + [Fact] + public void Explicit_validation_destination_does_not_add_a_subdirectory() + { + Assert.Equal(Path.GetFullPath("reports"), OutputPathService.PredictionValidation(ModelWork, "reports", "ignored")); + Assert.Equal(Path.Combine(Path.GetFullPath("exports"), "PredictionValidation"), + OutputPathService.PredictionValidation(ModelWork, null, "exports")); + Assert.Equal(Path.GetFullPath("exports"), OutputPathService.Clone(ModelWork, "exports", "ignored")); + } + + [Theory] + [InlineData("magicquant.final-survivors.json")] + [InlineData("magicquant-manifest/magicquant.final-survivors.json")] + [InlineData("magicquant-manifest\\magicquant.final-survivors.json")] + public void Manifest_links_have_one_directory_prefix_and_forward_slashes(string file) + { + Assert.Equal("magicquant-manifest/magicquant.final-survivors.json", MagicQuantManifestPathService.RelativeManifestPath(file)); + } + + [Fact] + public void Manifest_directory_creation_is_idempotent_with_trailing_separator() + { + string root = Path.Combine(Path.GetTempPath(), $"mq-manifest-{Guid.NewGuid():N}"); + try + { + string manifest = MagicQuantManifestPathService.EnsureManifestDirectory(root); + Assert.Equal(manifest, MagicQuantManifestPathService.EnsureManifestDirectory(manifest + Path.DirectorySeparatorChar)); + Assert.Empty(Directory.GetDirectories(manifest)); + } + finally + { + Directory.Delete(root, recursive: true); + } + } +} diff --git a/tests/MagicQuant.Tests/PreflightTests.cs b/tests/MagicQuant.Tests/PreflightTests.cs new file mode 100644 index 0000000..88fd772 --- /dev/null +++ b/tests/MagicQuant.Tests/PreflightTests.cs @@ -0,0 +1,102 @@ +using MagicQuant.Configuration; +using MagicQuant.Models; +using MagicQuant.Services; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class PreflightTests : IDisposable +{ + private readonly string _root = Path.Combine(Path.GetTempPath(), $"mq-preflight-{Guid.NewGuid():N}"); + private readonly MagicQuantYamlConfig _config = MagicQuantYamlConfig.CreateDefault(); + + public PreflightTests() + { + Directory.CreateDirectory(_root); + _config.Paths.ModelDir = Path.Combine(_root, "model"); + _config.Paths.MagicQuantRoot = Path.Combine(_root, "runtime"); + _config.Identity.ArchitectureFamilyName = "test-family"; + Directory.CreateDirectory(_config.Paths.ModelDir); + File.WriteAllText(Path.Combine(_config.Paths.ModelDir, "test.safetensors"), "fixture"); + File.WriteAllText(Path.Combine(_config.Paths.ModelDir, "config.json"), "{}"); + } + + [Fact] + public void Valid_preflight_does_not_create_runtime_or_model_work_directories() + { + CommandPreflight.Validate("pipeline", _config, []); + Assert.False(Directory.Exists(_config.Paths.MagicQuantRoot)); + Assert.False(Directory.Exists(Path.Combine(_config.Paths.ModelDir!, "MagicQuant"))); + } + + [Theory] + [InlineData(".")] + [InlineData("..")] + [InlineData("../..")] + [InlineData("GGUF")] + [InlineData("GGUF/nested")] + [InlineData("Benchmarks")] + public void Output_cannot_destroy_source_or_managed_artifacts(string output) + { + _config.Output.OutputDir = output; + Assert.Throws(() => CommandPreflight.Validate("pipeline", _config, [])); + } + + [Fact] + public void Symlinked_output_is_checked_against_the_physical_target() + { + if (OperatingSystem.IsWindows()) return; // Windows CI does not grant symlink privilege. + string link = Path.Combine(_root, "export-link"); + Directory.CreateSymbolicLink(link, _config.Paths.ModelDir!); + _config.Output.OutputDir = link; + Assert.Throws(() => CommandPreflight.Validate("pipeline", _config, [])); + } + + [Theory] + [InlineData("config.json")] + [InlineData("test.safetensors")] + public void Incomplete_models_fail_before_work_starts(string missing) + { + File.Delete(Path.Combine(_config.Paths.ModelDir!, missing)); + Assert.Throws(() => CommandPreflight.Validate("pipeline", _config, [])); + Assert.False(Directory.Exists(_config.Paths.MagicQuantRoot)); + } + + [Fact] + public void Clone_source_is_required_and_legacy_alias_is_accepted() + { + Assert.Throws(() => CommandPreflight.Validate("clone-repository-quants", _config, [])); + CommandPreflight.Validate("clone-repository-quants", _config, + [new CliArg { Name = "clone-json", Value = Path.Combine(_config.Paths.ModelDir!, "config.json") }]); + } + + [Fact] + public void Misspelled_baseline_mode_does_not_silently_enable_all_baselines() + { + _config.Baselines.StandardBaselinesMode = "selcted"; + Assert.Throws(() => CommandPreflight.Validate("pipeline", _config, [])); + } + + [Theory] + [InlineData("../downloads")] + [InlineData("/tmp/downloads")] + [InlineData("a\\b")] + public void External_cache_name_cannot_escape_model_storage(string name) + { + _config.Paths.ExternalBaselineCacheDirName = name; + Assert.Throws(() => CommandPreflight.Validate("pipeline", _config, [])); + } + + [Fact] + public void Export_cannot_clean_the_runtime_toolchain_or_a_scratch_parent() + { + _config.Output.OutputDir = Path.Combine(_config.Paths.MagicQuantRoot!, "llama.cpp"); + Assert.Throws(() => CommandPreflight.Validate("pipeline", _config, [])); + string scratchRoot = Path.Combine(_root, "scratch"); + _config.Paths.ScratchRoots = [scratchRoot]; + _config.Output.OutputDir = scratchRoot; + Assert.Throws(() => CommandPreflight.Validate("pipeline", _config, [])); + } + + public void Dispose() => Directory.Delete(_root, recursive: true); +} diff --git a/tests/MagicQuant.Tests/ProcessRunnerTests.cs b/tests/MagicQuant.Tests/ProcessRunnerTests.cs new file mode 100644 index 0000000..6a1d6f4 --- /dev/null +++ b/tests/MagicQuant.Tests/ProcessRunnerTests.cs @@ -0,0 +1,92 @@ +using System.Diagnostics; +using MagicQuant.Runtime; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class ProcessRunnerTests +{ + private static ProcessStartInfo Start(params string[] args) + { + var start = new ProcessStartInfo("dotnet"); + start.ArgumentList.Add(typeof(ProcessFixture.Program).Assembly.Location); + foreach (string arg in args) start.ArgumentList.Add(arg); + return start; + } + + [Fact] + public async Task Arguments_preserve_spaces_quotes_and_shell_metacharacters() + { + string[] values = ["folder with spaces", "file'with\"quotes", "$(touch not-a-command)", "C:\\models\\my model", "a;b&c"]; + var result = await new ProcessRunner().RunAsync(Start(["echo", .. values])); + Assert.True(result.Success); + Assert.Equal(string.Join(Environment.NewLine, values) + Environment.NewLine, result.StdOut); + } + + [Fact] + public async Task Drains_both_full_pipes_and_preserves_nonzero_exit_and_log() + { + string log = Path.GetTempFileName(); + try + { + using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var result = await new ProcessRunner().RunAsync(Start("flood"), log, ct: timeout.Token); + Assert.Equal(7, result.ExitCode); + Assert.Contains("stdout-11999", result.StdOut); + Assert.Contains("stderr-11999", result.StdErr); + Assert.Contains("stderr-11999", File.ReadAllText(log)); + using var exclusive = new FileStream(log, FileMode.Open, FileAccess.ReadWrite, FileShare.None); + } + finally { File.Delete(log); } + } + + [Fact] + public async Task Cancellation_reaps_native_work_and_releases_log() + { + string log = Path.GetTempFileName(); + using var cancel = new CancellationTokenSource(TimeSpan.FromSeconds(20)); + int pid = 0; + try + { + var task = new ProcessRunner().RunAsync(Start("wait"), log, (line, _) => + { + if (line.StartsWith("ready:")) { pid = int.Parse(line[6..]); cancel.Cancel(); } + }, cancel.Token); + await Assert.ThrowsAnyAsync(() => task); + Assert.True(pid > 0); + Assert.False(IsRunning(pid)); + using var exclusive = new FileStream(log, FileMode.Open, FileAccess.ReadWrite, FileShare.None); + } + finally { File.Delete(log); } + } + + [Fact] + public async Task Command_scope_cancellation_stops_legacy_callers_without_an_explicit_token() + { + using var cancel = new CancellationTokenSource(); + using (RunCancellation.Use(cancel.Token)) + { + cancel.Cancel(); + await Assert.ThrowsAnyAsync(() => new ProcessRunner().RunAsync(Start("wait"))); + } + Assert.False(RunCancellation.Token.IsCancellationRequested); + } + + [Fact] + public async Task Callback_failure_does_not_leave_the_child_running() + { + int pid = 0; + await Assert.ThrowsAsync(() => new ProcessRunner().RunAsync(Start("wait"), onLine: (line, _) => + { + pid = int.Parse(line[6..]); + throw new IOException("Simulated log failure"); + })); + Assert.False(IsRunning(pid)); + } + + private static bool IsRunning(int pid) + { + try { using var process = Process.GetProcessById(pid); return !process.HasExited; } + catch (ArgumentException) { return false; } + } +} diff --git a/tests/MagicQuant.Tests/QuantizationConcurrencyTests.cs b/tests/MagicQuant.Tests/QuantizationConcurrencyTests.cs new file mode 100644 index 0000000..61f7f85 --- /dev/null +++ b/tests/MagicQuant.Tests/QuantizationConcurrencyTests.cs @@ -0,0 +1,21 @@ +using MagicQuant.Services; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class QuantizationConcurrencyTests +{ + [Theory] + [InlineData(1, 4, 1, 1)] + [InlineData(4, 4, 1, 3)] + [InlineData(8, 4, 1, 6)] + [InlineData(32, 2, 2, 15)] + [InlineData(32, 10, 7, 4)] + public void Writer_capacity_and_cpu_budget_bound_concurrency(int threads, int writers, int concurrent, int perProcess) + { + var plan = QuantizationConcurrencyPlan.Create(threads, writers); + Assert.Equal(concurrent, plan.Concurrency); + Assert.Equal(perProcess, plan.ThreadsPerProcess); + Assert.True(plan.Concurrency * plan.ThreadsPerProcess <= Math.Max(1, threads)); + } +} diff --git a/tests/MagicQuant.Tests/QuantizationRunAndBuildHybridsRegressionTests.cs b/tests/MagicQuant.Tests/QuantizationRunAndBuildHybridsRegressionTests.cs new file mode 100644 index 0000000..7055a96 --- /dev/null +++ b/tests/MagicQuant.Tests/QuantizationRunAndBuildHybridsRegressionTests.cs @@ -0,0 +1,113 @@ +using MagicQuant.Commands; +using MagicQuant.Configuration; +using MagicQuant.Models; +using Microsoft.EntityFrameworkCore; +using Microsoft.Data.Sqlite; +using MQ.DB; +using MQ.DB.Data; +using MQ.DB.Models.DbModels; +using Xunit; + +namespace MagicQuant.Tests; + +public class QuantizationRunAndBuildHybridsRegressionTests +{ + [Fact] + public async Task QuantizationRun_PersistsAndLoads_ImatrixDefinitionForeignKey() + { + string tempRoot = Path.Combine(Path.GetTempPath(), "mq-quant-run-fk-" + Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempRoot); + string? priorMagicQuantDirectory = Cache.MagicQuantDirectory; + + try + { + Cache.MagicQuantDirectory = tempRoot; + + await using var db = new MagicQuantContext(); + + var model = new AiModelHash + { + UniqueHash = "model-" + Guid.NewGuid().ToString("N") + }; + + var combo = new TensorCombo(); + var architecture = new ArchitectureFamily + { + NormalizedName = "test-architecture-" + Guid.NewGuid().ToString("N"), + DisplayName = "Test architecture", + TensorSignatureHash = "signature-" + Guid.NewGuid().ToString("N"), + TensorCount = 1 + }; + var profile = new TensorGroupProfile + { + ArchitectureFamily = architecture, + FingerprintHash = "profile-" + Guid.NewGuid().ToString("N"), + SnapshotJson = "{}" + }; + db.AiModelHashes.Add(model); + db.TensorCombos.Add(combo); + db.ArchitectureFamilies.Add(architecture); + db.TensorGroupProfiles.Add(profile); + await db.SaveChangesAsync(); + + var imatrix = new ImatrixDefinition + { + AiModelHashId = model.Id, + IdentityHash = "imatrix-" + Guid.NewGuid().ToString("N"), + SourceKind = "test" + }; + db.ImatrixDefinitions.Add(imatrix); + await db.SaveChangesAsync(); + + var run = new QuantizationRun + { + ArchitectureFamilyId = architecture.Id, + TensorGroupProfileId = profile.Id, + AiModelHashId = model.Id, + ImatrixDefinitionId = imatrix.Id, + TensorComboId = combo.Id, + StartedUtc = DateTime.UtcNow.AddSeconds(-1), + CompletedUtc = DateTime.UtcNow, + DurationMs = 1000, + Succeeded = true, + OutputModelPath = Path.Combine(tempRoot, "output.gguf") + }; + db.QuantizationRuns.Add(run); + await db.SaveChangesAsync(); + + var loaded = await db.QuantizationRuns + .Include(x => x.ImatrixDefinition) + .SingleAsync(x => x.Id == run.Id); + + Assert.Equal(imatrix.Id, loaded.ImatrixDefinitionId); + Assert.NotNull(loaded.ImatrixDefinition); + Assert.Equal(imatrix.IdentityHash, loaded.ImatrixDefinition!.IdentityHash); + } + finally + { + Cache.MagicQuantDirectory = priorMagicQuantDirectory; + // Disposing the context returns connections to SQLite's pool. Release + // those handles before deleting this test's database on Windows. + SqliteConnection.ClearAllPools(); + if (Directory.Exists(tempRoot)) + Directory.Delete(tempRoot, recursive: true); + } + } + + [Fact] + public async Task BuildHybrids_RunWithoutModel_RoutesThroughEvolutionValidation() + { + var priorConfig = Config.Current; + try + { + Config.Load(MagicQuantYamlConfig.CreateDefault()); + var command = new BuildHybrids(); + var ex = await Assert.ThrowsAsync(() => command.Run(new List())); + Assert.Contains("model directory", ex.Message, StringComparison.OrdinalIgnoreCase); + } + finally + { + Config.Load(priorConfig); + } + } +} diff --git a/tests/MagicQuant.Tests/ReadmeGenerationTests.cs b/tests/MagicQuant.Tests/ReadmeGenerationTests.cs new file mode 100644 index 0000000..8100c2f --- /dev/null +++ b/tests/MagicQuant.Tests/ReadmeGenerationTests.cs @@ -0,0 +1,29 @@ +using MagicQuant.Services; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class ReadmeGenerationTests +{ + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Exported_readmes_link_to_the_canonical_project(bool clone) + { + string root = Path.Combine(Path.GetTempPath(), "mq-readme-" + Guid.NewGuid().ToString("N")); + try + { + var service = new ReadmeGenerationService(); + string file = clone + ? await service.GenerateCloneAsync(root, "test", "owner/model", true, []) + : await service.GenerateAsync(root, "test", [], []); + string readme = await File.ReadAllTextAsync(file); + Assert.Contains("[MagicQuant](https://github.com/magiccodingman/MagicQuant)", readme); + Assert.DoesNotContain("magicquant-wiki", readme, StringComparison.OrdinalIgnoreCase); + } + finally + { + if (Directory.Exists(root)) Directory.Delete(root, recursive: true); + } + } +} diff --git a/tests/MagicQuant.Tests/RunProvenanceTests.cs b/tests/MagicQuant.Tests/RunProvenanceTests.cs new file mode 100644 index 0000000..a110a4b --- /dev/null +++ b/tests/MagicQuant.Tests/RunProvenanceTests.cs @@ -0,0 +1,35 @@ +using System.Text.Json; +using MagicQuant.Configuration; +using MagicQuant.Services; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class RunProvenanceTests +{ + [Theory] + [InlineData("completed")] + [InlineData("failed")] + [InlineData("canceled")] + public void Records_immutable_inputs_and_terminal_status_atomically(string status) + { + string root = Path.Combine(Path.GetTempPath(), $"mq-provenance-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + string configPath = Path.Combine(root, "config.yaml"); + File.WriteAllText(configPath, "paths: {}"); + var config = MagicQuantYamlConfig.CreateDefault(); + config.Paths.MagicQuantRoot = root; + var provenance = new RunProvenanceService("initialize-llama-cpp", ["initialize-llama-cpp"], new(configPath, config, [])); + config.Output.OutputNamePrefix = "changed-after-start"; + provenance.Complete(status); + using var json = JsonDocument.Parse(File.ReadAllText(provenance.ManifestPath)); + Assert.Equal(status, json.RootElement.GetProperty("status").GetString()); + Assert.Equal("Model", json.RootElement.GetProperty("configuration").GetProperty("Output").GetProperty("OutputNamePrefix").GetString()); + Assert.Equal(64, json.RootElement.GetProperty("configSha256").GetString()!.Length); + Assert.False(File.Exists(provenance.ManifestPath + ".tmp")); + } + finally { Directory.Delete(root, true); } + } +} diff --git a/tests/MagicQuant.Tests/RuntimeSearchSpaceObsoleteContractTests.cs b/tests/MagicQuant.Tests/RuntimeSearchSpaceObsoleteContractTests.cs new file mode 100644 index 0000000..fecd02f --- /dev/null +++ b/tests/MagicQuant.Tests/RuntimeSearchSpaceObsoleteContractTests.cs @@ -0,0 +1,39 @@ +using System.Reflection; +using MagicQuant.Helpers; +using Xunit; + +namespace MagicQuant.Tests; + +public class RuntimeSearchSpaceObsoleteContractTests +{ + [Fact] + public void AllowHighPrecisionHybrids_IsNotObsolete() + { + var prop = typeof(RuntimeSearchSpace).GetProperty(nameof(RuntimeSearchSpace.AllowHighPrecisionHybrids), BindingFlags.Public | BindingFlags.Static); + Assert.NotNull(prop); + Assert.Empty(prop!.GetCustomAttributes(typeof(ObsoleteAttribute), inherit: false)); + + RuntimeSearchSpace.AllowHighPrecisionHybrids = true; + Assert.True(RuntimeSearchSpace.AllowHighPrecisionHybrids); + } + + [Fact] + public void LegacySchemeWrappers_AreObsolete() + { + var wrappers = new[] + { + nameof(RuntimeSearchSpace.BanSchemeForGroup), + nameof(RuntimeSearchSpace.BanSchemeForGroupByLearnedBaselineAbsence), + nameof(RuntimeSearchSpace.BanAllExplicitTensorSchemesForGroup), + nameof(RuntimeSearchSpace.IsSchemeRuntimeBannedForGroup), + nameof(RuntimeSearchSpace.IsGroupExplicitQuantBanned) + }; + + foreach (var methodName in wrappers) + { + var method = typeof(RuntimeSearchSpace).GetMethod(methodName, BindingFlags.Public | BindingFlags.Static); + Assert.NotNull(method); + Assert.NotEmpty(method!.GetCustomAttributes(typeof(ObsoleteAttribute), inherit: false)); + } + } +} diff --git a/tests/MagicQuant.Tests/ScratchStorageServiceTests.cs b/tests/MagicQuant.Tests/ScratchStorageServiceTests.cs new file mode 100644 index 0000000..45dafca --- /dev/null +++ b/tests/MagicQuant.Tests/ScratchStorageServiceTests.cs @@ -0,0 +1,53 @@ +using MagicQuant.Services; +using MQ.DB; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class ScratchStorageServiceTests +{ + [Fact] + public async Task EmptyScratchRoots_FallsBackToSingleWriterCapacity() + { + var temp = Path.Combine(Path.GetTempPath(), "mq-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(temp); + + Cache.MagicQuantDirectory = temp; + Cache.ModelDirectory = temp; + Cache.ModelMagicQuantDirectory = Path.Combine(temp, "MagicQuant"); + Cache.CurrentModelId = "test-model"; + Cache.ScratchRoots = new List(); + + var paths = new ModelArtifactPathService(); + var service = new ScratchStorageService(paths); + + Assert.Equal(1, service.WriterCapacity); + + await service.CleanupStaleScratchArtifactsAsync(); + } + + [Fact] + public async Task PreserveOutput_PreventsLeaseDeletion() + { + var temp = Path.Combine(Path.GetTempPath(), "mq-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(temp); + + Cache.MagicQuantDirectory = temp; + Cache.ModelDirectory = temp; + Cache.ModelMagicQuantDirectory = Path.Combine(temp, "MagicQuant"); + Cache.CurrentModelId = "test-model"; + Cache.ScratchRoots = new List { temp }; + + var paths = new ModelArtifactPathService(); + var service = new ScratchStorageService(paths); + + var lease = await service.AcquireAsync(ScratchArtifactKind.Other, "artifact"); + await File.WriteAllTextAsync(lease.GgufPath, "x"); + lease.PreserveOutput(); + await lease.DisposeAsync(); + + Assert.True(Directory.Exists(lease.LeaseDirectory)); + + await service.CleanupStaleScratchArtifactsAsync(); + } +} diff --git a/tests/MagicQuant.Tests/SmartBaselineTuningFallbackTests.cs b/tests/MagicQuant.Tests/SmartBaselineTuningFallbackTests.cs new file mode 100644 index 0000000..3072606 --- /dev/null +++ b/tests/MagicQuant.Tests/SmartBaselineTuningFallbackTests.cs @@ -0,0 +1,41 @@ +using MagicQuant.Services; +using MQ.DB.Models; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class SmartBaselineTuningFallbackTests +{ + [Fact] + public void LearningOnlyBaselineWithMissingIsolationCoverageIsNotTunable() + { + var eligible = SmartBaselineTuningFallbackService.IsEligibleForSmartFallbackTuning( + BaselineQuants.Q6_K, + [BaselineQuants.Q8_0], + hasCompleteIsolationCoverage: false); + + Assert.False(eligible); + } + + [Fact] + public void ExplicitCandidateWithMissingIsolationCoverageRemainsAHardFailurePath() + { + var eligible = SmartBaselineTuningFallbackService.IsEligibleForSmartFallbackTuning( + BaselineQuants.Q6_K, + [BaselineQuants.Q8_0, BaselineQuants.Q6_K], + hasCompleteIsolationCoverage: false); + + Assert.True(eligible); + } + + [Fact] + public void HistoricalCompleteCoverageAllowsTuningWithoutCurrentExplicitRole() + { + var eligible = SmartBaselineTuningFallbackService.IsEligibleForSmartFallbackTuning( + BaselineQuants.Q6_K, + [BaselineQuants.Q8_0], + hasCompleteIsolationCoverage: true); + + Assert.True(eligible); + } +} diff --git a/tests/MagicQuant.Tests/SynergyTransferConfigTests.cs b/tests/MagicQuant.Tests/SynergyTransferConfigTests.cs new file mode 100644 index 0000000..e7c67ec --- /dev/null +++ b/tests/MagicQuant.Tests/SynergyTransferConfigTests.cs @@ -0,0 +1,45 @@ +using MagicQuant.Configuration; +using Xunit; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace MagicQuant.Tests; + +public sealed class SynergyTransferConfigTests +{ + [Fact] + public void ContextStrata_DeserializeControlledBlanketQuantLists() + { + const string yaml = """ + synergy_detection: + transfer_probe_context_strata: + high_fidelity_reference_quants: [Q6_K, Q5_K] + mid_fidelity_reference_quants: [Q4_K_M] + low_fidelity_reference_quants: [IQ3_S] + low_fidelity_enabled: true + exploratory_context_pair_enabled: true + max_exploratory_context_pairs_per_run: 11 + exploratory_pair_bit_ranges: [3, 4] + exploratory_pair_context_strata: [mid-fidelity, low-fidelity] + context_scoped_rule_application_enabled: true + max_non_rule_group_context_mismatches: 2 + """; + + var config = new DeserializerBuilder() + .IgnoreUnmatchedProperties() + .WithNamingConvention(UnderscoredNamingConvention.Instance) + .Build() + .Deserialize(yaml); + + Assert.Equal(["Q6_K", "Q5_K"], config.SynergyDetection.TransferProbeContextStrata.HighFidelityReferenceQuants); + Assert.Equal(["Q4_K_M"], config.SynergyDetection.TransferProbeContextStrata.MidFidelityReferenceQuants); + Assert.Equal(["IQ3_S"], config.SynergyDetection.TransferProbeContextStrata.LowFidelityReferenceQuants); + Assert.True(config.SynergyDetection.TransferProbeContextStrata.LowFidelityEnabled); + Assert.True(config.SynergyDetection.ExploratoryContextPairEnabled); + Assert.Equal(11, config.SynergyDetection.MaxExploratoryContextPairsPerRun); + Assert.Equal([3, 4], config.SynergyDetection.ExploratoryPairBitRanges); + Assert.Equal(["mid-fidelity", "low-fidelity"], config.SynergyDetection.ExploratoryPairContextStrata); + Assert.True(config.SynergyDetection.ContextScopedRuleApplicationEnabled); + Assert.Equal(2, config.SynergyDetection.MaxNonRuleGroupContextMismatches); + } +} diff --git a/tests/MagicQuant.Tests/SynergyTransferPlanningTests.cs b/tests/MagicQuant.Tests/SynergyTransferPlanningTests.cs new file mode 100644 index 0000000..83dddab --- /dev/null +++ b/tests/MagicQuant.Tests/SynergyTransferPlanningTests.cs @@ -0,0 +1,100 @@ +using MagicQuant.Models; +using MagicQuant.Services; +using MQ.DB; +using MQ.DB.Models; +using Xunit; + +namespace MagicQuant.Tests; + +[Collection(AnomalyContextScopeCollection.Name)] +public sealed class SynergyTransferPlanningTests +{ + [Fact] + public void ControlledTransfer_ChangesOneGroupInsideExplicitLowFidelityBlanket() + { + var priorUnusedGroups = Cache.UnusedTensorGroups.ToList(); + + try + { + Cache.UnusedTensorGroups.Clear(); + + bool built = AnomalyWorkflowService.TryBuildControlledTransferConfig( + BaselineQuants.IQ3_S.UniqueId, + [(TReg.Embeddings.UniqueId, BaselineQuants.IQ4_NL.UniqueId)], + out var reference, + out var probe, + out var changed); + + Assert.True(built); + var movement = new QuantFidelityComparerService(); + Assert.All(movement.ActiveGroups, group => + Assert.Equal(BaselineQuants.IQ3_S.UniqueId, movement.EffectiveQuantId(reference, group))); + Assert.Equal(BaselineQuants.IQ4_NL.UniqueId, movement.EffectiveQuantId(probe, TReg.Embeddings)); + Assert.All(movement.ActiveGroups.Where(x => x.UniqueId != TReg.Embeddings.UniqueId), group => + Assert.Equal(BaselineQuants.IQ3_S.UniqueId, movement.EffectiveQuantId(probe, group))); + + var groupChange = Assert.Single(changed); + Assert.Equal(TReg.Embeddings.UniqueId, groupChange.Group.UniqueId); + Assert.Equal(QuantMovementKind.Upgrade, groupChange.Movement); + } + finally + { + Cache.UnusedTensorGroups.Clear(); + Cache.UnusedTensorGroups.AddRange(priorUnusedGroups); + } + } + + [Fact] + public void ControlledTransfer_SkipsTemplateStateEqualToBlanket() + { + bool built = AnomalyWorkflowService.TryBuildControlledTransferConfig( + BaselineQuants.Q4_K_M.UniqueId, + [(TReg.Embeddings.UniqueId, BaselineQuants.Q4_K_M.UniqueId)], + out _, + out _, + out var changed); + + Assert.False(built); + Assert.Empty(changed); + } + + [Fact] + public void ControlledRankPair_ComparesTwoRecipesInsideSameLowFidelityBlanket() + { + var priorUnusedGroups = Cache.UnusedTensorGroups.ToList(); + + try + { + Cache.UnusedTensorGroups.Clear(); + + bool built = AnomalyWorkflowService.TryBuildControlledRankPairConfig( + BaselineQuants.IQ3_S.UniqueId, + TReg.Embeddings, + BaselineQuants.Q4_K_M.UniqueId, + BaselineQuants.IQ4_NL.UniqueId, + out var reference, + out var probe, + out var changed); + + Assert.True(built); + var movement = new QuantFidelityComparerService(); + Assert.Equal(BaselineQuants.Q4_K_M.UniqueId, movement.EffectiveQuantId(reference, TReg.Embeddings)); + Assert.Equal(BaselineQuants.IQ4_NL.UniqueId, movement.EffectiveQuantId(probe, TReg.Embeddings)); + Assert.All(movement.ActiveGroups.Where(x => x.UniqueId != TReg.Embeddings.UniqueId), group => + { + Assert.Equal(BaselineQuants.IQ3_S.UniqueId, movement.EffectiveQuantId(reference, group)); + Assert.Equal(BaselineQuants.IQ3_S.UniqueId, movement.EffectiveQuantId(probe, group)); + }); + + var groupChange = Assert.Single(changed); + Assert.Equal(BaselineQuants.Q4_K_M.UniqueId, groupChange.ReferenceQuantId); + Assert.Equal(BaselineQuants.IQ4_NL.UniqueId, groupChange.CandidateQuantId); + Assert.Equal(QuantMovementKind.LateralOrEquivalent, groupChange.Movement); + } + finally + { + Cache.UnusedTensorGroups.Clear(); + Cache.UnusedTensorGroups.AddRange(priorUnusedGroups); + } + } +} diff --git a/tests/MagicQuant.Tests/YamlDiagnosticsTests.cs b/tests/MagicQuant.Tests/YamlDiagnosticsTests.cs new file mode 100644 index 0000000..28ab89e --- /dev/null +++ b/tests/MagicQuant.Tests/YamlDiagnosticsTests.cs @@ -0,0 +1,52 @@ +using MagicQuant.Configuration; +using Xunit; + +namespace MagicQuant.Tests; + +public sealed class YamlDiagnosticsTests +{ + [Fact] + public void Unknown_nested_and_sequence_keys_report_the_full_setting_path() + { + var warnings = YamlConfigurationDiagnostics.Inspect(""" + paths: + modle_dir: /models/test + baselines: + custom_repositories: + - repo_id: owner/model + revison: main + """); + Assert.Contains(warnings, x => x.Contains("paths.modle_dir")); + Assert.Contains(warnings, x => x.Contains("baselines.custom_repositories[0].revison")); + } + + [Fact] + public void Frontmatter_and_gpu_dictionary_keys_are_not_schema_properties() + { + Assert.Empty(YamlConfigurationDiagnostics.Inspect(""" + readme: + frontmatter: + arbitrary_metadata: [hello, world] + hardware: + gpu_memory_limits_gb: + 0: 12 + """)); + } + + [Fact] + public void Removed_destructive_key_is_rejected_but_comments_are_not_options() + { + Assert.Throws(() => YamlConfigurationDiagnostics.Inspect("flags:\n force_relearn_baseline_tensor_mappings: true")); + Assert.Empty(YamlConfigurationDiagnostics.Inspect("# force_relearn_baseline_tensor_mappings was removed\npaths: {}")); + } + + [Fact] + public void Invalid_shape_and_nonfinite_numbers_fail_with_setting_names() + { + var config = MagicQuantYamlConfig.CreateDefault(); + config.Prediction = null!; + Assert.Contains("prediction", Assert.Throws(() => ConfigurationShapeValidator.Validate(config)).Message); + config.Prediction = new RuntimePredictionConfig { DefaultBitStressThreshold = double.NaN }; + Assert.Contains("default_bit_stress_threshold", Assert.Throws(() => ConfigurationShapeValidator.Validate(config)).Message); + } +} diff --git a/tests/MagicQuant.Tests/packages.lock.json b/tests/MagicQuant.Tests/packages.lock.json new file mode 100644 index 0000000..6439e93 --- /dev/null +++ b/tests/MagicQuant.Tests/packages.lock.json @@ -0,0 +1,359 @@ +{ + "version": 1, + "dependencies": { + "net10.0": { + "Microsoft.NET.Test.Sdk": { + "type": "Direct", + "requested": "[17.14.1, )", + "resolved": "17.14.1", + "contentHash": "HJKqKOE+vshXra2aEHpi2TlxYX7Z9VFYkr+E5rwEvHC8eIXiyO+K9kNm8vmNom3e2rA56WqxU+/N9NJlLGXsJQ==", + "dependencies": { + "Microsoft.CodeCoverage": "17.14.1", + "Microsoft.TestPlatform.TestHost": "17.14.1" + } + }, + "xunit": { + "type": "Direct", + "requested": "[2.9.2, )", + "resolved": "2.9.2", + "contentHash": "7LhFS2N9Z6Xgg8aE5lY95cneYivRMfRI8v+4PATa4S64D5Z/Plkg0qa8dTRHSiGRgVZ/CL2gEfJDE5AUhOX+2Q==", + "dependencies": { + "xunit.analyzers": "1.16.0", + "xunit.assert": "2.9.2", + "xunit.core": "[2.9.2]" + } + }, + "xunit.runner.visualstudio": { + "type": "Direct", + "requested": "[2.8.2, )", + "resolved": "2.8.2", + "contentHash": "vm1tbfXhFmjFMUmS4M0J0ASXz3/U5XvXBa6DOQUL3fEz4Vt6YPhv+ESCarx6M6D+9kJkJYZKCNvJMas1+nVfmQ==" + }, + "Blake3": { + "type": "Transitive", + "resolved": "2.2.0", + "contentHash": "RM6sZLZDx2wGi00aTj9s2jUcrI4s9dS2ibcT7lSujpUpBGp+TLf71F3XdBKJyYSxHnZ+FL7Dm36Pl0Y+cfcXvw==" + }, + "DuckDB.NET.Bindings.Full": { + "type": "Transitive", + "resolved": "1.4.3", + "contentHash": "hZwm0zTKJ5HdUGKcase2JX52Lquyh7dCUFweECvR877QEA2gF8gSl3qrtb71BvRlgZ7pfjh0bRBCiAKOJMLE+A==" + }, + "DuckDB.NET.Data.Full": { + "type": "Transitive", + "resolved": "1.4.3", + "contentHash": "tg1FWmePN+k536O1cx2VhKWa3xT7DXrcGg4kGgiSWQyur9UWwZ2i2YMpD+XVYv1ARozyy1Tt7OW2cMrNPoPj9g==", + "dependencies": { + "DuckDB.NET.Bindings.Full": "1.4.3" + } + }, + "LibGit2Sharp": { + "type": "Transitive", + "resolved": "0.31.0", + "contentHash": "b3+UfV7LjKMjAHWwl7VawejiOv2gJIC6dTCA/S0puLTHACAA/Oeb5JJmWUQMeyH/T/WR/LaIK8bk2RbdFnrZvg==", + "dependencies": { + "LibGit2Sharp.NativeBinaries": "[2.0.323]" + } + }, + "LibGit2Sharp.NativeBinaries": { + "type": "Transitive", + "resolved": "2.0.323", + "contentHash": "Kg+fJGWhGj5qRXG0Ilj4ddhuodGXZg57yhfX6OVUDR0M2DKg/UR42/d74+qv5l1qotc1qJilo/ho7xQnULP6yA==" + }, + "Microsoft.CodeCoverage": { + "type": "Transitive", + "resolved": "17.14.1", + "contentHash": "pmTrhfFIoplzFVbhVwUquT+77CbGH+h4/3mBpdmIlYtBi9nAB+kKI6dN3A/nV4DFi3wLLx/BlHIPK+MkbQ6Tpg==" + }, + "Microsoft.Data.Sqlite": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "7je7UELzm131GiLYc4PpZvfKXIgIyzPM+v+tjcd/nbnuWRfgcONYKzDTqJlURxwVCFsVnlpmq6y6yn4qvR8QXQ==", + "dependencies": { + "Microsoft.Data.Sqlite.Core": "10.0.11", + "SQLitePCLRaw.bundle_e_sqlite3": "2.1.12", + "SQLitePCLRaw.core": "2.1.12" + } + }, + "Microsoft.Data.Sqlite.Core": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "hubA20AGenQ4Sx0ElWaPpB8DISjXpdx463+1zOGRslsT0e/t/06ITv+pHsop8CcJ0d8PZLfgnT7juCDVD79Dkw==", + "dependencies": { + "SQLitePCLRaw.core": "2.1.12" + } + }, + "Microsoft.EntityFrameworkCore": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "VOSGU8en6HZJs8t7UMFN+9vGcRgVOOn6fA44Ngcg2NyvJ3P1KE94iAb0XzaVaGhXGtt+qaM/VtEn0/hzluQJeg==", + "dependencies": { + "Microsoft.EntityFrameworkCore.Abstractions": "10.0.11", + "Microsoft.EntityFrameworkCore.Analyzers": "10.0.11", + "Microsoft.Extensions.Caching.Memory": "10.0.11", + "Microsoft.Extensions.Logging": "10.0.11" + } + }, + "Microsoft.EntityFrameworkCore.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "6auJR+9+9VunznKfH7WGrHMrnrmA0F7JZ22EXzwXvVhjfnbu9Xq7NSIWaOf3KJsOanM2qf5ajJ2JR5TlcPZTLA==" + }, + "Microsoft.EntityFrameworkCore.Analyzers": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "Bv7X4wSSnzCQED9WYXKJ8fwgyvKwf0xZM1GO8xkf6CF9zl+UBnvjxmcPnokJRy0JKjc1SlHSzzhx1HcL4jitTQ==" + }, + "Microsoft.EntityFrameworkCore.Relational": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "grznnTJgEYxaWpdKAsTzg6j+89jHgCXWYp+QGtlX5O92+w/VuhWM6JLPYb+uw8M9VhGUvOTsO76dYOy9vNPd5Q==", + "dependencies": { + "Microsoft.EntityFrameworkCore": "10.0.11", + "Microsoft.Extensions.Caching.Memory": "10.0.11", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.Logging": "10.0.11" + } + }, + "Microsoft.EntityFrameworkCore.Sqlite": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "jc7iVrhQyInR3loraMESfEFaFOtQOB1mRKHjX6QYC9o7YDbfMNbAPnIwlpffnFwhXd6/27FKaaV+sWSoLd4F1g==", + "dependencies": { + "Microsoft.EntityFrameworkCore.Sqlite.Core": "10.0.11", + "Microsoft.Extensions.Caching.Memory": "10.0.11", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.DependencyModel": "10.0.11", + "Microsoft.Extensions.Logging": "10.0.11", + "SQLitePCLRaw.bundle_e_sqlite3": "2.1.12", + "SQLitePCLRaw.core": "2.1.12" + } + }, + "Microsoft.EntityFrameworkCore.Sqlite.Core": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "ywTQKt32xnVhCzjEQAqFufpEyXkOUfvW/EC/s4xnS8Xaor2xXE+TMUyzhgACqXtZEU5IR95y94RDzHto55Fx7w==", + "dependencies": { + "Microsoft.Data.Sqlite.Core": "10.0.11", + "Microsoft.EntityFrameworkCore.Relational": "10.0.11", + "Microsoft.Extensions.Caching.Memory": "10.0.11", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.11", + "Microsoft.Extensions.DependencyModel": "10.0.11", + "Microsoft.Extensions.Logging": "10.0.11", + "SQLitePCLRaw.core": "2.1.12" + } + }, + "Microsoft.Extensions.Caching.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "vUl798SmruTqqlt/xH2gDk3tJlhk6k3HdOXAHirlRfbNKDym4g/kRpUL9S4sl6F6FsOTOMW+ZsDapqlZMOOiEw==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Caching.Memory": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "el1g0mBEbDBGY2bT9mcSfrTWO8QlPdq2nOCnvQugioOFwHV+bVBMeiakoI0dNOdj8d6Hi9K6HY2xzRUWJiDR3w==", + "dependencies": { + "Microsoft.Extensions.Caching.Abstractions": "10.0.11", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11", + "Microsoft.Extensions.Logging.Abstractions": "10.0.11", + "Microsoft.Extensions.Options": "10.0.11", + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "fVi053xdpda9Em7vSkmgVxO/PtgC2m78ekReKWsgcyskqY0U82Bz/MONwxpGzI0hElYKJfw+fupqMVeKW3fSaA==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "PSmotV19c7E3lKed++uYo1kSiXFI+uTl37CBSrhq+CfLC3FCHjG7R91+xPnNehQfHS1b0Tzo/CCLPWH3qaEheg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "/a1aJz4m7ylhEDf25ugQChLQoN5XwoGjWw/BoR/ZWWKsO1v4DdJElS1uyngahz4B/eOzjFk1KNTkarRLE5wsIg==" + }, + "Microsoft.Extensions.DependencyModel": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "PJPtFYsZ+r+uz9qqXWUTEyKeJ1EiBGIJtqavkg9ZXijjGSFAk4Fgi5sqIxj+uAyLZwEKgexDUQXhWhvU6l3+og==" + }, + "Microsoft.Extensions.Logging": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "nUOJwgFkSiLHiVGFpU22pIJtuWYewuSYQ3JVuP/gdK8ASMT807Px+TYQiRWs6uSsOmoyFTaVCwKXTasczV6BpA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "10.0.11", + "Microsoft.Extensions.Logging.Abstractions": "10.0.11", + "Microsoft.Extensions.Options": "10.0.11" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "Ljd0Uxoq5XpScD2Bg0nM/r3mwx7Ao5Uq24eo2ARxbGvqJ7Zht6rt2cJtwVRH4Cv+1ZVMdXz6TB43KbpmsxRrvQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11" + } + }, + "Microsoft.Extensions.Options": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "eY1GAKcTfD2maP27J84X9IovT3yjHJ2dVDzPmDg6/XqYvt3jMzJhtfQCLjG9pVsZGAd+8DQ2QrjaDcs2+VQLGw==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.11", + "Microsoft.Extensions.Primitives": "10.0.11" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "SXcz+kF+4Oo9b1+55zntpJFYfwb1jw66ioxptyNOOTDc8g2FHnBFWjZpsWfCvZIhzr0x+4e2trVTs4OKwQfBtw==" + }, + "Microsoft.TestPlatform.ObjectModel": { + "type": "Transitive", + "resolved": "17.14.1", + "contentHash": "xTP1W6Mi6SWmuxd3a+jj9G9UoC850WGwZUps1Wah9r1ZxgXhdJfj1QqDLJkFjHDCvN42qDL2Ps5KjQYWUU0zcQ==" + }, + "Microsoft.TestPlatform.TestHost": { + "type": "Transitive", + "resolved": "17.14.1", + "contentHash": "d78LPzGKkJwsJXAQwsbJJ7LE7D1wB+rAyhHHAaODF+RDSQ0NgMjDFkSA1Djw18VrxO76GlKAjRUhl+H8NL8Z+Q==", + "dependencies": { + "Microsoft.TestPlatform.ObjectModel": "17.14.1", + "Newtonsoft.Json": "13.0.3" + } + }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.3", + "contentHash": "HrC5BXdl00IP9zeV+0Z848QWPAoCr9P3bDEZguI+gkLcBKAOxix/tLEAAHC+UvDNPv4a2d18lOReHMOagPa+zQ==" + }, + "Spectre.Console": { + "type": "Transitive", + "resolved": "0.54.0", + "contentHash": "StDXCFayfy0yB1xzUHT2tgEpV1/HFTiS4JgsAQS49EYTfMixSwwucaQs/bIOCwXjWwIQTMuxjUIxcB5XsJkFJA==" + }, + "SQLitePCLRaw.bundle_e_sqlite3": { + "type": "Transitive", + "resolved": "2.1.12", + "contentHash": "mAgscpQMLw5/nfA1Q5oJVAT29yROUo1ifZGbbTpx/lwZpSxMUGoYbKfmvdm8oXER+RzxqBmmQzeBEVKfeHv2nw==", + "dependencies": { + "SQLitePCLRaw.lib.e_sqlite3": "2.1.12", + "SQLitePCLRaw.provider.e_sqlite3": "2.1.12" + } + }, + "SQLitePCLRaw.core": { + "type": "Transitive", + "resolved": "2.1.12", + "contentHash": "ETpNw9DY3ckWLgRRAeCHj+GKOuPi61aeczkXhgHexUvqoZBAYg8RYESE2J7O1M7+o6QbdSEZwrw9bfqztUVWXg==" + }, + "SQLitePCLRaw.lib.e_sqlite3": { + "type": "Transitive", + "resolved": "2.1.12", + "contentHash": "fWi8Dbknuhgg72fWinIdjXVaqO1hHL4YBBwVLnr7e1c9TAZwJ0QE38j9syW1hwx6HaqEVTwI+O07WPdZn8Rp0w==" + }, + "SQLitePCLRaw.provider.e_sqlite3": { + "type": "Transitive", + "resolved": "2.1.12", + "contentHash": "W3oH4XIfCzFrgUSDKHhN6N+dgzA5YHOR2VxX8GB6Qy7CyrJJgxPEG8NirgYWlPQC5P2jz2knSsexWu4tDUL33g==", + "dependencies": { + "SQLitePCLRaw.core": "2.1.12" + } + }, + "System.CodeDom": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "v40pNeBoZTYsiVxz+PzyZmmIr2JIhpK4VsFpQqZSZCXa51PDlNXIN2ESm8kDU0voZYVfLhxF9HvmBsxCJmkiRg==" + }, + "System.Management": { + "type": "Transitive", + "resolved": "10.0.11", + "contentHash": "xyNn8KGbWI88LoUwg3rB8qcpFFST6dr8Ro/qS8GBu2GOwR0v7J82kVFHTiiPtvEKS79VbMTxs/sIKQ+Cq1Zs1g==", + "dependencies": { + "System.CodeDom": "10.0.11" + } + }, + "xunit.abstractions": { + "type": "Transitive", + "resolved": "2.0.3", + "contentHash": "pot1I4YOxlWjIb5jmwvvQNbTrZ3lJQ+jUGkGjWE3hEFM0l5gOnBWS+H3qsex68s5cO52g+44vpGzhAt+42vwKg==" + }, + "xunit.analyzers": { + "type": "Transitive", + "resolved": "1.16.0", + "contentHash": "hptYM7vGr46GUIgZt21YHO4rfuBAQS2eINbFo16CV/Dqq+24Tp+P5gDCACu1AbFfW4Sp/WRfDPSK8fmUUb8s0Q==" + }, + "xunit.assert": { + "type": "Transitive", + "resolved": "2.9.2", + "contentHash": "QkNBAQG4pa66cholm28AxijBjrmki98/vsEh4Sx5iplzotvPgpiotcxqJQMRC8d7RV7nIT8ozh97957hDnZwsQ==" + }, + "xunit.core": { + "type": "Transitive", + "resolved": "2.9.2", + "contentHash": "O6RrNSdmZ0xgEn5kT927PNwog5vxTtKrWMihhhrT0Sg9jQ7iBDciYOwzBgP2krBEk5/GBXI18R1lKvmnxGcb4w==", + "dependencies": { + "xunit.extensibility.core": "[2.9.2]", + "xunit.extensibility.execution": "[2.9.2]" + } + }, + "xunit.extensibility.core": { + "type": "Transitive", + "resolved": "2.9.2", + "contentHash": "Ol+KlBJz1x8BrdnhN2DeOuLrr1I/cTwtHCggL9BvYqFuVd/TUSzxNT5O0NxCIXth30bsKxgMfdqLTcORtM52yQ==", + "dependencies": { + "xunit.abstractions": "2.0.3" + } + }, + "xunit.extensibility.execution": { + "type": "Transitive", + "resolved": "2.9.2", + "contentHash": "rKMpq4GsIUIJibXuZoZ8lYp5EpROlnYaRpwu9Zr0sRZXE7JqJfEEbCsUriZqB+ByXCLFBJyjkTRULMdC+U566g==", + "dependencies": { + "xunit.extensibility.core": "[2.9.2]" + } + }, + "YamlDotNet": { + "type": "Transitive", + "resolved": "17.0.1", + "contentHash": "qVir5fehR/W5nTJyoJUibypETXaW4iRAF9cQa0FQIC9TJ3VC0qDOwm4o/RxANewj8KzPF8WMF2abBfUgi6LC4w==" + }, + "magicquant": { + "type": "Project", + "dependencies": { + "Blake3": "[2.2.0, )", + "DuckDB.NET.Data.Full": "[1.4.3, )", + "LibGit2Sharp": "[0.31.0, )", + "MQ.DB": "[1.0.0, )", + "Spectre.Console": "[0.54.0, )", + "System.Management": "[10.0.11, )", + "YamlDotNet": "[17.0.1, )" + } + }, + "magicquant.processfixture": { + "type": "Project" + }, + "mq.db": { + "type": "Project", + "dependencies": { + "Microsoft.Data.Sqlite": "[10.0.11, )", + "Microsoft.EntityFrameworkCore": "[10.0.11, )", + "Microsoft.EntityFrameworkCore.Sqlite": "[10.0.11, )", + "YamlDotNet": "[17.0.1, )" + } + } + } + } +} \ No newline at end of file diff --git a/wiki/index.md b/wiki/index.md index 04fb1a6..9da77c4 100644 --- a/wiki/index.md +++ b/wiki/index.md @@ -1,5 +1,7 @@ # MagicQuant v2 Documentation +For installation and CLI usage see the [project README](../README.md) and [program docs](../docs/index.md). The [research overview](overview.md) retains worked examples and motivation. + MagicQuant is a benchmark-driven GGUF evaluation and hybrid-discovery system. These pages explain not only what the pipeline does, but why its search, measurement, and survivor rules exist. ## Start Here diff --git a/wiki/overview.md b/wiki/overview.md new file mode 100644 index 0000000..ed33a4a --- /dev/null +++ b/wiki/overview.md @@ -0,0 +1,224 @@ +# MagicQuant (v2.0) + +**MagicQuant is a benchmark-driven GGUF evaluation and hybrid-discovery system.** + +> **Which quantized models are actually worth using at each size?** + +Most quant releases give you a pile of files, AKA: Q8, Q6, Q5, Q4, and leave you to guess. MagicQuant replaces that guesswork with benchmarks, tensor-group probing, mixed hybrid GGUF builds when they are worth it, and a final survivor list built around meaningful size/fidelity tradeoffs. + +--- + +## What MagicQuant Does + +MagicQuant takes the messy quantization space and turns it into a judged survivor list. + +It tests standard baselines, learns from external quant strategies, and builds mixed tensor-group hybrids when there may be a better size/fidelity trade hiding between normal quant levels. + +Then it validates the results. + +MagicQuant does not assume hybrids are better. It does not assume baselines are safe. Every option has to earn its slot. + +A final MagicQuant release is meant to show: + +* what is smallest +* what is safest +* what is meaningfully in-between +* what was removed as redundant or not worth the damage +* and what the real benchmark numbers say + +If a model survives MagicQuant, it survived because the trade was worth showing. + +--- + +## Example + +The following example is Qwen3-4B-2507-Instruct going through MagicQuants pipeline and the final results: + +| Name | Provider | Quant Family | KLD | Size (GB) | +| ----------------------------------------------------------------------------------------- | ---------- | ------------ | -------: | --------: | +| LM-Q8_0 | llama.cpp | Q8_0 | 0.001339 | 3.99 | +| MQ-Q6_K_1 | MagicQuant | Q6_K | 0.001817 | 3.58 | +| UD-Q6_K_XL | Unsloth | UD-Q6_K_XL | 0.002111 | 3.41 | +| LM-Q6_K | llama.cpp | Q6_K | 0.004640 | 3.08 | +| [MQ-Q5_K_1](#winner-notes "Replaced: MQ-Q5_K") | MagicQuant | Q5_K | 0.006632 | 2.88 | +| [UD-Q5_K_XL](#winner-notes "Replaced: LM-Q5_K, LM-Q5_K_S") | Unsloth | UD-Q5_K_XL | 0.009839 | 2.73 | +| [MQ-Q4_K_M_1](#winner-notes "Replaced: MQ-Q4_K_M, UD-Q4_K_XL, LM-Q4_K_M + 1 more") | MagicQuant | Q4_K_M | 0.020346 | 2.44 | +| [LM-Q4_K_S](#winner-notes "Replaced: LM-IQ4_NL") | llama.cpp | Q4_K_S | 0.029803 | 2.22 | +| LM-IQ4_XS | llama.cpp | IQ4_XS | 0.031300 | 2.11 | +| UD-Q3_K_XL | Unsloth | UD-Q3_K_XL | 0.072278 | 1.98 | + +## Winner notes + +The table above includes a mix of standard llama.cpp quantizations, Unsloth Dynamic GGUF models, and MagicQuant hybrids. + +In some cases, dominance is absolute. For example, Unsloth’s **Q5_K_XL** fully replaces the standard llama.cpp **Q5_K**, as MagicQuant determined the baseline offered no meaningful tradeoff in comparison. + +More interesting are the hybrid outcomes. **MQ-Q4_K_M_1** emerged as a clear dominant variant, replacing multiple candidates simultaneously (_UD-Q4_K_XL, MQ-Q4_K_M, LM-Q4_K_M_). While baseline quants can sometimes achieve similar dominance, this case highlights a hybrid configuration that decisively outperformed across the board. + +**MQ-Q5_K_1** is another notable result. It leverages Unsloth’s learned tensor behavior (_Q5_K_XL_) within the `ffn_up_gate`, discovering a middle ground between **UD-Q5_K_XL** and **LM-Q6_K**. The result is a hybrid that achieves a disproportionately large KLD improvement relative to the additional size cost, exceeding a simple linear tradeoff. + +The table below breaks down these MagicQuant hybrids by tensor group, showing the assigned quantization for each, whether derived from llama.cpp baselines or Unsloth’s learned tensor mappings. + +| Name | embeddings | attn_q | attn_kv | attn_output | ffn_up_gate | ffn_down | +| ----------- | ---------- | ------ | ------- | ----------- | ----------- | -------- | +| MQ-Q6_K_1 | Q8_0 | Q8_0 | Q8_0 | Q8_0 | Q6_K | Q8_0 | +| MQ-Q5_K_1 | Q8_0 | Q5_K | Q8_0 | Q6_K | UD-Q5_K_XL | Q5_K_S | +| MQ-Q4_K_M_1 | Q8_0 | Q5_K | Q8_0 | Q6_K | IQ4_XS | IQ4_XS | + +--- + +## Nonlinear Wins + +MagicQuant does not look for simple "winners" in sub space between baselines. Instead it only allows nonlinear trade wins. Documentation presented later goes further into detail on this subject, but here's the TLDR: + +Imagine a graph like this: +``` +Size → +| +| Q6 +| / +| / +| Q5 +| / +|Q4 ++---------------- +``` + +A nonlinear win looks like: +``` + Q6 + / + / ← MQ-Q5_K_1 (above the line) + Q5 + / +Q4 +``` + +That hybrid sits above the straight line between Q4 and Q5. + +Meaning: +👉 It’s a **more efficient trade** than the normal step-up + +This is what MagicQuant calls a "nonlinear trade/win" when such wordage is used. + +--- + +## Deeper Understanding + +For a deeper dive into MagicQuant and how it works, the [wiki index](https://github.com/magiccodingman/MagicQuant/blob/main/wiki/index.md) is a good place to start. + +When you see a MagicQuant hybrid, it’s not just a “Q4.5” sitting somewhere between Q4 and Q5. It represents a discovered configuration where the **KLD reduction is non-linear relative to the size increase**, a genuinely better trade space. Not universally “better” than everything else, but a variant that earned its place through measurable advantage. + +Whether the winner is a hybrid or a pure baseline from llama.cpp or Unsloth, any quant that removes another from the final selection does so because its dominance made the alternative no longer worth considering. + +The goal is not to flood the space with near-duplicates offering negligible KLD gains for minimal size differences, nor to claim superiority for the sake of it. In fact, that’s explicitly what MagicQuant avoids. + +MagicQuant is built around transparency, honesty, maintainability, and most importantly trust. As it evaluates new architectures and quant families, it doesn’t invent quantization schemes in isolation. Instead, it learns from proven tensor assignments provided by trusted sources like llama.cpp and Unsloth. If those baselines are stable, MagicQuant operates within that same safe space, extending rather than reinventing. + +Historical sources expand that tensor vocabulary; they do not vote on the current winner. MagicQuant pins the source revision, rebuilds the available recipes under current controlled conditions, and relearns their effects rather than replaying an old final mixture. + +That said, the system is designed to adapt. Edge cases can exist, but the architecture is intentionally flexible to handle them. + +### How MagicQuant Works + +``` + ┌────────────────────────────┐ + │ Input Quantized Models │ + │ ───────────────────────── │ + │ llama.cpp / Unsloth / etc │ + └────────────┬──────────────┘ + │ + │ Inspect tensors + ▼ + ┌────────────────────────────┐ + │ Tensor Extraction Layer │ + │ ───────────────────────── │ + │ - Read all tensors │ + │ - Detect quant types │ + │ - Capture F32 / BF16 │ + └────────────┬──────────────┘ + │ + │ Group by role + ▼ + ┌────────────────────────────┐ + │ Tensor Group Mapping │ + │ ───────────────────────── │ + │ embeddings │ + │ attn_q / attn_kv / output │ + │ ffn_up_gate / ffn_down │ + │ lm_head / moe_* │ + └────────────┬──────────────┘ + │ + │ Learn configs + ▼ + ┌────────────────────────────┐ + │ Learned Config Library │ + │ ───────────────────────── │ + │ "Q5_K attn_q pattern" │ + │ "UD-Q5_K_XL ffn pattern" │ + │ etc │ + └────────────┬──────────────┘ + │ + │ Normalize external configs + ▼ + ┌────────────────────────────┐ + │ Controlled Rebuild Layer │ + │ ───────────────────────── │ + │ - Apply configs to BF16 │ + │ - Use MagicQuant imatrix │ + │ - Equal comparison ground │ + └────────────┬──────────────┘ + │ + │ Feed into + ▼ + ┌────────────────────────────┐ + │ Hybrid Construction Engine │ + │ ───────────────────────── │ + │ Mix tensor groups across │ + │ learned configurations │ + └────────────┬──────────────┘ + │ + │ Evaluate candidates + ▼ + ┌────────────────────────────┐ + │ Prediction + Isolation │ + │ ───────────────────────── │ + │ - Group-level testing │ + │ - Rank-safe prediction │ + │ - Controlled context tests │ + └────────────┬──────────────┘ + │ + │ Build real GGUF + ▼ + ┌────────────────────────────┐ + │ Benchmark Layer │ + │ ───────────────────────── │ + │ - KLD (primary) │ + │ - PPL (secondary) │ + │ - Measured GPU scheduling │ + └────────────┬──────────────┘ + │ + │ Final decision + ▼ + ┌────────────────────────────┐ + │ Survivor Selection │ + │ ───────────────────────── │ + │ - Dominance pruning │ + │ - Nonlinear winners │ + │ - Spacing collapse │ + └────────────────────────────┘ +``` + +The controlled context tests check whether a promising group choice still behaves the same way when the surrounding model moves from a Q4-or-better regime into more aggressive compression. They are bounded and evidence-driven because exhaustive context testing would recreate the full combinatorial problem. + +GPU scheduling is also measured rather than assumed. A large benchmark can use multiple GPUs in one shared process, while batches of smaller candidates can run concurrently on independent GPUs when that produces higher aggregate throughput. + +The final release is a curated survivor menu. Research campaigns and cross-run audits should preserve the full nondominated evidence frontier before applying spacing, so that a presentation decision does not erase valid results. + +## Deep Dive Documentation + +- [Wiki index](./index.md) +- [Prediction Engine](./docs/Prediction-Engine.md) +- [Regime-Aware Tensor Search](./docs/Regime-Aware-Search.md) +- [GPU Benchmark Scheduling](./docs/GPU-Benchmark-Scheduling.md) +- [Pareto Archives and Reproducibility](./docs/Pareto-Archives-And-Reproducibility.md)