Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
120 changes: 82 additions & 38 deletions .github/workflows/build-and-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@ name: Build and Release
# Every push to master builds the module and publishes a release, tagged with
# the build date. Pushing a v* tag by hand publishes under that tag instead.
#
# A pull request runs the same build but stops short of publishing, which is
# how a risky change gets verified without leaving half-working commits on
# master. Nothing here can be built or tested on Linux, so CI is the only
# feedback available before merging.
#
# This does not recurse. The tag created below is pushed with GITHUB_TOKEN, and
# GitHub deliberately does not start a new workflow run for events raised by
# that token — so "push to master -> create v* tag" does not re-enter the
Expand All @@ -11,30 +16,35 @@ on:
push:
branches: [master]
tags: ['v*']
pull_request:
branches: [master]
workflow_dispatch:

# The date-based sequence number is derived from tags that already exist, so
# two runs racing each other could pick the same number. Keep runs serialised
# and let queued ones finish rather than cancelling them — every push is meant
# to produce its own release.
# to produce its own release. Pull requests get their own group per branch so
# that iterating on a PR does not queue up behind unrelated runs.
concurrency:
group: build-and-release
cancel-in-progress: false
group: build-and-release-${{ github.event_name == 'pull_request' && github.ref || 'master' }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

permissions:
contents: write # create tags and releases

env:
SOLUTION: ImageStore.sln
PROJECT: ImageStore/ImageStore.csproj
CONFIGURATION: Release
BUILD_OUTPUT: ImageStore\bin\Release
# dotnet publish, not build: it gathers the full runtime dependency set into
# one directory, which is what a PowerShell module folder has to contain.
BUILD_OUTPUT: publish
# Date-based tags follow this timezone rather than the runner's UTC clock, so
# a tag reads the same date the commit was authored in.
TAG_TIMEZONE: China Standard Time

jobs:
build:
# .NET Framework 4.8.1 + Windows Forms — this cannot be built on Linux.
# net10.0-windows + Windows Forms — this cannot be built on Linux.
runs-on: windows-latest

steps:
Expand All @@ -48,8 +58,17 @@ jobs:
- name: Resolve version
id: version
shell: pwsh
env:
PR_NUMBER: ${{ github.event.pull_request.number }}
run: |
if ($env:GITHUB_REF -like 'refs/tags/*') {
if ($env:GITHUB_EVENT_NAME -eq 'pull_request') {
# Never published, so it gets a name that cannot be mistaken for a
# release if the artifact is downloaded and passed around.
$tag = "pr$($env:PR_NUMBER)-$($env:GITHUB_SHA.Substring(0, 7))"
$tagExists = 'false'
Write-Host "Pull request build, not a release: $tag"
}
elseif ($env:GITHUB_REF -like 'refs/tags/*') {
# A human pushed a tag — publish under it verbatim.
$tag = $env:GITHUB_REF -replace '^refs/tags/', ''
$tagExists = 'true'
Expand Down Expand Up @@ -84,59 +103,76 @@ jobs:
Add-Content -Path $env:GITHUB_OUTPUT -Value "tag=$tag"
Add-Content -Path $env:GITHUB_OUTPUT -Value "tag_exists=$tagExists"

- name: Set up MSBuild
uses: microsoft/setup-msbuild@v3

- name: Set up NuGet
uses: nuget/setup-nuget@v4

- name: Restore NuGet packages
shell: pwsh
# Legacy csproj with a PackageReference (Microsoft.PowerShell.5.ReferenceAssemblies);
# restore has to run before msbuild.
run: nuget restore $env:SOLUTION
- name: Set up .NET
uses: actions/setup-dotnet@v6
with:
dotnet-version: '10.0.x'

- name: Build
- name: Build and publish
shell: pwsh
run: |
msbuild $env:SOLUTION `
/p:Configuration=$env:CONFIGURATION `
'/p:Platform=Any CPU' `
/m /nologo /verbosity:minimal
dotnet publish $env:PROJECT `
--configuration $env:CONFIGURATION `
--output $env:BUILD_OUTPUT `
--nologo

- name: Verify build output
shell: pwsh
run: |
# Import-Module fails at load time if any of these is missing from the
# module folder, so publishing without them would ship a broken release.
# The BCL packages the 4.8.1 build needed (System.Buffers, System.Memory,
# System.Numerics.Vectors, System.Runtime.CompilerServices.Unsafe) are
# part of the framework on .NET 10 and no longer appear here.
$required = @(
'ImageStore.dll'
'ImageStore.deps.json'
'Shipwreck.Phash.dll'
'Shipwreck.Phash.Bitmaps.dll'
'System.Buffers.dll'
'System.Memory.dll'
'System.Numerics.Vectors.dll'
'System.Runtime.CompilerServices.Unsafe.dll'
'Microsoft.Data.SqlClient.dll'
)
$missing = $required | Where-Object { -not (Test-Path (Join-Path $env:BUILD_OUTPUT $_)) }
if ($missing) {
Write-Error "Missing from $($env:BUILD_OUTPUT): $($missing -join ', ')"
exit 1
}

# And the opposite check. Reference assemblies have no method bodies
# and must never be shipped — the PowerShell host supplies the real
# System.Management.Automation at run time. ExcludeAssets=runtime in
# the csproj keeps it out of the output; this is the net that catches
# a regression there, since a merely-too-large zip looks fine.
$banned = @('System.Management.Automation.dll')
$leaked = $banned | Where-Object { Test-Path (Join-Path $env:BUILD_OUTPUT $_) }
# SqlClient's native SNI library lives in a runtimes\ subdirectory. A
# module missing it loads fine and then fails on the first connection,
# so check it separately from the flat files above.
if (-not (Get-ChildItem $env:BUILD_OUTPUT -Recurse -File -Filter 'Microsoft.Data.SqlClient.SNI.dll')) {
Write-Error "Native SNI library missing from $($env:BUILD_OUTPUT)\runtimes"
exit 1
}

# And the opposite check: nothing belonging to the PowerShell host may
# ship. System.Management.Automation.dll is a reference assembly with no
# method bodies, and the native payload beside it is the host's own
# (including Linux and macOS libraries, in a Windows-only module).
# ExcludeAssets="runtime;native" in the csproj keeps them out; this is
# the net for a regression, since an oversized package still looks fine.
# Recursive, because the native files sit under runtimes\<rid>\native.
$banned = @(
'System.Management.Automation.dll'
'pwrshplugin.dll'
'PowerShell.Core.Instrumentation.dll'
'libpsl-native.*'
'getfilesiginforedist.dll'
)
$leaked = $banned |
ForEach-Object { Get-ChildItem $env:BUILD_OUTPUT -Recurse -File -Filter $_ } |
Select-Object -ExpandProperty Name -Unique
if ($leaked) {
Write-Error "Reference assembly leaked into $($env:BUILD_OUTPUT): $($leaked -join ', ')"
Write-Error "PowerShell host payload leaked into the package: $($leaked -join ', ')"
exit 1
}

Get-ChildItem $env:BUILD_OUTPUT | Format-Table Name, Length, LastWriteTime
# Recursive: Microsoft.Data.SqlClient carries a native SNI library under
# runtimes\win-*\native, and that layout has to survive into the package.
Get-ChildItem $env:BUILD_OUTPUT -Recurse -File |
Select-Object @{n='Path';e={$_.FullName.Substring((Resolve-Path $env:BUILD_OUTPUT).Path.Length + 1)}}, Length |
Sort-Object Path |
Format-Table -AutoSize

- name: Package
id: package
Expand All @@ -149,11 +185,16 @@ jobs:
run: |
$tag = $env:TAG

# The module: dlls only — the .pdb is not part of a release.
# The module: the whole publish output minus debug symbols. Copied
# wholesale rather than picking out *.dll, because the tree matters —
# ImageStore.deps.json drives dependency resolution, and SqlClient's
# native SNI library lives under runtimes\win-*\native. Flattening it
# produces a module that loads and then fails on first connect.
$moduleStage = Join-Path $env:RUNNER_TEMP 'module'
$moduleZip = "ImageStore-$tag.zip"
New-Item -ItemType Directory -Path $moduleStage -Force | Out-Null
Copy-Item "$env:BUILD_OUTPUT\*.dll" -Destination $moduleStage
Copy-Item "$env:BUILD_OUTPUT\*" -Destination $moduleStage -Recurse -Force
Get-ChildItem $moduleStage -Recurse -Include '*.pdb' | Remove-Item -Force
Compress-Archive -Path "$moduleStage\*" -DestinationPath $moduleZip -Force

# The database: shipped as its own asset. It is a one-time download
Expand All @@ -180,6 +221,9 @@ jobs:
if-no-files-found: error

- name: Publish release
# Pull requests stop here: they have proved the build and produced a
# downloadable artifact, which is the whole point of the PR run.
if: github.event_name != 'pull_request'
shell: pwsh
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Expand Down
101 changes: 59 additions & 42 deletions CLAUDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -17,39 +17,50 @@ Duplicate detection has two independent pipelines:

- **Same File** — byte-identical files, detected via SHA-1 (`SameFile` table).
- **Similar File** — visually similar images, detected via pHash
([Shipwreck.Phash](https://github.com/scegg/phash), a fork of pgrho/phash) (`SimilarFile` table).
([Shipwreck.Phash](https://github.com/pgrho/phash)) (`SimilarFile` table).

## Build and toolchain

| | |
|---|---|
| Target | .NET Framework **4.8.1**, `AnyCPU`, `OutputType=Library` |
| UI | Windows Forms (`System.Windows.Forms`, `System.Drawing`) |
| Project style | **Legacy (non-SDK) csproj**, ToolsVersion 15.0 |
| Build | Visual Studio 2017+ or `msbuild ImageStore.sln` on **Windows** |
| Target | `net10.0-windows`, `OutputType=Library` |
| UI | Windows Forms (`UseWindowsForms`) |
| Project style | SDK-style csproj (`Microsoft.NET.Sdk`) |
| Build | `dotnet build` / `dotnet publish ImageStore/ImageStore.csproj -c Release` |
| Host | **PowerShell 7.6+** only. 7.6 is the first release built on .NET 10; Windows PowerShell 5.1 cannot load the module at all. |
| Root namespace | `SecretNest.ImageStore` (assembly name `ImageStore`) |
| Tests | None. There is no test project. |
| CI | `.github/workflows/build-and-release.yml` — builds on `windows-latest` and publishes a release on every push to `master`. |
| CI | `.github/workflows/build-and-release.yml` — builds on `windows-latest`, publishes a release on every push to `master`, builds without publishing on pull requests. |

**The project cannot be built on Linux/macOS.** `dotnet build` will not work — this is a legacy
csproj targeting .NET Framework with WinForms. On a non-Windows machine, restrict work to source
edits, review, and documentation; do not claim a change compiles unless it was actually built on
Windows.
**It builds on Linux**, which is worth knowing because the WinForms target suggests otherwise:

**Adding or removing a source file requires editing `ImageStore/ImageStore.csproj` by hand.**
Legacy csproj has no glob includes. A new `.cs` file that is not listed in a `<Compile Include=.../>`
item is silently excluded from the build. WinForms files need the matching structure too:

```xml
<Compile Include="Area\MyForm.cs"><SubType>Form</SubType></Compile>
<Compile Include="Area\MyForm.Designer.cs"><DependentUpon>MyForm.cs</DependentUpon></Compile>
<EmbeddedResource Include="Area\MyForm.resx"><DependentUpon>MyForm.cs</DependentUpon></EmbeddedResource>
```
dotnet build ImageStore/ImageStore.csproj -c Release -p:EnableWindowsTargeting=true
dotnet publish ImageStore/ImageStore.csproj -c Release -o /tmp/out -p:EnableWindowsTargeting=true
```

That compiles and produces a complete publish tree, so compile errors, analyzer errors and the
exact package contents can all be checked locally. It cannot be *run* there — use it to verify
the build, then rely on CI or a Windows box for anything behavioural.

Dependencies come from NuGet; nothing is committed to the repo:

Third-party assemblies (`Shipwreck.Phash*.dll`, `System.Memory.dll`, `System.Buffers.dll`,
`System.Numerics.Vectors.dll`, `System.Runtime.CompilerServices.Unsafe.dll`) are **committed to the
repo** under `ImageStore/` and referenced by `HintPath`, not by NuGet. Only
`Microsoft.PowerShell.5.ReferenceAssemblies` comes from a `PackageReference`.
| Package | Note |
|---|---|
| `Shipwreck.Phash`, `Shipwreck.Phash.Bitmaps` | Upstream. A fork used to be vendored here for extra `GetCrossCorrelation` overloads; upstream 0.5.0 has them. |
| `Microsoft.Data.SqlClient` | Replaces `System.Data.SqlClient`, which has no .NET 10 story. |
| `System.Management.Automation` | `ExcludeAssets="runtime;native"` — see below. |

**`ExcludeAssets` on `System.Management.Automation` must keep both `runtime` and `native`.**
The PowerShell host supplies that assembly at run time, and the package's managed dll is a
reference assembly with no method bodies. Excluding only `runtime` still drags the host's *native*
payload into the output — `pwrshplugin.dll`, `PowerShell.Core.Instrumentation.dll`, and
`libpsl-native.so`/`.dylib` for Linux and macOS, in a Windows-only module. That mistake costs
~36 files and is invisible unless the package is opened.

Use `dotnet publish`, not `dotnet build`, when producing something to ship: the module needs its
full dependency closure, `ImageStore.deps.json`, and `runtimes/win-*/native/Microsoft.Data.SqlClient.SNI.dll`.
A package missing SNI loads fine and then fails on the first database connection.

## Repository layout

Expand Down Expand Up @@ -98,7 +109,9 @@ one connection is reused for the whole operation. Neither setting survives a Pow

### ADO.NET conventions

Raw `System.Data.SqlClient` throughout — no ORM, no EF, no async. The consistent shape is:
Raw `Microsoft.Data.SqlClient` throughout — no ORM, no EF, no async. Only the namespace differs
from the old `System.Data.SqlClient`; the type names are the same, and `SqlDbType` is still
`System.Data.SqlDbType`. The consistent shape is:

```csharp
var connection = DatabaseConnection.Current;
Expand Down Expand Up @@ -169,8 +182,9 @@ careful about adding per-file state to those structures.

`Select-ImageStoreSameFile` and `Resolve-ImageStoreSimilarFiles` call
`Application.EnableVisualStyles()` in `BeginProcessing` and then `ShowDialog()`. There is no
message loop of the module's own — the dialogs rely on the host thread being STA (the default in
the Windows PowerShell console host). The heaviest UI is
message loop of the module's own — the dialogs rely on the host thread being STA. `pwsh` is STA by
default on Windows (restored in 7.0-preview.3 precisely so WinForms and WPF work), but VS Code's
PowerShell Integrated Console is MTA, where these two cmdlets misbehave. The heaviest UI is
`SimilarFile/SimilarFileInGroupManager.cs`; `DoubleBufferedDataGridView` / `DoubleBufferedListView`
exist to keep large lists from flickering.

Expand Down Expand Up @@ -214,33 +228,36 @@ wipes both the pair table and those thresholds.

## CI and releases

`.github/workflows/build-and-release.yml` runs on `windows-latest` (the only option — see
build constraints above) and does restore → build → verify → package → release.
`.github/workflows/build-and-release.yml` runs on `windows-latest` and does
publish → verify → package → release.

**Every push to `master` publishes a real release.** The tag is date-based, `v<yyyy.MM.dd>.<n>`,
where `n` continues from the highest tag already published that day; the date is stamped in
`TAG_TIMEZONE` (China Standard Time), not the runner's UTC clock. Pushing a `v*` tag by hand
publishes under that tag verbatim instead. Because the sequence number is derived from existing
tags, the workflow is serialised with a `concurrency` group — do not remove that.

Each release carries two assets: `ImageStore-<tag>.zip` (every `.dll` from `ImageStore\bin\Release`)
and `ImageStore-Database-<tag>.zip` (the empty `.mdf`/`.ldf` and `CreateDatabase.txt`). The database
is deliberately separate — its contents are identical in every release and are only needed once,
when setting up a project.
**A pull request runs the same build but skips publishing** (`if: github.event_name != 'pull_request'`
on the last step) and labels its artifact `pr<n>-<sha>` so it cannot be mistaken for a release.
Use it for anything risky: nothing here can be *run* outside Windows, so CI is the only behavioural
feedback before merging.

Each release carries two assets: `ImageStore-<tag>.zip` (the whole publish tree minus `.pdb`) and
`ImageStore-Database-<tag>.zip` (the empty `.mdf`/`.ldf` and `CreateDatabase.txt`). The database is
deliberately separate — its contents are identical in every release and are only needed once, when
setting up a project. The module archive is copied wholesale rather than as flat `*.dll`, because
`deps.json` and `runtimes/win-*/native/` have to keep their layout.

The "Verify build output" step guards the package in both directions:

- **Nothing missing.** The six third-party dlls must sit next to `ImageStore.dll`, because
`Import-Module` fails at load time if any is absent. They reach the output through `<Reference>`
CopyLocal, *not* through their `<Content>` entries (those carry no `CopyToOutputDirectory` and
copy nothing) — so switching a dependency to a plain `<Content>` item would silently stop
packaging it.
- **Nothing extra.** `System.Management.Automation.dll` must not appear. It comes from the
`Microsoft.PowerShell.5.ReferenceAssemblies` package and is a *reference assembly* — metadata
only, no method bodies — and the PowerShell host supplies the real one at run time.
`<ExcludeAssets>runtime</ExcludeAssets>` on that `PackageReference` keeps it out of the output;
the check is the net for a regression, since an oversized zip otherwise looks perfectly healthy.
v2026.08.15.1 shipped with it by mistake.
- **Nothing missing.** `ImageStore.dll`, `ImageStore.deps.json`, both `Shipwreck.Phash*` dlls,
`Microsoft.Data.SqlClient.dll`, and — checked recursively — the native SNI library.
- **Nothing extra.** No `System.Management.Automation.dll`, `pwrshplugin.dll`,
`PowerShell.Core.Instrumentation.dll`, `libpsl-native.*` or `getfilesiginforedist.dll`. These
belong to the PowerShell host and are kept out by `ExcludeAssets="runtime;native"`; the check is
the net for a regression, since an oversized package otherwise looks perfectly healthy. Both
halves of this have already caught a real mistake — v2026.08.15.1 shipped the reference assembly,
and the native payload leaked in during the .NET 10 upgrade.

The workflow does not touch `AssemblyInfo.cs`: the dll stays at `1.0.0.0` and the version lives
only in the tag, release title, and asset name.
Expand Down
2 changes: 1 addition & 1 deletion ImageStore/Database/CompressDatabaseCmdlet.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
using System;
using System.Collections.Generic;
using System.Data.SqlClient;
using Microsoft.Data.SqlClient;
using System.Linq;
using System.Management.Automation;
using System.Text;
Expand Down
2 changes: 1 addition & 1 deletion ImageStore/DatabaseShared/DatabaseConnection.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
using System;
using System.Collections.Generic;
using System.Data.SqlClient;
using Microsoft.Data.SqlClient;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
Expand Down
2 changes: 1 addition & 1 deletion ImageStore/DatabaseShared/WhereCauseBuilder.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
using System;
using System.Collections.Generic;
using System.Data.SqlClient;
using Microsoft.Data.SqlClient;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
Expand Down
2 changes: 1 addition & 1 deletion ImageStore/Extension/AddExtensionCmdlet.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
using System;
using System.Collections.Generic;
using System.Data.SqlClient;
using Microsoft.Data.SqlClient;
using System.Linq;
using System.Management.Automation;
using System.Text;
Expand Down
Loading